Skip to content

Commit

Permalink
Merge branch 'master' of github.com:airbytehq/airbyte into bmoric/big…
Browse files Browse the repository at this point in the history
…-query-standard-async
  • Loading branch information
benmoriceau committed Oct 2, 2023
2 parents f906e72 + a1ca73a commit 0ee0181
Show file tree
Hide file tree
Showing 2,673 changed files with 54,280 additions and 45,101 deletions.
80 changes: 80 additions & 0 deletions .devcontainer/destination-duckdb/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/python
{
"name": "DuckDB Destination Connector DevContainer (Python)",

// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
"image": "mcr.microsoft.com/devcontainers/python:0-3.9",

// Features to add to the dev container. More info: https://containers.dev/features.
"features": {
"ghcr.io/devcontainers-contrib/features/poetry:2": {},
"ghcr.io/devcontainers/features/docker-in-docker": {}
},
"overrideFeatureInstallOrder": [
// Deterministic order maximizes cache reuse
"ghcr.io/devcontainers-contrib/features/poetry",
"ghcr.io/devcontainers/features/docker-in-docker"
],

"workspaceFolder": "/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb",

// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": [
// Python extensions:
"charliermarsh.ruff",
"matangover.mypy",
"ms-python.black-formatter",
"ms-python.python",
"ms-python.vscode-pylance",

// Toml support
"tamasfe.even-better-toml",

// Yaml and JSON Schema support:
"redhat.vscode-yaml",

// Contributing:
"GitHub.vscode-pull-request-github"
],
"settings": {
"extensions.ignoreRecommendations": true,
"git.autofetch": true,
"git.openRepositoryInParentFolders": "always",
"python.defaultInterpreterPath": ".venv/bin/python",
"python.interpreter.infoVisibility": "always",
"python.terminal.activateEnvironment": true,
"python.testing.pytestEnabled": true,
"python.testing.cwd": "/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb",
"python.testing.pytestArgs": [
"--rootdir=/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb",
"."
],
"[python]": {
"editor.defaultFormatter": "ms-python.black-formatter"
}
}
}
},
"containerEnv": {
"POETRY_VIRTUALENVS_IN_PROJECT": "true"
},

// Mark the root directory as 'safe' for git.
"initializeCommand": "git config --add safe.directory /workspaces/airbyte",

// Use 'postCreateCommand' to run commands after the container is created.
// Post-create tasks:
// 1. Create a symlink directory.
// 2. Create symlinks for the devcontainer.json and docs markdown file.
// 3. Install the Python/Poetry dependencies.
"postCreateCommand": "mkdir -p ./.symlinks && echo '*' > ./.symlinks/.gitignore && ln -sf /workspaces/airbyte/.devcontainer/destination-duckdb/devcontainer.json ./.symlinks/devcontainer.json && ln -sf /workspaces/airbyte/docs/integrations/destinations/duckdb.md ./.symlinks/duckdb-docs.md && poetry install"

// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],

// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
}
58 changes: 58 additions & 0 deletions .devcontainer/java-connectors-generic/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
{
"name": "Connector Development DevContainer (Generic)",

"image": "mcr.microsoft.com/devcontainers/java:0-17",
"features": {
"ghcr.io/devcontainers/features/docker-in-docker": {},
"ghcr.io/devcontainers/features/java:1": {
"installGradle": true,
"version": "latest",
"jdkDistro": "open",
"gradleVersion": "7.5.1"
},
// Python needed for `airbyte-ci` CLI
"ghcr.io/devcontainers/features/python:1": {
"installGradle": true,
"version": "3.10",
"installTools": true
},
"ghcr.io/devcontainers-contrib/features/poetry:2": {}
},

// Deterministic order reduces cache busting
"overrideFeatureInstallOrder": [
"ghcr.io/devcontainers/features/docker-in-docker",
"ghcr.io/devcontainers/features/java",
"ghcr.io/devcontainers/features/python",
"ghcr.io/devcontainers-contrib/features/poetry"
],

// Configure tool-specific properties.
"customizations": {
"vscode": {
"extensions": ["vscjava.vscode-gradle", "tamasfe.even-better-toml"],
"settings": {
"extensions.ignoreRecommendations": true,
"git.openRepositoryInParentFolders": "always"
}
}
},

// Mark the root directory as 'safe' for git.
"initializeCommand": "git config --add safe.directory /workspaces/airbyte",

// Install Gradle, `airbyte-ci` CLI, and Dagger (installed via airbyte-ci --help)
"postCreateCommand": "./gradlew --version && pipx install --editable ./airbyte-ci/connectors/pipelines/ || airbyte-ci --help || true",

"containerEnv": {
// Deterministic Poetry virtual env location: `./.venv`
"POETRY_VIRTUALENVS_IN_PROJECT": "true"
}

// Override to change the directory that the IDE opens by default:
// "workspaceFolder": "/workspaces/airbyte"

// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "root"
}
47 changes: 0 additions & 47 deletions .github/actions/cache-build-artifacts/action.yml

This file was deleted.

16 changes: 8 additions & 8 deletions .github/actions/runner-prepare-for-build/action.yml
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
name: 'Prepare Runner for Building'
description: 'Prepare Runner for Building project'
name: "Prepare Runner for Building"
description: "Prepare Runner for Building project"
inputs:
install_java:
description: ''
description: ""
required: false
default: 'true'
default: "true"
install_node:
description: ''
description: ""
required: false
default: 'true'
default: "true"
install_python:
description: ''
description: ""
required: false
default: 'true'
default: "true"
runs:
using: "composite"
steps:
Expand Down
12 changes: 12 additions & 0 deletions .github/workflows/airbyte-ci-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,15 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/pipelines"
- name: Run airbyte-ci/connectors/base_images tests
id: run-airbyte-ci-connectors-base-images-tests
uses: ./.github/actions/run-dagger-pipeline
with:
context: "pull_request"
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
subcommand: "test airbyte-ci/connectors/base_images"
9 changes: 5 additions & 4 deletions .github/workflows/connector-performance-command.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,9 +92,10 @@ jobs:
if: steps.regex.outputs.first_match != github.event.inputs.connector
run: echo "The connector provided has an invalid format!" && exit 1
- name: Filter supported connectors
if: "${{ github.event.inputs.connector != 'connectors/source-postgres' &&
github.event.inputs.connector != 'connectors/source-mysql' &&
github.event.inputs.connector != 'connectors/destination-snowflake' }}"
if:
"${{ github.event.inputs.connector != 'connectors/source-postgres' &&
github.event.inputs.connector != 'connectors/source-mysql' &&
github.event.inputs.connector != 'connectors/destination-snowflake' }}"
run: echo "Only connectors/source-postgres, source-mysql and destination-snowflake currently supported by harness" && exit 1
- name: Checkout Airbyte
uses: actions/checkout@v3
Expand Down Expand Up @@ -181,7 +182,7 @@ jobs:
- name: Link comment to workflow run
uses: peter-evans/create-or-update-comment@v2
with:
reactions: '+1'
reactions: "+1"
comment-id: ${{ github.event.inputs.comment-id }}
body: |
## Performance test Result:
Expand Down
11 changes: 10 additions & 1 deletion .github/workflows/connector_checklist_require.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,16 @@
name: Require Connector Checklist
on:
pull_request:
types: [opened, edited, synchronize, labeled, unlabeled, reopened, ready_for_review]
types:
[
opened,
edited,
synchronize,
labeled,
unlabeled,
reopened,
ready_for_review,
]
paths:
- "airbyte-integrations/connectors/source-**"
- "airbyte-integrations/connectors/destination-**"
Expand Down
1 change: 0 additions & 1 deletion .github/workflows/connector_teams_review_requirements.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,3 @@ jobs:
status: ${{ steps.get-mandatory-reviewers.outputs.MANDATORY_REVIEWERS }}
token: ${{ secrets.OCTAVIA_4_ROOT_ACCESS }}
requirements-file: .github/connector_org_review_requirements.yaml

37 changes: 16 additions & 21 deletions .github/workflows/connectors_tests.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
name: Connector Ops CI - Connectors Acceptance Tests

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
# This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
#
# - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
# ensuring that only one run per pull request is active at a time.
#
# - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
# This allows us to run multiple workflow dispatches in parallel.
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

on:
workflow_dispatch:
Expand All @@ -25,23 +33,16 @@ jobs:
steps:
- name: Checkout Airbyte
uses: actions/checkout@v3
- name: Check PAT rate limits
run: |
./tools/bin/find_non_rate_limited_PAT \
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
- name: Extract branch name [WORKFLOW DISPATCH]
shell: bash
if: github.event_name == 'workflow_dispatch'
run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
id: extract_branch
# - name: Format connectors [PULL REQUESTS]
# if: github.event_name == 'pull_request'
# uses: ./.github/actions/run-dagger-pipeline
# with:
# context: "pull_request"
# docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
# docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
# gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
# git_branch: ${{ github.head_ref }}
# git_revision: ${{ github.sha }}
# github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
# subcommand: "connectors --modified format"
- name: Fetch last commit id from remote branch [PULL REQUESTS]
if: github.event_name == 'pull_request'
id: fetch_last_commit_id_pr
Expand All @@ -50,12 +51,6 @@ jobs:
if: github.event_name == 'workflow_dispatch'
id: fetch_last_commit_id_wd
run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
- name: Pull formatting changes [PULL REQUESTS]
if: github.event_name == 'pull_request'
uses: actions/checkout@v3
with:
repository: ${{ github.event.inputs.repo }}
ref: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
- name: Test connectors [WORKFLOW DISPATCH]
if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/run-dagger-pipeline
Expand All @@ -67,7 +62,7 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
git_branch: ${{ steps.extract_branch.outputs.branch }}
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_token: ${{ env.PAT }}
subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test"
- name: Test connectors [PULL REQUESTS]
if: github.event_name == 'pull_request'
Expand All @@ -80,5 +75,5 @@ jobs:
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
git_branch: ${{ github.head_ref }}
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
github_token: ${{ secrets.GITHUB_TOKEN }}
github_token: ${{ env.PAT }}
subcommand: "connectors --modified test"
2 changes: 1 addition & 1 deletion .github/workflows/connectors_weekly_build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,4 @@ jobs:
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
git_branch: ${{ steps.extract_branch.outputs.branch }}
github_token: ${{ secrets.GITHUB_TOKEN }}
subcommand: "--show-dagger-logs connectors ${{ inputs.test-connectors-options || '--concurrency=3 --metadata-query=\"(data.ab_internal.ql > 100) & (data.ab_internal.sl < 200)\"' }} test"
subcommand: '--show-dagger-logs connectors ${{ inputs.test-connectors-options || ''--concurrency=3 --metadata-query="(data.ab_internal.ql > 100) & (data.ab_internal.sl < 200)"'' }} test'
Loading

0 comments on commit 0ee0181

Please sign in to comment.