Skip to content

Commit

Permalink
chore: auto-fix lint and format issues
Browse files Browse the repository at this point in the history
  • Loading branch information
octavia-squidington-iii committed Dec 18, 2024
1 parent f02fc11 commit 0303939
Show file tree
Hide file tree
Showing 1,555 changed files with 7,445 additions and 5,799 deletions.
3 changes: 2 additions & 1 deletion airbyte-ci/connectors/base_images/base_images/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
import dagger
import inquirer # type: ignore
import semver
from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry
from jinja2 import Environment, FileSystemLoader

from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry


async def _generate_docs(dagger_client: dagger.Client):
"""This function will generate the README.md file from the templates/README.md.j2 template.
Expand Down
4 changes: 2 additions & 2 deletions airbyte-ci/connectors/base_images/base_images/consts.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

"""This module declares constants used by the base_images module.
"""
"""This module declares constants used by the base_images module."""

import dagger


REMOTE_REGISTRY = "docker.io"
PLATFORMS_WE_PUBLISH_FOR = (dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64"))
3 changes: 1 addition & 2 deletions airbyte-ci/connectors/base_images/base_images/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#

"""This module contains the exceptions used by the base_images module.
"""
"""This module contains the exceptions used by the base_images module."""

from typing import Union

Expand Down
1 change: 1 addition & 0 deletions airbyte-ci/connectors/base_images/base_images/hacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import dagger


# If we perform addition dagger operations on the container, we need to make sure that a mapping exists for the new field name.
DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION = {
"from": lambda field: f'FROM {field.args.get("address")}',
Expand Down
7 changes: 4 additions & 3 deletions airbyte-ci/connectors/base_images/base_images/java/bases.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import Callable, Final

import dagger

from base_images import bases, published_image
from base_images import sanity_checks as base_sanity_checks
from base_images.python import sanity_checks as python_sanity_checks
Expand All @@ -22,9 +23,9 @@ class AirbyteJavaConnectorBaseImage(bases.AirbyteConnectorBaseImage):

DD_AGENT_JAR_URL: Final[str] = "https://dtdg.co/latest-java-tracer"
BASE_SCRIPT_URL = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh"
JAVA_BASE_SCRIPT_URL: Final[
str
] = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh"
JAVA_BASE_SCRIPT_URL: Final[str] = (
"https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh"
)

def get_container(self, platform: dagger.Platform) -> dagger.Container:
"""Returns the container used to build the base image for java connectors
Expand Down
1 change: 1 addition & 0 deletions airbyte-ci/connectors/base_images/base_images/publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@


import dagger

from base_images import bases, consts, published_image


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import Callable, Final

import dagger

from base_images import bases, published_image
from base_images import sanity_checks as base_sanity_checks
from base_images.python import sanity_checks as python_sanity_checks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#

import dagger

from base_images import errors
from base_images import sanity_checks as base_sanity_checks

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from .published_image import PublishedImage


PYTHON_3_9_18 = PublishedImage(
registry="docker.io",
repository="python",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import Optional

import dagger

from base_images import errors


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import List, Tuple

import dagger

from base_images import console, published_image


Expand All @@ -31,7 +32,6 @@ def get_credentials() -> Tuple[str, str]:


class CraneClient:

CRANE_IMAGE_ADDRESS = "gcr.io/go-containerregistry/crane/debug:c195f151efe3369874c72662cd69ad43ee485128@sha256:94f61956845714bea3b788445454ae4827f49a90dcd9dac28255c4cccb6220ad"

def __init__(self, dagger_client: dagger.Client, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@

import dagger
import semver

from base_images import consts, published_image
from base_images.bases import AirbyteConnectorBaseImage
from base_images.java.bases import AirbyteJavaConnectorBaseImage
from base_images.python.bases import AirbyteManifestOnlyConnectorBaseImage, AirbytePythonConnectorBaseImage
from base_images.utils import docker
from connector_ops.utils import ConnectorLanguage # type: ignore


MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage, AirbyteJavaConnectorBaseImage]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from . import SecretsManager


logger = Logger()

ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from dataclasses import dataclass


DEFAULT_SECRET_FILE = "config"


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret


DEFAULT_SECRET_FILE_WITH_EXT = DEFAULT_SECRET_FILE + ".json"

GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

from .logger import Logger


TOKEN_TTL = 3600


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
from typing import Dict, List, Optional, Set, Tuple, Union

import yaml

from connector_ops import utils


# The breaking change reviewers is still in active use.
BREAKING_CHANGE_REVIEWERS = {"breaking-change-reviewers"}
CERTIFIED_MANIFEST_ONLY_CONNECTOR_REVIEWERS = {"dev-python"}
Expand Down
1 change: 1 addition & 0 deletions airbyte-ci/connectors/connector_ops/connector_ops/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from rich.console import Console
from simpleeval import simple_eval


console = Console()

DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import dagger
from anyio import Semaphore
from connector_ops.utils import Connector # type: ignore

from connectors_insights.insights import generate_insights_for_connector
from connectors_insights.result_backends import GCSBucket, LocalDir
from connectors_insights.utils import gcs_uri_to_bucket_key, get_all_connectors_in_directory, remove_strict_encrypt_suffix
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import TYPE_CHECKING

import requests

from connectors_insights.hacks import get_ci_on_master_report
from connectors_insights.models import ConnectorInsights
from connectors_insights.pylint import get_pylint_output
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from typing import TYPE_CHECKING

from connector_ops.utils import ConnectorLanguage # type: ignore

from connectors_insights.utils import never_fail_exec

if TYPE_CHECKING:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from typing import TYPE_CHECKING

from connector_ops.utils import Connector # type: ignore

from connectors_qa.models import Check, CheckCategory, CheckResult

if TYPE_CHECKING:
Expand All @@ -27,7 +28,6 @@ class CheckConnectorIconIsAvailable(AssetsCheck):
requires_metadata = False

def _check_is_valid_svg(self, icon_path: Path) -> Tuple[bool, str | None]:

try:
# Ensure the file has an .svg extension
if not icon_path.suffix.lower() == ".svg":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@

import requests # type: ignore
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore

from connectors_qa.models import Check, CheckCategory, CheckResult

from .helpers import (
generate_description,
prepare_changelog_to_compare,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,10 @@

import toml
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore

from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore


class MetadataCheck(Check):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
import semver
import toml
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from pydash.objects import get # type: ignore

from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore


class PackagingCheck(Check):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@
from typing import Iterable, Optional, Set, Tuple

from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from pydash.objects import get # type: ignore

from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore


class SecurityCheck(Check):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@


from connector_ops.utils import Connector # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.collections import find # type: ignore

from connectors_qa.models import Check, CheckCategory, CheckResult


class TestingCheck(Check):
category = CheckCategory.TESTING
Expand Down
3 changes: 2 additions & 1 deletion airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@
import asyncclick as click
import asyncer
from connector_ops.utils import Connector # type: ignore
from jinja2 import Environment, PackageLoader, select_autoescape

from connectors_qa.checks import ENABLED_CHECKS
from connectors_qa.consts import CONNECTORS_QA_DOC_TEMPLATE_NAME
from connectors_qa.models import Check, CheckCategory, CheckResult, CheckStatus, Report
from connectors_qa.utils import get_all_connectors_in_directory, remove_strict_encrypt_suffix
from jinja2 import Environment, PackageLoader, select_autoescape


# HELPERS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from typing import Dict, List, Optional

from connector_ops.utils import Connector, ConnectorLanguage # type: ignore

from connectors_qa import consts

ALL_LANGUAGES = [
Expand Down Expand Up @@ -289,9 +290,9 @@ def to_json(self) -> str:
" ", "_"
)
connectors_report[connector_technical_name]["badge_text"] = badge_text
connectors_report[connector_technical_name][
"badge_url"
] = f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}"
connectors_report[connector_technical_name]["badge_url"] = (
f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}"
)
return json.dumps(
{
"generation_timestamp": datetime.utcnow().isoformat(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from typing import Set

from connector_ops.utils import Connector # type: ignore

from connectors_qa import consts


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,23 +6,28 @@
import git
import pytest
from asyncclick.testing import CliRunner

from connectors_qa.cli import generate_documentation

DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO = Path("docs/contributing-to-airbyte/resources/qa-checks.md")


@pytest.fixture
def airbyte_repo():
return git.Repo(search_parent_directories=True)


@pytest.mark.asyncio
async def test_generated_qa_checks_documentation_is_up_to_date(airbyte_repo, tmp_path):
# Arrange
current_doc = (airbyte_repo.working_dir / DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO).read_text()
newly_generated_doc_path = tmp_path / "qa-checks.md"

# Act
await CliRunner().invoke(generate_documentation, [str(tmp_path / "qa-checks.md")], catch_exceptions=False)

# Assert
suggested_command = f"connectors-qa generate-documentation {DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO}"
assert newly_generated_doc_path.read_text() == current_doc, f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes."
assert (
newly_generated_doc_path.read_text() == current_doc
), f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes."
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ def test_fail_when_icon_path_is_not_named_icon_svg(self, tmp_path, mocker):
assert result.status == CheckStatus.FAILED
assert result.message == "Icon file is not a SVG file"


def test_fail_when_icon_file_is_not_valid_svg(self, tmp_path, mocker):
# Arrange
connector = mocker.MagicMock()
Expand Down
Loading

0 comments on commit 0303939

Please sign in to comment.