-
Notifications
You must be signed in to change notification settings - Fork 4.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Source GitHub: migrate repo and branches to array in spec (#31056)
Co-authored-by: artem1205 <[email protected]>
- Loading branch information
Showing
11 changed files
with
282 additions
and
57 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
airbyte-integrations/connectors/source-github/source_github/config_migrations.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
import abc | ||
import logging | ||
from abc import ABC | ||
from typing import Any, List, Mapping | ||
|
||
from airbyte_cdk.config_observation import create_connector_config_control_message | ||
from airbyte_cdk.entrypoint import AirbyteEntrypoint | ||
from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository | ||
|
||
from .source import SourceGithub | ||
|
||
logger = logging.getLogger("airbyte_logger") | ||
|
||
|
||
class MigrateStringToArray(ABC): | ||
""" | ||
This class stands for migrating the config at runtime, | ||
while providing the backward compatibility when falling back to the previous source version. | ||
Specifically, starting from `1.4.6`, the `repository` and `branch` properties should be like : | ||
> List(["<repository_1>", "<repository_2>", ..., "<repository_n>"]) | ||
instead of, in `1.4.5`: | ||
> JSON STR: "repository_1 repository_2" | ||
""" | ||
|
||
message_repository: MessageRepository = InMemoryMessageRepository() | ||
|
||
@property | ||
@abc.abstractmethod | ||
def migrate_from_key(self) -> str: | ||
... | ||
|
||
@property | ||
@abc.abstractmethod | ||
def migrate_to_key(self) -> str: | ||
... | ||
|
||
@classmethod | ||
def _should_migrate(cls, config: Mapping[str, Any]) -> bool: | ||
""" | ||
This method determines whether config require migration. | ||
Returns: | ||
> True, if the transformation is necessary | ||
> False, otherwise. | ||
""" | ||
if cls.migrate_from_key in config and cls.migrate_to_key not in config: | ||
return True | ||
return False | ||
|
||
@classmethod | ||
def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGithub = None) -> Mapping[str, Any]: | ||
# assign old values to new property that will be used within the new version | ||
config[cls.migrate_to_key] = config[cls.migrate_to_key] if cls.migrate_to_key in config else [] | ||
data = set(filter(None, config.get(cls.migrate_from_key).split(" "))) | ||
config[cls.migrate_to_key] = list(data | set(config[cls.migrate_to_key])) | ||
return config | ||
|
||
@classmethod | ||
def _modify_and_save(cls, config_path: str, source: SourceGithub, config: Mapping[str, Any]) -> Mapping[str, Any]: | ||
# modify the config | ||
migrated_config = cls._transform_to_array(config, source) | ||
# save the config | ||
source.write_config(migrated_config, config_path) | ||
# return modified config | ||
return migrated_config | ||
|
||
@classmethod | ||
def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: | ||
# add the Airbyte Control Message to message repo | ||
cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) | ||
# emit the Airbyte Control Message from message queue to stdout | ||
for message in cls.message_repository._message_queue: | ||
print(message.json(exclude_unset=True)) | ||
|
||
@classmethod | ||
def migrate(cls, args: List[str], source: SourceGithub) -> None: | ||
""" | ||
This method checks the input args, should the config be migrated, | ||
transform if necessary and emit the CONTROL message. | ||
""" | ||
# get config path | ||
config_path = AirbyteEntrypoint(source).extract_config(args) | ||
# proceed only if `--config` arg is provided | ||
if config_path: | ||
# read the existing config | ||
config = source.read_config(config_path) | ||
# migration check | ||
if cls._should_migrate(config): | ||
cls._emit_control_message( | ||
cls._modify_and_save(config_path, source, config), | ||
) | ||
|
||
|
||
class MigrateRepository(MigrateStringToArray): | ||
|
||
migrate_from_key: str = "repository" | ||
migrate_to_key: str = "repositories" | ||
|
||
|
||
class MigrateBranch(MigrateStringToArray): | ||
|
||
migrate_from_key: str = "branch" | ||
migrate_to_key: str = "branches" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
8 changes: 8 additions & 0 deletions
8
airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
{ | ||
"credentials": { | ||
"personal_access_token": "personal_access_token" | ||
}, | ||
"repository": "airbytehq/airbyte airbytehq/airbyte-platform", | ||
"start_date": "2000-01-01T00:00:00Z", | ||
"branch": "airbytehq/airbyte/master airbytehq/airbyte-platform/main" | ||
} |
76 changes: 76 additions & 0 deletions
76
...ntegrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,76 @@ | ||
# | ||
# Copyright (c) 2023 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
|
||
import json | ||
import os | ||
from typing import Any, Mapping | ||
|
||
from airbyte_cdk.models import OrchestratorType, Type | ||
from airbyte_cdk.sources import Source | ||
from source_github.config_migrations import MigrateBranch, MigrateRepository | ||
from source_github.source import SourceGithub | ||
|
||
# BASE ARGS | ||
CMD = "check" | ||
TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" | ||
NEW_TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_new_config.json" | ||
SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] | ||
SOURCE: Source = SourceGithub() | ||
|
||
|
||
# HELPERS | ||
def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: | ||
with open(config_path, "r") as config: | ||
return json.load(config) | ||
|
||
|
||
def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: | ||
with open(config_path, "r") as test_config: | ||
config = json.load(test_config) | ||
config.pop("repositories") | ||
with open(config_path, "w") as updated_config: | ||
config = json.dumps(config) | ||
updated_config.write(config) | ||
|
||
|
||
def test_migrate_config(): | ||
migration_instance = MigrateRepository | ||
# migrate the test_config | ||
migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) | ||
# load the updated config | ||
test_migrated_config = load_config() | ||
# check migrated property | ||
assert "repositories" in test_migrated_config | ||
assert isinstance(test_migrated_config["repositories"], list) | ||
# check the old property is in place | ||
assert "repository" in test_migrated_config | ||
assert isinstance(test_migrated_config["repository"], str) | ||
# test CONTROL MESSAGE was emitted | ||
control_msg = migration_instance.message_repository._message_queue[0] | ||
assert control_msg.type == Type.CONTROL | ||
assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG | ||
# new repositories is of type(list) | ||
assert isinstance(control_msg.control.connectorConfig.config["repositories"], list) | ||
# check the migrated values | ||
revert_migration() | ||
|
||
|
||
def test_config_is_reverted(): | ||
# check the test_config state, it has to be the same as before tests | ||
test_config = load_config() | ||
# check the config no longer has the migrated property | ||
assert "repositories" not in test_config | ||
assert "branches" not in test_config | ||
# check the old property is still there | ||
assert "repository" in test_config | ||
assert "branch" in test_config | ||
assert isinstance(test_config["repository"], str) | ||
assert isinstance(test_config["branch"], str) | ||
|
||
|
||
def test_should_not_migrate_new_config(): | ||
new_config = load_config(NEW_TEST_CONFIG_PATH) | ||
for instance in MigrateBranch, MigrateRepository: | ||
assert not instance._should_migrate(new_config) |
Oops, something went wrong.