From 4baf3262833c4035992980d2089316559b3be048 Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Sat, 17 Oct 2020 21:55:08 -0700 Subject: [PATCH] fix codegen (#605) --- .../airbyte_protocol/models/__init__.py | 10 ++------ .../models/airbyte_catalog.py | 19 -------------- .../models/airbyte_connection_status.py | 25 ------------------- .../airbyte_protocol/models/airbyte_type.py | 13 ---------- .../bin/generate-protocol-files.sh | 25 +++++++++++++------ airbyte-integrations/base-python/build.gradle | 5 ++-- 6 files changed, 22 insertions(+), 75 deletions(-) delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/models/airbyte_catalog.py delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/models/airbyte_connection_status.py delete mode 100644 airbyte-integrations/base-python/airbyte_protocol/models/airbyte_type.py diff --git a/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py b/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py index d23b3b625db3..ae7a8cfb1ccc 100644 --- a/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py +++ b/airbyte-integrations/base-python/airbyte_protocol/models/__init__.py @@ -1,8 +1,2 @@ -from .airbyte_catalog import AirbyteStream -from .airbyte_catalog import AirbyteCatalog -from .airbyte_connection_status import AirbyteConnectionStatus -from .airbyte_message import AirbyteMessage -from .airbyte_message import AirbyteLogMessage -from .airbyte_message import AirbyteRecordMessage -from .airbyte_message import AirbyteStateMessage -from .airbyte_type import AirbyteType +# generated by generate-protocol-files +from .airbyte_message import * diff --git a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_catalog.py b/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_catalog.py deleted file mode 100644 index 12349bbf8329..000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_catalog.py +++ /dev/null @@ -1,19 +0,0 @@ -# generated by datamodel-codegen: -# filename: airbyte_catalog.yaml - -from __future__ import annotations - -from typing import Any, Dict, List - -from pydantic import BaseModel, Field - - -class AirbyteStream(BaseModel): - name: str = Field(..., description="Stream's name.") - json_schema: Dict[str, Any] = Field( - ..., description='Stream schema using Json Schema specs.' - ) - - -class AirbyteCatalog(BaseModel): - streams: List[AirbyteStream] diff --git a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_connection_status.py b/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_connection_status.py deleted file mode 100644 index b3b2ff37b8ad..000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_connection_status.py +++ /dev/null @@ -1,25 +0,0 @@ -# generated by datamodel-codegen: -# filename: airbyte_connection_status.yaml - -from __future__ import annotations - -from enum import Enum -from typing import List, Optional - -from pydantic import BaseModel - - -class Status(Enum): - SUCCESS = 'SUCCESS' - FAILED = 'FAILED' - - -class SpecFieldMessage(BaseModel): - specFieldName: str - message: str - - -class AirbyteConnectionStatus(BaseModel): - status: Status - message: Optional[str] = None - fieldMessages: Optional[List[SpecFieldMessage]] = None diff --git a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_type.py b/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_type.py deleted file mode 100644 index e44ee64090d8..000000000000 --- a/airbyte-integrations/base-python/airbyte_protocol/models/airbyte_type.py +++ /dev/null @@ -1,13 +0,0 @@ -# generated by datamodel-codegen: -# filename: airbyte_type.yaml - -from __future__ import annotations - -from enum import Enum - - -class AirbyteType(Enum): - boolean = 'boolean' - number = 'number' - text = 'text' - object = 'object' diff --git a/airbyte-integrations/base-python/bin/generate-protocol-files.sh b/airbyte-integrations/base-python/bin/generate-protocol-files.sh index 9d13d65f6ce9..72c822e992ab 100755 --- a/airbyte-integrations/base-python/bin/generate-protocol-files.sh +++ b/airbyte-integrations/base-python/bin/generate-protocol-files.sh @@ -2,15 +2,24 @@ set -e +[ -z "$ROOT_DIR" ] && exit 1 + YAML_DIR=airbyte-protocol/models/src/main/resources/airbyte_protocol OUTPUT_DIR=airbyte-integrations/base-python/airbyte_protocol/models -FILES=$(cd ../../$YAML_DIR; ls -1 | grep yaml) -for f in $FILES -do - FILE_WITHOUT_EXTENSION=""$(echo -n $f | cut -d'.' -f 1) - docker run -v "$(pwd)"/../..:/airbyte airbyte/code-generator:dev \ - --input airbyte/"$YAML_DIR"/"$FILE_WITHOUT_EXTENSION".yaml \ - --output airbyte/"$OUTPUT_DIR"/"$FILE_WITHOUT_EXTENSION".py \ - --disable-timestamp +function main() { + rm -rf "$ROOT_DIR/$OUTPUT_DIR"/*.py + echo "# generated by generate-protocol-files" > "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + for f in "$ROOT_DIR/$YAML_DIR"/*.yaml; do + filename_wo_ext=$(basename "$f" | cut -d . -f 1) + echo "from .$filename_wo_ext import *" >> "$ROOT_DIR/$OUTPUT_DIR"/__init__.py + + docker run -v "$ROOT_DIR":/airbyte airbyte/code-generator:dev \ + --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ + --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ + --disable-timestamp done +} + +main "$@" diff --git a/airbyte-integrations/base-python/build.gradle b/airbyte-integrations/base-python/build.gradle index 2ad536c93fc5..8e829424f35e 100644 --- a/airbyte-integrations/base-python/build.gradle +++ b/airbyte-integrations/base-python/build.gradle @@ -1,10 +1,11 @@ apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') task generateProtocolClassFiles(type: Exec) { - workingDir '.' + environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/generate-protocol-files.sh' - dependsOn ':airbyte-integrations:base-python:code-generator:buildImage' + outputs.upToDateWhen { false } + dependsOn ':airbyte-integrations:base-python:code-generator:buildImage' } build.dependsOn generateProtocolClassFiles