Skip to content

Commit

Permalink
🐙 Bootstrap octavia-cli project (airbytehq#9070)
Browse files Browse the repository at this point in the history
  • Loading branch information
alafanechere authored Jan 5, 2022
1 parent 37c4a75 commit 34c9366
Show file tree
Hide file tree
Showing 17 changed files with 320 additions and 9 deletions.
70 changes: 70 additions & 0 deletions .github/workflows/gradle.yml
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,76 @@ jobs:
SLACK_TITLE: "Build failure"
SLACK_FOOTER: ""

- name: Slack Notification - Success
if: success() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildbot
SLACK_TITLE: "Build Success"
SLACK_FOOTER: ""
octavia-cli-build:
runs-on: ubuntu-latest
name: "Octavia CLI: Build"
timeout-minutes: 90
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2

# this intentionally does not use restore-keys so we don't mess with gradle caching
- name: Gradle Caching
uses: actions/cache@v2
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
**/.venv
key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}

- uses: actions/setup-java@v1
with:
java-version: "17"

# octavia-cli install and testing requires Python.
# We use 3.8 in this project because 3.7 is not supported on Apple M1.
- uses: actions/setup-python@v2
with:
python-version: "3.8"

- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Format
run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace

- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"

- name: Build
run: |
SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
- name: Slack Notification - Failure
if: failure() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildozer
SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
SLACK_COLOR: DC143C
SLACK_TITLE: "Build failure"
SLACK_FOOTER: ""

- name: Slack Notification - Success
if: success() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
.gradle
.idea
.vscode
*.iml
*.swp
build
Expand Down
3 changes: 3 additions & 0 deletions octavia-cli/.coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[report]
# show lines missing coverage
show_missing = true
3 changes: 3 additions & 0 deletions octavia-cli/.dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
build
.venv
octavia_cli.egg-info
2 changes: 2 additions & 0 deletions octavia-cli/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
.coverage
.venv
1 change: 1 addition & 0 deletions octavia-cli/.python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.8.12
Empty file added octavia-cli/CHANGELOG.md
Empty file.
17 changes: 17 additions & 0 deletions octavia-cli/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
FROM python:3.8.12-slim as base

RUN apt-get upgrade \
&& pip install --upgrade pip

WORKDIR /home/octavia-cli
COPY . ./

RUN pip install --no-cache-dir .

RUN useradd --create-home --shell /bin/bash octavia-cli
USER octavia-cli

ENTRYPOINT ["octavia"]

LABEL io.airbyte.version=dev
LABEL io.airbyte.name=airbyte/octavia-cli
21 changes: 21 additions & 0 deletions octavia-cli/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
MIT License

Copyright (c) 2020 Airbyte, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
55 changes: 55 additions & 0 deletions octavia-cli/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# 🐙 Octavia CLI

Octavia CLI is a tool to manage Airbyte configuration in YAML.
It has the following features:
* Scaffolding of a readable directory architecture that will host the YAML configs.
* Auto-generation of YAML config file that matches the resources' schemas.
* Manage Airbyte resources with YAML config files.
* Safe resources update through diff display and validation.
* Simple secret management to avoid versioning credentials.

The project is under development: readers can refer to our [tech spec deck](https://docs.google.com/presentation/d/10RjkCzBiVhCivnjSh63icYI7wG6S0N0ZIErEIsmXTqM/edit?usp=sharing) for an introduction to the tool.

# Usage
We encourage users to use the CLI with docker to avoid the hassle of setting up a Python installation.
The project is under development: we have not yet published any docker image to our Docker registry.

1. Build the image locally:
```bash
docker build -t octavia-cli:dev --rm .
```
2. Run the CLI from docker:
```bash
docker run octavia-cli:dev
````
3. Create an `octavia` alias in your `.bashrc` or `.zshrc`:
````bash
echo 'alias octavia="docker run octavia-cli:dev"' >> ~/.zshrc
source ~/.zshrc
octavia
````

# Current development status
Octavia is currently under development.
You can find a detailed and updated execution plan [here](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0).
We welcome community contributions!

Summary of achievements:

| Date | Milestone |
|------------|-------------------------------------|
| 2021-12-22 | Bootstrapping the project's code base |
# Developing locally
1. Install Python 3.10.0. We suggest doing it through `pyenv`
2. Create a virtualenv: `python -m venv .venv`
3. Activate the virtualenv: `source .venv/bin/activate`
4. Install dev dependencies: `pip install -e .\[dev\]`
5. Install `pre-commit` hooks: `pre-commit install`
6. Run the test suite: `pytest --cov=octavia_cli unit_tests`
7. Iterate; please check the [Contributing](#contributing) for instructions on contributing.
# Contributing
1. Please sign up to [Airbyte's Slack workspace](https://slack.airbyte.io/) and join the `#octavia-cli`. We'll sync up community efforts in this channel.
2. Read the [execution plan](https://docs.google.com/spreadsheets/d/1weB9nf0Zx3IR_QvpkxtjBAzyfGb7B0PWpsVt6iMB5Us/edit#gid=0) and find a task you'd like to work on.
3. Open a PR, make sure to test your code thoroughly.
9 changes: 9 additions & 0 deletions octavia-cli/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
plugins {
id 'airbyte-python'
id 'airbyte-docker'
}

airbytePython {
moduleDirectory 'octavia_cli'
}

Empty file.
42 changes: 42 additions & 0 deletions octavia-cli/octavia_cli/entrypoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

import click


@click.group()
@click.option("--airbyte-url", envvar="AIRBYTE_URL", default="http://localhost:8000", help="The URL of your Airbyte instance.")
def octavia(airbyte_url):
# TODO: check if the airbyte_url is reachable
click.secho(f"🐙 - Octavia is targetting your Airbyte instance running at {airbyte_url}")


@octavia.command(help="Scaffolds a local project directories.")
def init():
raise click.ClickException("The init command is not yet implemented.")


@octavia.command(name="list", help="List existing resources on the Airbyte instance.")
def _list():
raise click.ClickException("The init command is not yet implemented.")


@octavia.command(name="import", help="Import an existing resources from the Airbyte instance.")
def _import():
raise click.ClickException("The init command is not yet implemented.")


@octavia.command(help="Generate a YAML configuration file to manage a resource.")
def create():
raise click.ClickException("The init command is not yet implemented.")


@octavia.command(help="Create or update resources according to YAML configurations.")
def apply():
raise click.ClickException("The init command is not yet implemented.")


@octavia.command(help="Delete resources")
def delete():
raise click.ClickException("The init command is not yet implemented.")
5 changes: 5 additions & 0 deletions octavia-cli/pytest.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[pytest]
log_cli = 1
log_cli_level = INFO
log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
log_cli_date_format=%Y-%m-%d %H:%M:%S
53 changes: 53 additions & 0 deletions octavia-cli/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

import pathlib

from setuptools import find_packages, setup

# The directory containing this file
HERE = pathlib.Path(__file__).parent

# The text of the README file
README = (HERE / "README.md").read_text()

setup(
name="octavia-cli",
version="0.1.0",
description="A command line interface to manage Airbyte configurations",
long_description=README,
author="Airbyte",
author_email="[email protected]",
license="MIT",
url="https://github.com/airbytehq/airbyte",
classifiers=[
# This information is used when browsing on PyPi.
# Dev Status
"Development Status :: 3 - Alpha",
# Project Audience
"Intended Audience :: Developers",
"Topic :: Scientific/Engineering",
"Topic :: Software Development :: Libraries :: Python Modules",
"License :: OSI Approved :: MIT License",
# Python Version Support
"Programming Language :: Python :: 3.10",
],
keywords="airbyte cli command-line-interface configuration",
project_urls={
"Documentation": "https://docs.airbyte.io/",
"Source": "https://github.com/airbytehq/airbyte",
"Tracker": "https://github.com/airbytehq/airbyte/issues",
},
packages=find_packages(exclude=("tests", "docs")),
install_requires=["click~=8.0.3"],
python_requires=">=3.8.12",
extras_require={
"dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"],
"sphinx-docs": [
"Sphinx~=4.2",
"sphinx-rtd-theme~=1.0",
],
},
entry_points={"console_scripts": ["octavia=octavia_cli.entrypoint:octavia"]},
)
25 changes: 25 additions & 0 deletions octavia-cli/unit_tests/test_entrypoint.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

import pytest
from click.testing import CliRunner
from octavia_cli import entrypoint


def test_octavia():
runner = CliRunner()
result = runner.invoke(entrypoint.octavia)
assert result.exit_code == 0
assert result.output.startswith("Usage: octavia [OPTIONS] COMMAND [ARGS]...")


@pytest.mark.parametrize(
"command",
[entrypoint.init, entrypoint.apply, entrypoint.create, entrypoint.delete, entrypoint._list, entrypoint._import],
)
def test_not_implemented_commands(command):
runner = CliRunner()
result = runner.invoke(command)
assert result.exit_code == 1
assert result.output.endswith("not yet implemented.\n")
22 changes: 13 additions & 9 deletions settings.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,27 +19,30 @@ sourceControl {

rootProject.name = 'airbyte'

// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE. Blank is equivalent to all.
// SUB_BUILD is an enum of <blank>, PLATFORM, CONNECTORS_BASE, OCTAVIA_CLI. Blank is equivalent to all.
if (!System.getenv().containsKey("SUB_BUILD")) {
println("Building all of Airbyte.")
} else {
def subBuild = System.getenv().get("SUB_BUILD")
println("Building Airbyte Sub Build: " + subBuild)
if (subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE") {
throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE", subBuild))
if (subBuild != "PLATFORM" && subBuild != "CONNECTORS_BASE" && subBuild != "OCTAVIA_CLI") {
throw new IllegalArgumentException(String.format("%s is invalid. Must be unset or PLATFORM or CONNECTORS_BASE or OCTAVIA_CLI", subBuild))
}
}

// shared
include ':airbyte-api'
include ':airbyte-commons'
include ':airbyte-commons-docker'

// shared by CONNECTORS_BASE and PLATFORM sub builds
include ':airbyte-api'
include ':airbyte-commons-cli'
include ':airbyte-commons-docker'
include ':airbyte-config:models' // reused by acceptance tests in connector base.
include ':airbyte-db:lib' // reused by acceptance tests in connector base.
include ':airbyte-json-validation'
include ':airbyte-protocol:models'
include ':airbyte-metrics'
include ':airbyte-oauth'
include ':airbyte-protocol:models'
include ':airbyte-queue'
include ':airbyte-test-utils'

Expand All @@ -60,7 +63,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
include ':airbyte-config:specs'
include ':airbyte-container-orchestrator'
include ':airbyte-webapp-e2e-tests'
include ':airbyte-oauth'
include ':airbyte-scheduler:app'
include ':airbyte-scheduler:client'
include ':airbyte-server'
Expand All @@ -86,8 +88,6 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
include ':airbyte-integrations:connector-templates:generator'
include ':airbyte-integrations:bases:debezium'

include ':airbyte-oauth'

// Needed by normalization integration tests
include ':airbyte-integrations:connectors:destination-bigquery'
include ':airbyte-integrations:connectors:destination-jdbc'
Expand All @@ -106,6 +106,10 @@ if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD"
include ':tools:code-generator'
}

if (!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") == "OCTAVIA_CLI") {
include ':octavia-cli'
}

// connectors
if (!System.getenv().containsKey("SUB_BUILD")) {
// include all connector projects
Expand Down

0 comments on commit 34c9366

Please sign in to comment.