From d9da1172a2e4e7edac237c983eed4ada8c173239 Mon Sep 17 00:00:00 2001 From: Michel Tricot Date: Tue, 26 Jan 2021 18:54:12 -0800 Subject: [PATCH] Quick & dirty setup of a read only airbyte instance (#1802) --- .gitignore | 9 +- .../on-gcp-compute-engine.md | 4 +- terraform/aws/demo/core/init.sh | 38 ++++ terraform/aws/demo/core/main.tf | 58 ++++++ terraform/aws/demo/core/outputs.tf | 3 + terraform/aws/demo/core/variables.tf | 15 ++ terraform/aws/demo/lb/auth.html | 21 ++ terraform/aws/demo/lb/main.tf | 179 ++++++++++++++++++ terraform/aws/demo/lb/variables.tf | 27 +++ terraform/aws/demo/main.tf | 27 +++ terraform/aws/demo/terraform.tf | 16 ++ terraform/aws/demo/variables.tf | 35 ++++ tools/internal/README.md | 9 + tools/internal/demo.sh | 48 +++++ tools/lib/lib.sh | 2 +- 15 files changed, 487 insertions(+), 4 deletions(-) create mode 100644 terraform/aws/demo/core/init.sh create mode 100644 terraform/aws/demo/core/main.tf create mode 100644 terraform/aws/demo/core/outputs.tf create mode 100644 terraform/aws/demo/core/variables.tf create mode 100644 terraform/aws/demo/lb/auth.html create mode 100644 terraform/aws/demo/lb/main.tf create mode 100644 terraform/aws/demo/lb/variables.tf create mode 100644 terraform/aws/demo/main.tf create mode 100644 terraform/aws/demo/terraform.tf create mode 100644 terraform/aws/demo/variables.tf create mode 100644 tools/internal/README.md create mode 100755 tools/internal/demo.sh diff --git a/.gitignore b/.gitignore index 5f25693e0496..d6cafd854f0c 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,11 @@ __pycache__ .ipynb_checkpoints # dbt -profiles.yml \ No newline at end of file +profiles.yml + +# Terraform +.terraform/ +crash.log +*.tfstate +*.tfstate.backup +*.lock.hcl diff --git a/docs/deploying-airbyte/on-gcp-compute-engine.md b/docs/deploying-airbyte/on-gcp-compute-engine.md index b7fdb01cc673..be6d55ec76c4 100644 --- a/docs/deploying-airbyte/on-gcp-compute-engine.md +++ b/docs/deploying-airbyte/on-gcp-compute-engine.md @@ -60,7 +60,7 @@ gcloud --project $PROJECT_ID compute instances list ```bash # In your workstation terminal -gcloud --project=$PROJECT_ID beta compute ssh airbyte +gcloud --project=$PROJECT_ID beta compute ssh $INSTANCE_NAME ``` * Install `docker` @@ -99,7 +99,7 @@ logout ```bash # In your workstation terminal -gcloud --project=$PROJECT_ID beta compute ssh airbyte +gcloud --project=$PROJECT_ID beta compute ssh $INSTANCE_NAME ``` * Install Airbyte diff --git a/terraform/aws/demo/core/init.sh b/terraform/aws/demo/core/init.sh new file mode 100644 index 000000000000..21f9870f7e94 --- /dev/null +++ b/terraform/aws/demo/core/init.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -ex + +install_init() { + sudo yum update -y +} + +install_docker() { + sudo yum install -y docker + sudo service docker start + sudo usermod -a -G docker ec2-user +} + +install_docker_compose() { + sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + docker-compose --version +} + +install_airbyte() { + mkdir airbyte && cd airbyte + wget https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml} + API_URL=/api/v1/ AIRBYTE_ROLE=demo IS_DEMO=true docker-compose up -d +} + +install_demo_pg() { + docker run --rm --name postgres-demo -e POSTGRES_PASSWORD=password -p 3000:5432 -d postgres +} + +main() { + install_init + install_docker + install_docker_compose + install_airbyte +} + +main > /tmp/init.log 2>&1 diff --git a/terraform/aws/demo/core/main.tf b/terraform/aws/demo/core/main.tf new file mode 100644 index 000000000000..97df8d121699 --- /dev/null +++ b/terraform/aws/demo/core/main.tf @@ -0,0 +1,58 @@ +data "aws_security_group" "default-sg" { + id = var.default-sg +} + +data "aws_ami" "amazon-linux-2" { + # Hardcoded 'Amazon' owner id + owners = [137112412989] + most_recent = true + + filter { + name = "owner-alias" + values = ["amazon"] + } + + filter { + name = "name" + values = ["amzn2-ami-hvm-2*"] + } +} + +# Ensure we can ssh to the airbyte instance +resource "aws_security_group" "airbyte-ssh-sg" { + name = "${var.name}-airbyte-ssh-sg" + description = "Allow ssh traffic" + + ingress { + description = "ssh" + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} + +resource "aws_instance" "airbyte-instance" { + lifecycle { + # If you need to destroy the instance, make sure you back up the airbyte configuration. + prevent_destroy = true + # So we can edit the init.sh script without having to re-create the instance. + ignore_changes = [user_data] + } + + instance_type = var.instance-size + ami = data.aws_ami.amazon-linux-2.id + + security_groups = [ + data.aws_security_group.default-sg.name, + aws_security_group.airbyte-ssh-sg.name + ] + + key_name = var.key-name + + user_data = file("${path.module}/init.sh") + + tags = { + Name = "${var.name}-airbyte-app" + } +} diff --git a/terraform/aws/demo/core/outputs.tf b/terraform/aws/demo/core/outputs.tf new file mode 100644 index 000000000000..0ba9225513e5 --- /dev/null +++ b/terraform/aws/demo/core/outputs.tf @@ -0,0 +1,3 @@ +output "instance-id" { + value = aws_instance.airbyte-instance.id +} diff --git a/terraform/aws/demo/core/variables.tf b/terraform/aws/demo/core/variables.tf new file mode 100644 index 000000000000..4c1a4399289e --- /dev/null +++ b/terraform/aws/demo/core/variables.tf @@ -0,0 +1,15 @@ +variable "name" { + type = string +} + +variable "default-sg" { + type = string +} + +variable "instance-size" { + type = string +} + +variable "key-name" { + type = string +} diff --git a/terraform/aws/demo/lb/auth.html b/terraform/aws/demo/lb/auth.html new file mode 100644 index 000000000000..0c3894faea32 --- /dev/null +++ b/terraform/aws/demo/lb/auth.html @@ -0,0 +1,21 @@ + + + + +
+ + + + +
+ + + + + diff --git a/terraform/aws/demo/lb/main.tf b/terraform/aws/demo/lb/main.tf new file mode 100644 index 000000000000..65a8a5d79064 --- /dev/null +++ b/terraform/aws/demo/lb/main.tf @@ -0,0 +1,179 @@ +data "aws_security_group" "default-sg" { + id = var.default-sg +} + +data "aws_vpc" "vpc" { + id = var.vpc +} + +resource "aws_security_group" "airbyte-alb-sg" { + name = "${var.name}-airbyte-alb-sg" + description = "Allow traffic to the elb" + + ingress { + description = "https" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } +} + +# Create target groups + +resource "aws_lb_target_group" "airbyte-webapp" { + name = "${var.name}-airbyte-webapp-tg" + port = 8000 + protocol = "HTTP" + vpc_id = data.aws_vpc.vpc.id + + health_check { + path = "/" + } +} + +resource "aws_lb_target_group_attachment" "airbyte-webapp" { + target_group_arn = aws_lb_target_group.airbyte-webapp.arn + target_id = var.instance-id + port = 8000 +} + +resource "aws_lb_target_group" "airbyte-api" { + name = "${var.name}-airbyte-api-tg" + port = 8001 + protocol = "HTTP" + vpc_id = data.aws_vpc.vpc.id + + health_check { + path = "/api/v1/health" + } +} + +resource "aws_lb_target_group_attachment" "airbyte-api" { + target_group_arn = aws_lb_target_group.airbyte-api.arn + target_id = var.instance-id + port = 8001 +} + +# Build load balancer + +resource "aws_lb" "airbyte-alb" { + enable_deletion_protection = true + + name = "${var.name}-airbyte-alb" + + internal = false + load_balancer_type = "application" + security_groups = [ + data.aws_security_group.default-sg.id, + aws_security_group.airbyte-alb-sg.id + ] + subnets = var.subnets +} + +resource "aws_lb_listener" "airbyte-alb-listener" { + load_balancer_arn = aws_lb.airbyte-alb.arn + port = "443" + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-2016-08" + certificate_arn = var.certificate + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.airbyte-webapp.arn + } +} + +# By default we deny all api calls +resource "aws_lb_listener_rule" "deny-all-api" { + listener_arn = aws_lb_listener.airbyte-alb-listener.arn + priority = 100 + + action { + type = "fixed-response" + + fixed_response { + content_type = "application/json" + message_body = "{}" + status_code = "401" + } + } + + condition { + path_pattern { + values = ["/api/v1/*"] + } + } +} + +# Then we allow all the read endpoints +resource "aws_lb_listener_rule" "allow-read-api" { + listener_arn = aws_lb_listener.airbyte-alb-listener.arn + priority = 99 + + action { + type = "forward" + target_group_arn = aws_lb_target_group.airbyte-api.arn + } + + condition { + path_pattern { + values = [ + "/api/v1/*/list", + "/api/v1/*/get", + "/api/v1/*/get_by_slug", + "/api/v1/*/health", + ] + } + } +} + +# Check for secret cookie to enable write +resource "aws_lb_listener_rule" "allow-all-api" { + listener_arn = aws_lb_listener.airbyte-alb-listener.arn + priority = 98 + + action { + type = "forward" + target_group_arn = aws_lb_target_group.airbyte-api.arn + } + + condition { + http_header { + http_header_name = "cookie" + values = ["*hack-auth-token=${var.auth-secret}*"] + } + } + + condition { + path_pattern { + values = [ + "/api/v1/*" + ] + } + } +} + +# Auth hack + +# By default we deny all api calls +resource "aws_lb_listener_rule" "auth-hack" { + listener_arn = aws_lb_listener.airbyte-alb-listener.arn + priority = 97 + + action { + type = "fixed-response" + + fixed_response { + content_type = "text/html" + message_body = file("${path.module}/auth.html") + status_code = "200" + } + } + + condition { + path_pattern { + values = ["/hack/auth"] + } + } +} diff --git a/terraform/aws/demo/lb/variables.tf b/terraform/aws/demo/lb/variables.tf new file mode 100644 index 000000000000..9e9e20a3a7e0 --- /dev/null +++ b/terraform/aws/demo/lb/variables.tf @@ -0,0 +1,27 @@ +variable "name" { + type = string +} + +variable "vpc" { + type = string +} + +variable "default-sg" { + type = string +} + +variable "subnets" { + type = list(string) +} + +variable "certificate" { + type = string +} + +variable "instance-id" { + type = string +} + +variable "auth-secret" { + type = string +} diff --git a/terraform/aws/demo/main.tf b/terraform/aws/demo/main.tf new file mode 100644 index 000000000000..53e96f0d49f5 --- /dev/null +++ b/terraform/aws/demo/main.tf @@ -0,0 +1,27 @@ +provider "aws" { + region = "us-east-1" + shared_credentials_file = "~/.aws/credentials" +} + +module "airbyte-instance" { + source = "./core" + + name = var.name + + default-sg = var.default-sg + instance-size = var.instance-size + key-name = var.key-name +} + +module "public-lb" { + source = "./lb" + + name = var.name + + vpc = var.vpc + subnets = var.subnets + default-sg = var.default-sg + certificate = var.certificate + instance-id = module.airbyte-instance.instance-id + auth-secret = var.auth-secret +} diff --git a/terraform/aws/demo/terraform.tf b/terraform/aws/demo/terraform.tf new file mode 100644 index 000000000000..2ced3c129414 --- /dev/null +++ b/terraform/aws/demo/terraform.tf @@ -0,0 +1,16 @@ +terraform { + backend "remote" { + organization = "airbyte" + + workspaces { + name = "aws-demo" + } + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "3.24.1" + } + } +} diff --git a/terraform/aws/demo/variables.tf b/terraform/aws/demo/variables.tf new file mode 100644 index 000000000000..c5509be1ec74 --- /dev/null +++ b/terraform/aws/demo/variables.tf @@ -0,0 +1,35 @@ +# +# Define all the variables that you might want to override +# + +variable "name" { + type = string +} + +variable "vpc" { + type = string +} + +variable "default-sg" { + type = string +} + +variable "subnets" { + type = list(string) +} + +variable "instance-size" { + type = string +} + +variable "key-name" { + type = string +} + +variable "certificate" { + type = string +} + +variable "auth-secret" { + type = string +} diff --git a/tools/internal/README.md b/tools/internal/README.md new file mode 100644 index 000000000000..4262363017aa --- /dev/null +++ b/tools/internal/README.md @@ -0,0 +1,9 @@ +Scripts in this directory are for Airbyte's employees + +# `demo.sh` +This script helps maintain Airbyte's demo instance: + +```shell +./tools/internal/demo.sh ssh # connects you to the airbyte instance +./tools/internal/demo.sh tunnel # creates a local tunnel so you can access the configurable version of airbyte +``` diff --git a/tools/internal/demo.sh b/tools/internal/demo.sh new file mode 100755 index 000000000000..f22398498981 --- /dev/null +++ b/tools/internal/demo.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -e + +. tools/lib/lib.sh + +TUNNEL_PORT=${TUNNEL_PORT:-4242} +SSH_KEY=${SSH_KEY:-~/.ssh/airbyte-app.pem} +INSTANCE_NAME=${INSTANCE_NAME:-demo-airbyte-app} +PRIVATE_LB_NAME=${PRIVATE_LB_NAME:-demo-airbyte-admin-alb} + +USAGE="$0 (tunnel|ssh)" + +_get_instance_ip() { + aws --region us-east-1 ec2 describe-instances --filters Name=tag:Name,Values="$INSTANCE_NAME" Name=instance-state-name,Values=running --query "Reservations[0].Instances[0].[PublicIpAddress]" --output text +} + +_get_private_alb_dns() { + aws --region us-east-1 elbv2 describe-load-balancers --names "$PRIVATE_LB_NAME" --query "LoadBalancers[0].[DNSName]" --output text +} + +cmd_tunnel() { + local instance_ip; instance_ip=$(_get_instance_ip) + local private_alb_dns; private_alb_dns=$(_get_private_alb_dns) + + echo "Tunnel: (Instance: $instance_ip, ALB: $private_alb_dns)" + + echo "Connect to http://localhost:$TUNNEL_PORT" + ssh -i "$SSH_KEY" -L "$TUNNEL_PORT":"$private_alb_dns":80 -N ec2-user@"$instance_ip" +} + +cmd_ssh() { + local instance_ip; instance_ip=$(_get_instance_ip) + + echo "Ssh: (Instance: $instance_ip)" + + ssh -i "$SSH_KEY" ec2-user@"$instance_ip" +} + +main () { + assert_root + + local cmd=$1; shift || error "Missing command\n\n$USAGE" + + cmd_"$cmd" +} + +main "$@" diff --git a/tools/lib/lib.sh b/tools/lib/lib.sh index 91c3ae3a3934..605bed3df4a1 100644 --- a/tools/lib/lib.sh +++ b/tools/lib/lib.sh @@ -1,5 +1,5 @@ error() { - echo "$@" + echo -e "$@" exit 1 }