Quick & dirty setup of a read only airbyte instance (airbytehq#1802)

kmichel-aiven · Jan 27, 2021 · d9da117 · d9da117
1 parent de63293
commit d9da117
Show file tree

Hide file tree

Showing 15 changed files with 487 additions and 4 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,4 +17,11 @@ __pycache__
 .ipynb_checkpoints
 
 # dbt
-profiles.yml
+profiles.yml
+
+# Terraform
+.terraform/
+crash.log
+*.tfstate
+*.tfstate.backup
+*.lock.hcl
diff --git a/docs/deploying-airbyte/on-gcp-compute-engine.md b/docs/deploying-airbyte/on-gcp-compute-engine.md
@@ -60,7 +60,7 @@ gcloud --project $PROJECT_ID compute instances list
 
 ```bash
 # In your workstation terminal
-gcloud --project=$PROJECT_ID beta compute ssh airbyte
+gcloud --project=$PROJECT_ID beta compute ssh $INSTANCE_NAME
 ```
 
 * Install `docker`
@@ -99,7 +99,7 @@ logout
 
 ```bash
 # In your workstation terminal
-gcloud --project=$PROJECT_ID beta compute ssh airbyte
+gcloud --project=$PROJECT_ID beta compute ssh $INSTANCE_NAME
 ```
 
 * Install Airbyte

diff --git a/terraform/aws/demo/core/init.sh b/terraform/aws/demo/core/init.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ex
+
+install_init() {
+  sudo yum update -y
+}
+
+install_docker() {
+  sudo yum install -y docker
+  sudo service docker start
+  sudo usermod -a -G docker ec2-user
+}
+
+install_docker_compose() {
+  sudo wget https://github.com/docker/compose/releases/download/1.26.2/docker-compose-$(uname -s)-$(uname -m) -O /usr/local/bin/docker-compose
+  sudo chmod +x /usr/local/bin/docker-compose
+  docker-compose --version
+}
+
+install_airbyte() {
+  mkdir airbyte && cd airbyte
+  wget https://raw.githubusercontent.com/airbytehq/airbyte/master/{.env,docker-compose.yaml}
+  API_URL=/api/v1/ AIRBYTE_ROLE=demo IS_DEMO=true docker-compose up -d
+}
+
+install_demo_pg() {
+  docker run --rm --name postgres-demo -e POSTGRES_PASSWORD=password -p 3000:5432 -d postgres
+}
+
+main() {
+  install_init
+  install_docker
+  install_docker_compose
+  install_airbyte
+}
+
+main > /tmp/init.log 2>&1
diff --git a/terraform/aws/demo/core/main.tf b/terraform/aws/demo/core/main.tf
@@ -0,0 +1,58 @@
+data "aws_security_group" "default-sg" {
+  id = var.default-sg
+}
+
+data "aws_ami" "amazon-linux-2" {
+  # Hardcoded 'Amazon' owner id
+  owners = [137112412989]
+  most_recent = true
+
+  filter {
+    name   = "owner-alias"
+    values = ["amazon"]
+  }
+
+  filter {
+    name   = "name"
+    values = ["amzn2-ami-hvm-2*"]
+  }
+}
+
+# Ensure we can ssh to the airbyte instance
+resource "aws_security_group" "airbyte-ssh-sg" {
+  name        = "${var.name}-airbyte-ssh-sg"
+  description = "Allow ssh traffic"
+
+  ingress {
+    description = "ssh"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_instance" "airbyte-instance" {
+  lifecycle {
+    # If you need to destroy the instance, make sure you back up the airbyte configuration.
+    prevent_destroy = true
+    # So we can edit the init.sh script without having to re-create the instance.
+    ignore_changes = [user_data]
+  }
+
+  instance_type = var.instance-size
+  ami           = data.aws_ami.amazon-linux-2.id
+
+  security_groups = [
+    data.aws_security_group.default-sg.name,
+    aws_security_group.airbyte-ssh-sg.name
+  ]
+
+  key_name = var.key-name
+
+  user_data = file("${path.module}/init.sh")
+
+  tags = {
+    Name = "${var.name}-airbyte-app"
+  }
+}
diff --git a/terraform/aws/demo/core/outputs.tf b/terraform/aws/demo/core/outputs.tf
@@ -0,0 +1,3 @@
+output "instance-id" {
+  value = aws_instance.airbyte-instance.id
+}
diff --git a/terraform/aws/demo/core/variables.tf b/terraform/aws/demo/core/variables.tf
@@ -0,0 +1,15 @@
+variable "name" {
+  type = string
+}
+
+variable "default-sg" {
+  type = string
+}
+
+variable "instance-size" {
+  type = string
+}
+
+variable "key-name" {
+  type = string
+}
diff --git a/terraform/aws/demo/lb/auth.html b/terraform/aws/demo/lb/auth.html
@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+<body>
+
+    <form>
+        <label for="password">Enter secret: </label>
+        <input id="password" type="password" autocomplete="current-password"/>
+
+        <input id="auth" type="button" value="Auth" onclick="hack_auth(document.getElementById('password').value);"/>
+    </form>
+
+    <script>
+        function hack_auth(password) {
+            console.log(password);
+            document.cookie = `hack-auth-token=${password}; path=/; secure=True; SameSite=Strict;`;
+            document.location.replace("https://demo.airbyte.io");
+        }
+    </script>
+
+</body>
+</html>
diff --git a/terraform/aws/demo/lb/main.tf b/terraform/aws/demo/lb/main.tf
@@ -0,0 +1,179 @@
+data "aws_security_group" "default-sg" {
+  id = var.default-sg
+}
+
+data "aws_vpc" "vpc" {
+  id = var.vpc
+}
+
+resource "aws_security_group" "airbyte-alb-sg" {
+  name        = "${var.name}-airbyte-alb-sg"
+  description = "Allow traffic to the elb"
+
+  ingress {
+    description = "https"
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# Create target groups
+
+resource "aws_lb_target_group" "airbyte-webapp" {
+  name     = "${var.name}-airbyte-webapp-tg"
+  port     = 8000
+  protocol = "HTTP"
+  vpc_id = data.aws_vpc.vpc.id
+
+  health_check {
+    path = "/"
+  }
+}
+
+resource "aws_lb_target_group_attachment" "airbyte-webapp" {
+  target_group_arn = aws_lb_target_group.airbyte-webapp.arn
+  target_id        = var.instance-id
+  port             = 8000
+}
+
+resource "aws_lb_target_group" "airbyte-api" {
+  name     = "${var.name}-airbyte-api-tg"
+  port     = 8001
+  protocol = "HTTP"
+  vpc_id = data.aws_vpc.vpc.id
+
+  health_check {
+    path = "/api/v1/health"
+  }
+}
+
+resource "aws_lb_target_group_attachment" "airbyte-api" {
+  target_group_arn = aws_lb_target_group.airbyte-api.arn
+  target_id        = var.instance-id
+  port             = 8001
+}
+
+# Build load balancer
+
+resource "aws_lb" "airbyte-alb" {
+  enable_deletion_protection = true
+
+  name               = "${var.name}-airbyte-alb"
+
+  internal           = false
+  load_balancer_type = "application"
+  security_groups    = [
+    data.aws_security_group.default-sg.id,
+    aws_security_group.airbyte-alb-sg.id
+  ]
+  subnets = var.subnets
+}
+
+resource "aws_lb_listener" "airbyte-alb-listener" {
+  load_balancer_arn = aws_lb.airbyte-alb.arn
+  port              = "443"
+  protocol          = "HTTPS"
+  ssl_policy        = "ELBSecurityPolicy-2016-08"
+  certificate_arn   = var.certificate
+
+  default_action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.airbyte-webapp.arn
+  }
+}
+
+# By default we deny all api calls
+resource "aws_lb_listener_rule" "deny-all-api" {
+  listener_arn = aws_lb_listener.airbyte-alb-listener.arn
+  priority     = 100
+
+  action {
+    type = "fixed-response"
+
+    fixed_response {
+      content_type = "application/json"
+      message_body = "{}"
+      status_code  = "401"
+    }
+  }
+
+  condition {
+    path_pattern {
+      values = ["/api/v1/*"]
+    }
+  }
+}
+
+# Then we allow all the read endpoints
+resource "aws_lb_listener_rule" "allow-read-api" {
+  listener_arn = aws_lb_listener.airbyte-alb-listener.arn
+  priority     = 99
+
+  action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.airbyte-api.arn
+  }
+
+  condition {
+    path_pattern {
+      values = [
+        "/api/v1/*/list",
+        "/api/v1/*/get",
+        "/api/v1/*/get_by_slug",
+        "/api/v1/*/health",
+      ]
+    }
+  }
+}
+
+# Check for secret cookie to enable write
+resource "aws_lb_listener_rule" "allow-all-api" {
+  listener_arn = aws_lb_listener.airbyte-alb-listener.arn
+  priority     = 98
+
+  action {
+    type             = "forward"
+    target_group_arn = aws_lb_target_group.airbyte-api.arn
+  }
+
+  condition {
+    http_header {
+      http_header_name = "cookie"
+      values = ["*hack-auth-token=${var.auth-secret}*"]
+    }
+  }
+
+  condition {
+    path_pattern {
+      values = [
+        "/api/v1/*"
+      ]
+    }
+  }
+}
+
+# Auth hack
+
+# By default we deny all api calls
+resource "aws_lb_listener_rule" "auth-hack" {
+  listener_arn = aws_lb_listener.airbyte-alb-listener.arn
+  priority     = 97
+
+  action {
+    type = "fixed-response"
+
+    fixed_response {
+      content_type = "text/html"
+      message_body = file("${path.module}/auth.html")
+      status_code  = "200"
+    }
+  }
+
+  condition {
+    path_pattern {
+      values = ["/hack/auth"]
+    }
+  }
+}
diff --git a/terraform/aws/demo/lb/variables.tf b/terraform/aws/demo/lb/variables.tf
@@ -0,0 +1,27 @@
+variable "name" {
+  type = string
+}
+
+variable "vpc" {
+  type = string
+}
+
+variable "default-sg" {
+  type = string
+}
+
+variable "subnets" {
+  type = list(string)
+}
+
+variable "certificate" {
+  type = string
+}
+
+variable "instance-id" {
+  type = string
+}
+
+variable "auth-secret" {
+  type = string
+}