Fix unhealthy tsa (#33)

* Fix unhealthy TSA and add tests

* Refactor for clarity

* Add test for target health

* Remove fly download log statement

* Add wait loop for target group health

* Increase wait time between describing target groups

examples/basic/main.tf

@@ -55,7 +55,7 @@ module "concourse_atc" {
   github_client_secret = "sm:///concourse-deployment/github-oauth-client-secret"
   github_users         = ["itsdalmo"]
   github_teams         = ["telia-oss:concourse-owners"]
-  local_user           = "sm:///concourse-deployment/admin-user"
+  local_user           = "admin:${var.concourse_admin_password}"
   local_admin_user     = "admin"
 
   tags = {

examples/basic/variables.tf

@@ -7,6 +7,11 @@ variable "packer_ami" {
   type = string
 }
 
+variable "concourse_admin_password" {
+  type    = string
+  default = "dolphins"
+}
+
 variable "postgres_password" {
   type    = string
   default = "dolphins"

go.mod

@@ -3,7 +3,7 @@ module github.com/telia-oss/terraform-aws-concourse/v3
 go 1.12
 
 require (
-	github.com/aws/aws-sdk-go v1.23.3 // indirect
+	github.com/aws/aws-sdk-go v1.23.3
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/google/uuid v1.1.1 // indirect

modules/atc/main.tf

@@ -18,6 +18,15 @@ resource "aws_security_group_rule" "lb_ingress_atc" {
   source_security_group_id = module.external_lb.security_group_id
 }
 
+resource "aws_security_group_rule" "workers_ingress_atc" {
+  security_group_id = module.atc.security_group_id
+  type              = "ingress"
+  protocol          = "tcp"
+  from_port         = var.atc_port
+  to_port           = var.atc_port
+  cidr_blocks       = [data.aws_vpc.concourse.cidr_block]
+}
+
 resource "aws_security_group_rule" "workers_ingress_tsa" {
   security_group_id = module.atc.security_group_id
   type              = "ingress"

test/module.go

@@ -1,12 +1,23 @@
 package module
 
 import (
+	"bytes"
 	"encoding/json"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
+	"os/exec"
 	"path"
+	"path/filepath"
+	"runtime"
 	"testing"
+	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/stretchr/testify/assert"
 
 	asg "github.com/telia-oss/terraform-aws-asg/v3/test"
@@ -19,16 +30,52 @@ type Expectations struct {
 	WorkerAutoscaling asg.Expectations
 }
 
-func RunTestSuite(t *testing.T, endpoint, atcASGName, workerASGName string, region string, expected Expectations) {
+func RunTestSuite(t *testing.T, endpoint, atcASGName, workerASGName, adminUser, adminPassword, region string, expected Expectations) {
 	// Run test suites for the autoscaling groups.
 	asg.RunTestSuite(t, atcASGName, region, expected.ATCAutoscaling)
 	asg.RunTestSuite(t, workerASGName, region, expected.WorkerAutoscaling)
 
+	// Wait for ATC to register as healthy in the target groups (max 10min wait)
+	sess := NewSession(t, region)
+	WaitForHealthyTargets(t, sess, atcASGName, 20*time.Second, 10*time.Minute)
+
+	info := GetConcourseInfo(t, endpoint)
+	assert.Equal(t, expected.Version, info.Version)
+	assert.Equal(t, expected.WorkerVersion, info.WorkerVersion)
+
+	// Download and install fly binary.
+	tempDir, err := ioutil.TempDir("", "terraform-aws-concourse")
+	if err != nil {
+		t.Fatalf("failed to create temporary directory for fly binary: %s", err)
+	}
+	defer os.RemoveAll(tempDir)
+
+	fly := &Fly{
+		Endpoint:  endpoint,
+		Directory: tempDir,
+		Target:    "terraform-aws-concourse",
+	}
+
+	fly.Setup(t, adminUser, adminPassword)
+
+	workers := fly.Workers(t)
+	assert.Equal(t, int(expected.WorkerAutoscaling.MinSize), len(workers))
+	for _, worker := range workers {
+		assert.Equal(t, "linux", worker.Platform)
+		assert.Equal(t, "running", worker.State)
+	}
+}
+
+func parseURL(t *testing.T, endpoint string) *url.URL {
 	u, err := url.Parse(endpoint)
 	if err != nil {
 		t.Fatalf("failed to parse url from endpoint: %s", endpoint)
 	}
+	return u
+}
 
+func GetConcourseInfo(t *testing.T, endpoint string) ConcourseInfo {
+	u := parseURL(t, endpoint)
 	u.Path = path.Join(u.Path, "api", "v1", "info")
 
 	r, err := http.Get(u.String())
@@ -37,19 +84,135 @@ func RunTestSuite(t *testing.T, endpoint, atcASGName, workerASGName string, regi
 	}
 	defer r.Body.Close()
 
-	assert.Equal(t, 200, r.StatusCode)
+	if r.StatusCode != http.StatusOK {
+		t.Errorf("got non-200 response: %d", r.StatusCode)
+	}
 
-	var info concourseInfo
+	var info ConcourseInfo
 	err = json.NewDecoder(r.Body).Decode(&info)
 	if err != nil {
 		t.Fatalf("failed to deserialize JSON response: %s", err)
 	}
-
-	assert.Equal(t, expected.Version, info.Version)
-	assert.Equal(t, expected.WorkerVersion, info.WorkerVersion)
+	return info
 }
 
-type concourseInfo struct {
+type ConcourseInfo struct {
 	Version       string `json:"version"`
 	WorkerVersion string `json:"worker_version"`
 }
+
+func NewSession(t *testing.T, region string) *session.Session {
+	sess, err := session.NewSession(&aws.Config{
+		Region: aws.String(region),
+	})
+	if err != nil {
+		t.Fatalf("failed to create new AWS session: %s", err)
+	}
+	return sess
+}
+
+func DescribeTargetGroups(t *testing.T, sess *session.Session, asgName string) []*autoscaling.LoadBalancerTargetGroupState {
+	c := autoscaling.New(sess)
+
+	out, err := c.DescribeLoadBalancerTargetGroups(&autoscaling.DescribeLoadBalancerTargetGroupsInput{
+		AutoScalingGroupName: aws.String(asgName),
+	})
+	if err != nil {
+		t.Fatalf("failed to describe load balancer target groups: %s", err)
+	}
+	return out.LoadBalancerTargetGroups
+}
+
+func WaitForHealthyTargets(t *testing.T, sess *session.Session, asgName string, checkInterval time.Duration, timeoutLimit time.Duration) {
+	interval := time.NewTicker(checkInterval)
+	defer interval.Stop()
+
+	timeout := time.NewTimer(timeoutLimit)
+	defer timeout.Stop()
+
+WaitLoop:
+	for {
+		select {
+		case <-interval.C:
+			targetGroups := DescribeTargetGroups(t, sess, asgName)
+			for _, group := range targetGroups {
+				if aws.StringValue(group.State) != "InService" {
+					t.Logf("target group not ready: %s", aws.StringValue(group.LoadBalancerTargetGroupARN))
+					continue WaitLoop
+				}
+			}
+			break WaitLoop
+		case <-timeout.C:
+			t.Fatal("timeout reached while waiting for target group health checks")
+		}
+	}
+}
+
+type Fly struct {
+	Endpoint  string
+	Directory string
+	Target    string
+	bin       string
+}
+
+func (f *Fly) Setup(t *testing.T, username, password string) {
+	u := parseURL(t, f.Endpoint)
+	q := u.Query()
+
+	q.Set("arch", "amd64")
+	q.Set("platform", runtime.GOOS)
+
+	u.Path = path.Join(u.Path, "api", "v1", "cli")
+	u.RawQuery = q.Encode()
+
+	f.bin = filepath.Join(f.Directory, "fly")
+	file, err := os.Create(f.bin)
+	if err != nil {
+		t.Fatalf("failed to create new file: %s", err)
+	}
+	defer file.Close()
+
+	resp, err := http.Get(u.String())
+	if err != nil {
+		t.Fatalf("failed to get fly: %s", err)
+	}
+	defer resp.Body.Close()
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		t.Fatalf("failed to write fly to disk: %s", err)
+	}
+
+	err = file.Chmod(0755)
+	if err != nil {
+		t.Fatalf("failed to change fly permissions: %s", err)
+	}
+
+	cmd := exec.Command(f.bin, "--target", f.Target, "login", "--team-name", "main", "--concourse-url", f.Endpoint, "--username", username, "--password", password)
+	_, err = cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("failed to login to concourse: %s", err)
+	}
+}
+
+func (f *Fly) Workers(t *testing.T) []*ConcourseWorker {
+	cmd := exec.Command(f.bin, "--target", f.Target, "workers", "--json")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Errorf("failed to list workers: %s", err)
+	}
+
+	r := bytes.NewReader(out)
+
+	var workers []*ConcourseWorker
+	err = json.NewDecoder(r).Decode(&workers)
+	if err != nil {
+		t.Fatalf("failed to deserialize workers: %s", err)
+	}
+	return workers
+}
+
+type ConcourseWorker struct {
+	Platform string `json:"platform"`
+	State    string `json:"state"`
+}

test/module_test.go

@@ -22,13 +22,15 @@ func TestModule(t *testing.T) {
 		description string
 		directory   string
 		name        string
+		password    string
 		region      string
 		expected    concourse.Expectations
 	}{
 		{
 			description: "basic example",
 			directory:   "../examples/basic",
 			name:        fmt.Sprintf("concourse-basic-test-%s", random.UniqueId()),
+			password:    random.UniqueId(),
 			region:      "eu-west-1",
 			expected: concourse.Expectations{
 				Version:       "5.1.0",
@@ -47,7 +49,6 @@ func TestModule(t *testing.T) {
 						`Environment="CONCOURSE_GITHUB_CLIENT_SECRET=sm:///concourse-deployment/github-oauth-client-secret"`,
 						`Environment="CONCOURSE_MAIN_TEAM_GITHUB_USER=itsdalmo"`,
 						`Environment="CONCOURSE_MAIN_TEAM_GITHUB_TEAM=telia-oss:concourse-owners"`,
-						`Environment="CONCOURSE_ADD_LOCAL_USER=sm:///concourse-deployment/admin-user"`,
 						`Environment="CONCOURSE_MAIN_TEAM_LOCAL_USER=admin"`,
 						`Environment="CONCOURSE_POSTGRES_PORT=5439"`,
 						`Environment="CONCOURSE_POSTGRES_USER=superuser"`,
@@ -118,9 +119,10 @@ func TestModule(t *testing.T) {
 
 				Vars: map[string]interface{}{
 					// aws_db_subnet_group requires a lowercase name.
-					"name_prefix": strings.ToLower(tc.name),
-					"packer_ami":  amiID,
-					"region":      tc.region,
+					"name_prefix":              strings.ToLower(tc.name),
+					"concourse_admin_password": tc.password,
+					"packer_ami":               amiID,
+					"region":                   tc.region,
 				},
 
 				EnvVars: map[string]string{
@@ -135,6 +137,8 @@ func TestModule(t *testing.T) {
 				terraform.Output(t, options, "endpoint"),
 				terraform.Output(t, options, "atc_asg_id"),
 				terraform.Output(t, options, "worker_asg_id"),
+				"admin",
+				tc.password,
 				tc.region,
 				tc.expected,
 			)