Merge branch 'master' of github.com:loilo-inc/canarycage

cli/cage/rollout.go → cli/cage/commands/rollout.go

@@ -1,18 +1,18 @@
-package main
+package commands
 
 import (
-	"github.com/urfave/cli"
-	"github.com/loilo-inc/canarycage"
 	"encoding/json"
 	"fmt"
-	"os"
-	"io/ioutil"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/apex/log"
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/loilo-inc/canarycage"
+	"github.com/urfave/cli"
+	"io/ioutil"
+	"os"
 )
 
 func RollOutCommand() cli.Command {

cli/cage/up.go → cli/cage/commands/up.go

@@ -1,14 +1,14 @@
-package main
+package commands
 
 import (
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/urfave/cli"
-	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
-	"path/filepath"
 	"encoding/json"
-	"github.com/aws/aws-sdk-go/service/ecs"
 	"github.com/apex/log"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
 	"github.com/loilo-inc/canarycage"
+	"github.com/urfave/cli"
+	"path/filepath"
 )
 
 func UpCommand(ses *session.Session) cli.Command {

cli/cage/main.go

@@ -1,11 +1,12 @@
 package main
 
 import (
-	"github.com/urfave/cli"
-	"os"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/loilo-inc/canarycage/cli/cage/commands"
+	"github.com/urfave/cli"
 	"log"
+	"os"
 )
 
 func main() {
@@ -18,11 +19,11 @@ func main() {
 	}
 	app := cli.NewApp()
 	app.Name = "canarycage"
-	app.Version = "1.2.0-alpha"
+	app.Version = "1.2.1"
 	app.Description = "A gradual roll-out deployment tool for AWS ECS"
 	app.Commands = cli.Commands{
-		RollOutCommand(),
-		UpCommand(ses),
+		commands.RollOutCommand(),
+		commands.UpCommand(ses),
 	}
 	app.Run(os.Args)
 }

rollout.go

@@ -52,18 +52,26 @@ func (envars *Envars) StartGradualRollOut(
 		Cluster: envars.Cluster,
 		Services: []*string{
 			envars.CurrentServiceName,
+			envars.NextServiceName,
 		},
 	})
 	if err != nil {
 		log.Errorf("failed to describe current service due to: %s", err.Error())
 		return nil, err
 	}
-	service := out.Services[0]
-	originalDesiredCount = *service.DesiredCount
+	currentService := out.Services[0]
+	nextService := out.Services[1]
+	originalDesiredCount = *currentService.DesiredCount
+	var (
+		targetGroupArn *string
+	)
+	if len(nextService.LoadBalancers) > 0 {
+		targetGroupArn = nextService.LoadBalancers[0].TargetGroupArn
+	}
 	log.Infof("service '%s' ensured. start rolling out", *envars.NextServiceName)
-	if err := envars.RollOut(ctx, service, originalDesiredCount); err != nil {
+	if err := envars.RollOut(ctx, targetGroupArn, originalDesiredCount); err != nil {
 		log.Errorf("failed to roll out due to: %s", err)
-		if err := envars.Rollback(ctx, &originalDesiredCount, service.LoadBalancers[0].TargetGroupArn); err != nil {
+		if err := envars.Rollback(ctx, &originalDesiredCount, targetGroupArn); err != nil {
 			log.Errorf("😱 failed to rollback service '%s' due to: %s", err)
 			return nil, err
 		}
@@ -114,7 +122,7 @@ func (envars *Envars) CanaryTest(
 
 func (envars *Envars) RollOut(
 	ctx *Context,
-	nextService *ecs.Service,
+	targetGroupArn *string,
 	originalDesiredCount int64,
 ) (error) {
 	var (
@@ -129,15 +137,23 @@ func (envars *Envars) RollOut(
 		"currently %d tasks running on '%s', %d times roll out estimated",
 		originalDesiredCount, *envars.CurrentServiceName, estimatedRollOutCount,
 	)
-	lb := nextService.LoadBalancers[0]
-	o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
-		TargetGroupArns: []*string{lb.TargetGroupArn},
-	})
-	if err != nil {
-		log.Errorf("failed to describe target groups due to: %s", err)
-		return err
+	var (
+		loadBalancerArn *string
+	)
+	if targetGroupArn != nil {
+		o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
+			TargetGroupArns: []*string{targetGroupArn},
+		})
+		if err != nil {
+			log.Errorf("failed to describe target groups due to: %s", err)
+			return err
+		}
+		loadBalancerArn = o.TargetGroups[0].LoadBalancerArns[0]
+	} else {
+		// LBがないサービスはCanaryTestは行わない
+		log.Infof("service '%s' has no load balancer. will skip canary tests", *envars.NextServiceName)
+		envars.SkipCanary = aws.Bool(true)
 	}
-	tg := o.TargetGroups[0]
 	// next serviceのperiodic healthが安定し、current serviceのtaskの数が0になるまで繰り返す
 	for {
 		log.Infof("=== preparing for %dth roll out ===", totalRollOutCnt)
@@ -151,22 +167,22 @@ func (envars *Envars) RollOut(
 		scaleCnt := totalReplacedCnt + replaceCnt
 		// Phase1: service-nextにtask-nextを指定数配置
 		log.Infof("%dth roll out starting: will replace %d tasks", totalRollOutCnt, replaceCnt)
-		log.Infof("start adding of next tasks. this will update '%s' desired count %d to %d", *nextService.ServiceName, totalReplacedCnt, scaleCnt)
-		err := envars.UpdateDesiredCount(ctx, envars.NextServiceName, tg.TargetGroupArn, &originalDesiredCount, &scaleCnt, true)
+		log.Infof("start adding of next tasks. this will update '%s' desired count %d to %d", *envars.NextServiceName, totalReplacedCnt, scaleCnt)
+		err := envars.UpdateDesiredCount(ctx, envars.NextServiceName, targetGroupArn, &originalDesiredCount, &scaleCnt, true)
 		if err != nil {
 			log.Errorf("failed to add next tasks due to: %s", err)
 			return err
 		}
 		// Phase2: service-nextのperiodic healthを計測
 		if *envars.SkipCanary {
 			log.Infof("🤫 %dth canary test skipped.", totalRollOutCnt)
-		} else if err := envars.CanaryTest(ctx.Cw, tg.LoadBalancerArns[0], tg.TargetGroupArn, totalRollOutCnt); err != nil {
+		} else if err := envars.CanaryTest(ctx.Cw, loadBalancerArn, targetGroupArn, totalRollOutCnt); err != nil {
 			return err
 		}
 		// Phase3: service-currentからタスクを指定数消す
 		descaledCnt := originalDesiredCount - totalReplacedCnt - replaceCnt
 		log.Infof("updating service '%s' desired count to %d", *envars.CurrentServiceName, descaledCnt)
-		if err := envars.UpdateDesiredCount(ctx, envars.CurrentServiceName, tg.TargetGroupArn, &originalDesiredCount, &descaledCnt, false); err != nil {
+		if err := envars.UpdateDesiredCount(ctx, envars.CurrentServiceName, targetGroupArn, &originalDesiredCount, &descaledCnt, false); err != nil {
 			log.Errorf("failed to roll out tasks due to: %s", err.Error())
 			return err
 		}
@@ -199,7 +215,7 @@ func (envars *Envars) RollOut(
 			log.Infof("estimated roll out completed. Do final canary test...")
 			if *envars.SkipCanary {
 				log.Infof("😑 final canary test skipped...")
-			} else if err := envars.CanaryTest(ctx.Cw, tg.LoadBalancerArns[0], tg.TargetGroupArn, totalRollOutCnt); err != nil {
+			} else if err := envars.CanaryTest(ctx.Cw, loadBalancerArn, targetGroupArn, totalRollOutCnt); err != nil {
 				log.Errorf("final canary test has failed due to: %s", err)
 				return err
 			}
@@ -386,6 +402,10 @@ func (envars *Envars) UpdateDesiredCount(
 		log.Errorf("failed to wait service-stable due to: %s", err)
 		return err
 	}
+	// LBがないサービスはここで終わり
+	if targetGroupArn == nil {
+		return nil
+	}
 	o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
 		TargetGroupArns: []*string{targetGroupArn},
 	})

rollout_test.go

@@ -33,6 +33,7 @@ func DefaultEnvars() *Envars {
 		NextTaskDefinitionBase64: aws.String(o),
 		AvailabilityThreshold:    aws.Float64(0.9970),
 		ResponseTimeThreshold:    aws.Float64(1),
+		SkipCanary:               aws.Bool(false),
 	}
 }
 
@@ -205,6 +206,26 @@ func TestEnvars_StartGradualRollOut4(t *testing.T) {
 	}
 }
 
+func TestEnvars_StartGradualRollOut5(t *testing.T) {
+	// lbがないサービスの場合もロールアウトする
+	envars := DefaultEnvars()
+	d, _ := ioutil.ReadFile("fixtures/service.json")
+	input := &ecs.CreateServiceInput{}
+	_ = json.Unmarshal(d, input)
+	input.LoadBalancers = nil
+	newTimer = fakeTimer
+	defer recoverTimer()
+	o, _ := json.Marshal(input)
+	envars.NextServiceDefinitionBase64 = aws.String(base64.StdEncoding.EncodeToString(o))
+	ctrl := gomock.NewController(t)
+	_, ctx := envars.Setup(ctrl, 2)
+	if res, err := envars.StartGradualRollOut(ctx); err != nil {
+		t.Fatalf(err.Error())
+	} else if res.HandledError != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
 func TestEnvars_Rollback(t *testing.T) {
 	log.SetLevel(log.DebugLevel)
 	newTimer = fakeTimer

test-integration/integration_test.go

@@ -1,22 +1,22 @@
 package test_integration
 
 import (
-	"io/ioutil"
-	"github.com/aws/aws-sdk-go/service/ecs"
 	"encoding/json"
-	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
+	"fmt"
+	"github.com/apex/log"
 	"github.com/aws/aws-sdk-go/aws"
-	"testing"
-	"github.com/loilo-inc/canarycage"
 	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/apex/log"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/aws/aws-sdk-go/service/ecs"
+	"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
 	"github.com/aws/aws-sdk-go/service/elbv2"
+	"github.com/loilo-inc/canarycage"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sync/errgroup"
+	"io/ioutil"
 	"net/http"
+	"testing"
 	"time"
-	"golang.org/x/sync/errgroup"
-	"github.com/stretchr/testify/assert"
-	"fmt"
 )
 
 const kCurrentServiceName = "itg-test-service-current"
@@ -225,7 +225,10 @@ func TestHealthyToHealthy(t *testing.T) {
 	envars.NextTaskDefinitionArn = aws.String(kHealthyTDArn)
 	envars.CurrentServiceName = aws.String(kCurrentServiceName)
 	envars.NextServiceName = aws.String(kNextServiceName)
-	defer cleanupService(ctx.Ecs, envars, envars.CurrentServiceName)
+	defer func() {
+		cleanupService(ctx.Ecs, envars, envars.CurrentServiceName)
+		cleanupService(ctx.Ecs, envars, envars.NextServiceName)
+	}()
 	result, err := testInternal(t, envars);
 	if err != nil {
 		t.Fatalf(err.Error())