Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' of github.com:loilo-inc/canarycage
Browse files Browse the repository at this point in the history
keroxp committed Oct 1, 2018

Unverified

This user has not yet uploaded their public signing key.
2 parents 2d33f8d + b96b5cf commit 3d0393a
Showing 6 changed files with 92 additions and 47 deletions.
14 changes: 7 additions & 7 deletions cli/cage/rollout.go → cli/cage/commands/rollout.go
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
package main
package commands

import (
"github.com/urfave/cli"
"github.com/loilo-inc/canarycage"
"encoding/json"
"fmt"
"os"
"io/ioutil"
"github.com/aws/aws-sdk-go/aws"
"github.com/apex/log"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/loilo-inc/canarycage"
"github.com/urfave/cli"
"io/ioutil"
"os"
)

func RollOutCommand() cli.Command {
12 changes: 6 additions & 6 deletions cli/cage/up.go → cli/cage/commands/up.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
package main
package commands

import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/urfave/cli"
"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
"path/filepath"
"encoding/json"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/apex/log"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
"github.com/loilo-inc/canarycage"
"github.com/urfave/cli"
"path/filepath"
)

func UpCommand(ses *session.Session) cli.Command {
13 changes: 7 additions & 6 deletions cli/cage/main.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package main

import (
"github.com/urfave/cli"
"os"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/loilo-inc/canarycage/cli/cage/commands"
"github.com/urfave/cli"
"log"
"os"
)

func main() {
@@ -18,11 +19,11 @@ func main() {
}
app := cli.NewApp()
app.Name = "canarycage"
app.Version = "1.2.0-alpha"
app.Version = "1.2.1"
app.Description = "A gradual roll-out deployment tool for AWS ECS"
app.Commands = cli.Commands{
RollOutCommand(),
UpCommand(ses),
commands.RollOutCommand(),
commands.UpCommand(ses),
}
app.Run(os.Args)
}
56 changes: 38 additions & 18 deletions rollout.go
Original file line number Diff line number Diff line change
@@ -52,18 +52,26 @@ func (envars *Envars) StartGradualRollOut(
Cluster: envars.Cluster,
Services: []*string{
envars.CurrentServiceName,
envars.NextServiceName,
},
})
if err != nil {
log.Errorf("failed to describe current service due to: %s", err.Error())
return nil, err
}
service := out.Services[0]
originalDesiredCount = *service.DesiredCount
currentService := out.Services[0]
nextService := out.Services[1]
originalDesiredCount = *currentService.DesiredCount
var (
targetGroupArn *string
)
if len(nextService.LoadBalancers) > 0 {
targetGroupArn = nextService.LoadBalancers[0].TargetGroupArn
}
log.Infof("service '%s' ensured. start rolling out", *envars.NextServiceName)
if err := envars.RollOut(ctx, service, originalDesiredCount); err != nil {
if err := envars.RollOut(ctx, targetGroupArn, originalDesiredCount); err != nil {
log.Errorf("failed to roll out due to: %s", err)
if err := envars.Rollback(ctx, &originalDesiredCount, service.LoadBalancers[0].TargetGroupArn); err != nil {
if err := envars.Rollback(ctx, &originalDesiredCount, targetGroupArn); err != nil {
log.Errorf("😱 failed to rollback service '%s' due to: %s", err)
return nil, err
}
@@ -114,7 +122,7 @@ func (envars *Envars) CanaryTest(

func (envars *Envars) RollOut(
ctx *Context,
nextService *ecs.Service,
targetGroupArn *string,
originalDesiredCount int64,
) (error) {
var (
@@ -129,15 +137,23 @@ func (envars *Envars) RollOut(
"currently %d tasks running on '%s', %d times roll out estimated",
originalDesiredCount, *envars.CurrentServiceName, estimatedRollOutCount,
)
lb := nextService.LoadBalancers[0]
o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
TargetGroupArns: []*string{lb.TargetGroupArn},
})
if err != nil {
log.Errorf("failed to describe target groups due to: %s", err)
return err
var (
loadBalancerArn *string
)
if targetGroupArn != nil {
o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
TargetGroupArns: []*string{targetGroupArn},
})
if err != nil {
log.Errorf("failed to describe target groups due to: %s", err)
return err
}
loadBalancerArn = o.TargetGroups[0].LoadBalancerArns[0]
} else {
// LBがないサービスはCanaryTestは行わない
log.Infof("service '%s' has no load balancer. will skip canary tests", *envars.NextServiceName)
envars.SkipCanary = aws.Bool(true)
}
tg := o.TargetGroups[0]
// next serviceのperiodic healthが安定し、current serviceのtaskの数が0になるまで繰り返す
for {
log.Infof("=== preparing for %dth roll out ===", totalRollOutCnt)
@@ -151,22 +167,22 @@ func (envars *Envars) RollOut(
scaleCnt := totalReplacedCnt + replaceCnt
// Phase1: service-nextにtask-nextを指定数配置
log.Infof("%dth roll out starting: will replace %d tasks", totalRollOutCnt, replaceCnt)
log.Infof("start adding of next tasks. this will update '%s' desired count %d to %d", *nextService.ServiceName, totalReplacedCnt, scaleCnt)
err := envars.UpdateDesiredCount(ctx, envars.NextServiceName, tg.TargetGroupArn, &originalDesiredCount, &scaleCnt, true)
log.Infof("start adding of next tasks. this will update '%s' desired count %d to %d", *envars.NextServiceName, totalReplacedCnt, scaleCnt)
err := envars.UpdateDesiredCount(ctx, envars.NextServiceName, targetGroupArn, &originalDesiredCount, &scaleCnt, true)
if err != nil {
log.Errorf("failed to add next tasks due to: %s", err)
return err
}
// Phase2: service-nextのperiodic healthを計測
if *envars.SkipCanary {
log.Infof("🤫 %dth canary test skipped.", totalRollOutCnt)
} else if err := envars.CanaryTest(ctx.Cw, tg.LoadBalancerArns[0], tg.TargetGroupArn, totalRollOutCnt); err != nil {
} else if err := envars.CanaryTest(ctx.Cw, loadBalancerArn, targetGroupArn, totalRollOutCnt); err != nil {
return err
}
// Phase3: service-currentからタスクを指定数消す
descaledCnt := originalDesiredCount - totalReplacedCnt - replaceCnt
log.Infof("updating service '%s' desired count to %d", *envars.CurrentServiceName, descaledCnt)
if err := envars.UpdateDesiredCount(ctx, envars.CurrentServiceName, tg.TargetGroupArn, &originalDesiredCount, &descaledCnt, false); err != nil {
if err := envars.UpdateDesiredCount(ctx, envars.CurrentServiceName, targetGroupArn, &originalDesiredCount, &descaledCnt, false); err != nil {
log.Errorf("failed to roll out tasks due to: %s", err.Error())
return err
}
@@ -199,7 +215,7 @@ func (envars *Envars) RollOut(
log.Infof("estimated roll out completed. Do final canary test...")
if *envars.SkipCanary {
log.Infof("😑 final canary test skipped...")
} else if err := envars.CanaryTest(ctx.Cw, tg.LoadBalancerArns[0], tg.TargetGroupArn, totalRollOutCnt); err != nil {
} else if err := envars.CanaryTest(ctx.Cw, loadBalancerArn, targetGroupArn, totalRollOutCnt); err != nil {
log.Errorf("final canary test has failed due to: %s", err)
return err
}
@@ -386,6 +402,10 @@ func (envars *Envars) UpdateDesiredCount(
log.Errorf("failed to wait service-stable due to: %s", err)
return err
}
// LBがないサービスはここで終わり
if targetGroupArn == nil {
return nil
}
o, err := ctx.Alb.DescribeTargetGroups(&elbv2.DescribeTargetGroupsInput{
TargetGroupArns: []*string{targetGroupArn},
})
21 changes: 21 additions & 0 deletions rollout_test.go
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ func DefaultEnvars() *Envars {
NextTaskDefinitionBase64: aws.String(o),
AvailabilityThreshold: aws.Float64(0.9970),
ResponseTimeThreshold: aws.Float64(1),
SkipCanary: aws.Bool(false),
}
}

@@ -205,6 +206,26 @@ func TestEnvars_StartGradualRollOut4(t *testing.T) {
}
}

func TestEnvars_StartGradualRollOut5(t *testing.T) {
// lbがないサービスの場合もロールアウトする
envars := DefaultEnvars()
d, _ := ioutil.ReadFile("fixtures/service.json")
input := &ecs.CreateServiceInput{}
_ = json.Unmarshal(d, input)
input.LoadBalancers = nil
newTimer = fakeTimer
defer recoverTimer()
o, _ := json.Marshal(input)
envars.NextServiceDefinitionBase64 = aws.String(base64.StdEncoding.EncodeToString(o))
ctrl := gomock.NewController(t)
_, ctx := envars.Setup(ctrl, 2)
if res, err := envars.StartGradualRollOut(ctx); err != nil {
t.Fatalf(err.Error())
} else if res.HandledError != nil {
t.Fatalf(err.Error())
}
}

func TestEnvars_Rollback(t *testing.T) {
log.SetLevel(log.DebugLevel)
newTimer = fakeTimer
23 changes: 13 additions & 10 deletions test-integration/integration_test.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
package test_integration

import (
"io/ioutil"
"github.com/aws/aws-sdk-go/service/ecs"
"encoding/json"
"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
"fmt"
"github.com/apex/log"
"github.com/aws/aws-sdk-go/aws"
"testing"
"github.com/loilo-inc/canarycage"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/apex/log"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/ecs"
"github.com/aws/aws-sdk-go/service/ecs/ecsiface"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/loilo-inc/canarycage"
"github.com/stretchr/testify/assert"
"golang.org/x/sync/errgroup"
"io/ioutil"
"net/http"
"testing"
"time"
"golang.org/x/sync/errgroup"
"github.com/stretchr/testify/assert"
"fmt"
)

const kCurrentServiceName = "itg-test-service-current"
@@ -225,7 +225,10 @@ func TestHealthyToHealthy(t *testing.T) {
envars.NextTaskDefinitionArn = aws.String(kHealthyTDArn)
envars.CurrentServiceName = aws.String(kCurrentServiceName)
envars.NextServiceName = aws.String(kNextServiceName)
defer cleanupService(ctx.Ecs, envars, envars.CurrentServiceName)
defer func() {
cleanupService(ctx.Ecs, envars, envars.CurrentServiceName)
cleanupService(ctx.Ecs, envars, envars.NextServiceName)
}()
result, err := testInternal(t, envars);
if err != nil {
t.Fatalf(err.Error())

0 comments on commit 3d0393a

Please sign in to comment.