feat: Add canary task container health checks (#68)

loilo-inc · Apr 15, 2024 · 2267068 · 2267068
1 parent d153e0d
commit 2267068
Show file tree

Hide file tree

Showing 4 changed files with 388 additions and 162 deletions.
diff --git a/fixtures/task-definition.json b/fixtures/task-definition.json
@@ -124,6 +124,118 @@
                 "retries": 0,
                 "startPeriod": 0
             }
+        },
+        {
+            "name": "containerWithoutHealthCheck",
+            "image": "",
+            "repositoryCredentials": {
+                "credentialsParameter": ""
+            },
+            "cpu": 0,
+            "memory": 0,
+            "memoryReservation": 0,
+            "links": [
+                ""
+            ],
+            "portMappings": [
+                {
+                    "containerPort": 8000,
+                    "hostPort": 80
+                }
+            ],
+            "essential": true,
+            "entryPoint": [
+                ""
+            ],
+            "command": [
+                ""
+            ],
+            "environment": [
+                {
+                    "name": "",
+                    "value": ""
+                }
+            ],
+            "mountPoints": [
+                {
+                    "sourceVolume": "",
+                    "containerPath": "",
+                    "readOnly": true
+                }
+            ],
+            "volumesFrom": [
+                {
+                    "sourceContainer": "",
+                    "readOnly": true
+                }
+            ],
+            "linuxParameters": {
+                "capabilities": {
+                    "add": [
+                        ""
+                    ],
+                    "drop": [
+                        ""
+                    ]
+                },
+                "devices": [
+                    {
+                        "hostPath": "",
+                        "containerPath": "",
+                        "permissions": [
+                            "mknod"
+                        ]
+                    }
+                ],
+                "initProcessEnabled": true,
+                "sharedMemorySize": 0,
+                "tmpfs": [
+                    {
+                        "containerPath": "",
+                        "size": 0,
+                        "mountOptions": [
+                            ""
+                        ]
+                    }
+                ]
+            },
+            "hostname": "",
+            "user": "",
+            "workingDirectory": "",
+            "disableNetworking": true,
+            "privileged": true,
+            "readonlyRootFilesystem": true,
+            "dnsServers": [
+                ""
+            ],
+            "dnsSearchDomains": [
+                ""
+            ],
+            "extraHosts": [
+                {
+                    "hostname": "",
+                    "ipAddress": ""
+                }
+            ],
+            "dockerSecurityOptions": [
+                ""
+            ],
+            "dockerLabels": {
+                "KeyName": ""
+            },
+            "ulimits": [
+                {
+                    "name": "core",
+                    "softLimit": 0,
+                    "hardLimit": 0
+                }
+            ],
+            "logConfiguration": {
+                "logDriver": "gelf",
+                "options": {
+                    "KeyName": ""
+                }
+            }
         }
     ],
     "volumes": [

diff --git a/rollout.go b/rollout.go
@@ -96,6 +96,13 @@ func (c *cage) RollOut(ctx context.Context) (*RollOutResult, error) {
 			)
 		}
 	}(canaryTask, ret)
+
+	log.Infof("😷 ensuring canary task container(s) to become healthy...")
+	if err := c.waitUntilContainersBecomeHealthy(ctx, *canaryTask.task.TaskArn, nextTaskDefinition); err != nil {
+		return throw(err)
+	}
+	log.Info("🤩 canary task container(s) is healthy!")
+
 	log.Infof("canary task '%s' ensured.", *canaryTask.task.TaskArn)
 	if targetGroupArn != nil {
 		log.Infof("😷 ensuring canary task to become healthy...")
@@ -398,6 +405,46 @@ func (c *cage) StartCanaryTask(ctx context.Context, nextTaskDefinition *ecstypes
 	}, nil
 }
 
+func (c *cage) waitUntilContainersBecomeHealthy(ctx context.Context, taskArn string, nextTaskDefinition *ecstypes.TaskDefinition) error {
+	containerHasHealthChecks := map[string]struct{}{}
+	for _, definition := range nextTaskDefinition.ContainerDefinitions {
+		if definition.HealthCheck != nil {
+			containerHasHealthChecks[*definition.Name] = struct{}{}
+		}
+	}
+
+	for count := 0; count < 10; count++ {
+		<-newTimer(time.Duration(15) * time.Second).C
+		log.Infof("canary task '%s' waits until %d container(s) become healthy", taskArn, len(containerHasHealthChecks))
+		if o, err := c.ecs.DescribeTasks(ctx, &ecs.DescribeTasksInput{
+			Cluster: &c.env.Cluster,
+			Tasks:   []string{taskArn},
+		}); err != nil {
+			return err
+		} else {
+			task := o.Tasks[0]
+			if *task.LastStatus != "RUNNING" {
+				return fmt.Errorf("😫 canary task has stopped: %s", *task.StoppedReason)
+			}
+
+			for _, container := range task.Containers {
+				if _, ok := containerHasHealthChecks[*container.Name]; !ok {
+					continue
+				}
+				if container.HealthStatus != ecstypes.HealthStatusHealthy {
+					log.Infof("container '%s' is not healthy: %s", *container.Name, container.HealthStatus)
+					continue
+				}
+				delete(containerHasHealthChecks, *container.Name)
+			}
+			if len(containerHasHealthChecks) == 0 {
+				return nil
+			}
+		}
+	}
+	return fmt.Errorf("😨 canary task hasn't become to be healthy")
+}
+
 func (c *cage) StopCanaryTask(ctx context.Context, input *StartCanaryTaskOutput) error {
 	if input.registrationSkipped {
 		log.Info("no load balancer is attached to service. Skip deregisteration.")