diff --git a/eks-cluster.cfhighlander.rb b/eks-cluster.cfhighlander.rb index 6c4955d..988da75 100644 --- a/eks-cluster.cfhighlander.rb +++ b/eks-cluster.cfhighlander.rb @@ -7,7 +7,17 @@ ComponentParam 'EnvironmentType', 'development', allowedValues: ['development','production'], isGlobal: true ComponentParam 'VPCId', isGlobal: true, type: 'AWS::EC2::VPC::Id' ComponentParam 'SubnetIds' - end + ComponentParam 'BootstrapArguments' + ComponentParam 'KeyName', type: 'AWS::EC2::KeyPair::KeyName' + ComponentParam 'ImageId', type: 'AWS::EC2::Image::Id' + + if !spot.has_key?('instances') + ComponentParam 'InstanceType' + end + ComponentParam 'DesiredCapacity', '1' + ComponentParam 'MinSize', '1' + ComponentParam 'MaxSize', '2' + end end diff --git a/eks-cluster.cfndsl.rb b/eks-cluster.cfndsl.rb index 19e07c2..505c5f8 100644 --- a/eks-cluster.cfndsl.rb +++ b/eks-cluster.cfndsl.rb @@ -1,9 +1,7 @@ CloudFormation do tags = [] - tags << { Key: 'Environment', Value: Ref(:EnvironmentName) } - tags << { Key: 'EnvironmentType', Value: Ref(:EnvironmentType) } - extra_tags.each { |key,value| tags << { Key: key, Value: FnSub(value) } } + extra_tags.each { |key,value| tags << { Key: FnSub(key), Value: FnSub(value) } } if defined? extra_tags IAM_Role(:EksClusterRole) { AssumeRolePolicyDocument service_role_assume_policy('eks') @@ -14,21 +12,97 @@ ]) } - EC2_SecurityGroup(:EksClusterSecurityGroup) do + EC2_SecurityGroup(:EksClusterSecurityGroup) { VpcId Ref('VPCId') - GroupDescription "#{component_name} EKS Cluster communication with worker nodes" - Tags tags + GroupDescription "EKS Cluster communication with worker nodes" + Tags([{ Key: 'Name', Value: FnSub("${EnvironmentName}-eks-controller")}] + tags) Metadata({ cfn_nag: { rules_to_suppress: [ - { id: 'F1000', reason: 'This will be locked down by the cluster nodes component' } + { id: 'F1000', reason: 'adding rules using cfn resources' } ] } }) - end + } + + EC2_SecurityGroup('EksNodeSecurityGroup') { + VpcId Ref('VPCId') + GroupDescription "Security group for all nodes in the cluster" + Tags([ + { Key: 'Name', Value: FnSub("${EnvironmentName}-eks-nodes") }, + { Key: FnSub("kubernetes.io/cluster/${EksCluster}"), Value: 'owned' } + ] + tags) + Metadata({ + cfn_nag: { + rules_to_suppress: [ + { id: 'F1000', reason: 'adding rules using cfn resources' } + ] + } + }) + } + + EC2_SecurityGroupIngress(:NodeSecurityGroupIngress) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow node to communicate with each other' + GroupId Ref(:EksNodeSecurityGroup) + SourceSecurityGroupId Ref(:EksNodeSecurityGroup) + IpProtocol '-1' + FromPort 0 + ToPort 65535 + } + + EC2_SecurityGroupIngress(:NodeSecurityGroupFromControlPlaneIngress) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow worker Kubelets and pods to receive communication from the cluster control plane' + GroupId Ref(:EksNodeSecurityGroup) + SourceSecurityGroupId Ref(:EksClusterSecurityGroup) + IpProtocol 'tcp' + FromPort 1025 + ToPort 65535 + } + + EC2_SecurityGroupEgress(:ControlPlaneEgressToNodeSecurityGroup) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow the cluster control plane to communicate with worker Kubelet and pods' + GroupId Ref(:EksClusterSecurityGroup) + DestinationSecurityGroupId Ref(:EksNodeSecurityGroup) + IpProtocol 'tcp' + FromPort 1025 + ToPort 65535 + } + + EC2_SecurityGroupIngress(:NodeSecurityGroupFromControlPlaneOn443Ingress) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow pods running extension API servers on port 443 to receive communication from cluster control plane' + GroupId Ref(:EksNodeSecurityGroup) + SourceSecurityGroupId Ref(:EksClusterSecurityGroup) + IpProtocol 'tcp' + FromPort 443 + ToPort 443 + } + + EC2_SecurityGroupEgress(:ControlPlaneEgressToNodeSecurityGroupOn443) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow the cluster control plane to communicate with pods running extension API servers on port 443' + GroupId Ref(:EksClusterSecurityGroup) + DestinationSecurityGroupId Ref(:EksNodeSecurityGroup) + IpProtocol 'tcp' + FromPort 443 + ToPort 443 + } + + EC2_SecurityGroupIngress(:ClusterControlPlaneSecurityGroupIngress) { + DependsOn 'EksNodeSecurityGroup' + Description 'Allow pods to communicate with the cluster API Server' + GroupId Ref(:EksClusterSecurityGroup) + SourceSecurityGroupId Ref(:EksNodeSecurityGroup) + IpProtocol 'tcp' + ToPort 443 + FromPort 443 + } EKS_Cluster(:EksCluster) { - Version FnSub(cluster_name) if defined? cluster_name + Name FnSub(cluster_name) if defined? cluster_name ResourcesVpcConfig({ SecurityGroupIds: [ Ref(:EksClusterSecurityGroup) ], SubnetIds: FnSplit(',', Ref('SubnetIds')) @@ -37,4 +111,82 @@ Version eks_version if defined? eks_version } + policies = [] + iam['policies'].each do |name,policy| + policies << iam_policy_allow(name,policy['action'],policy['resource'] || '*') + end if defined? iam_policies + + IAM_Role(:EksNodeRole) { + AssumeRolePolicyDocument service_role_assume_policy(iam['services']) + Path '/' + ManagedPolicyArns(iam['managed_policies']) if iam.has_key?('managed_policies') + Policies(policies) if policies.any? + } + + IAM_InstanceProfile(:EksNodeInstanceProfile) do + Path '/' + Roles [Ref(:EksNodeRole)] + end + + # Setup userdata string + node_userdata = "#!/bin/bash\nset -o xtrace\n" + node_userdata << eks_bootstrap if defined? eks_bootstrap + node_userdata << userdata if defined? userdata + node_userdata << cfnsignal if defined? cfnsignal + + launch_template_tags = [ + { Key: 'Name', Value: FnSub("${EnvironmentName}-eks-node-xx") }, + { Key: FnSub("kubernetes.io/cluster/${EksCluster}"), Value: 'owned' } + ] + launch_template_tags += tags + + template_data = { + SecurityGroupIds: [ Ref(:EksNodeSecurityGroup) ], + TagSpecifications: [ + { ResourceType: 'instance', Tags: launch_template_tags }, + { ResourceType: 'volume', Tags: launch_template_tags } + ], + UserData: FnBase64(FnSub(node_userdata)), + IamInstanceProfile: { Name: Ref(:EksNodeInstanceProfile) }, + KeyName: Ref('KeyName'), + ImageId: Ref('ImageId'), + Monitoring: { Enabled: detailed_monitoring } + } + + # spot details + if spot['enabled'] + template_data[:InstanceMarketOptions] = { MarketType: 'spot' } + if spot.has_key?('instances') + if spot['instances'].is_a?(Hash) + spot_options = spot['instances'].map { |type,price| { SpotInstanceType: type, MaxPrice: price }} + elsif spot['instances'].is_a?(Array) + spot_options = spot['instances'].map { |type| { SpotInstanceType: type }} + end + template_data[:InstanceMarketOptions][:SpotOptions] = spot_options + else + template_data[:InstanceType] = Ref('InstanceType') + end + end + + EC2_LaunchTemplate(:EksNodeLaunchTemplate) { + LaunchTemplateData(template_data) + } + + AutoScaling_AutoScalingGroup(:EksNodeAutoScalingGroup) { + UpdatePolicy(:AutoScalingRollingUpdate, { + MaxBatchSize: '1', + MinInstancesInService: Ref('DesiredCapacity'), + SuspendProcesses: %w(HealthCheck ReplaceUnhealthy AZRebalance AlarmNotification ScheduledActions), + PauseTime: 'PT5M' + }) + DesiredCapacity Ref('DesiredCapacity') + MinSize Ref('MinSize') + MaxSize Ref('MaxSize') + VPCZoneIdentifier FnSplit(',', Ref('SubnetIds')) + LaunchTemplate({ + LaunchTemplateId: Ref(:EksNodeLaunchTemplate), + Version: FnGetAtt(:EksNodeLaunchTemplate, :LatestVersionNumber) + }) + } + end diff --git a/eks-cluster.config.yaml b/eks-cluster.config.yaml index d306fa0..d91e1bf 100644 --- a/eks-cluster.config.yaml +++ b/eks-cluster.config.yaml @@ -1 +1,23 @@ # Default configuration + +detailed_monitoring: false + +spot: + enabled: false + +iam: + services: + - ec2 + managed_policies: + - arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy + - arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + +eks_bootstrap: | + /etc/eks/bootstrap.sh ${EksCluster} ${BootstrapArguments} + +cfnsignal: | + /opt/aws/bin/cfn-signal --exit-code $? \ + --stack ${AWS::StackName} \ + --resource NodeGroup \ + --region ${AWS::Region} diff --git a/tests/all_config.test.yaml b/tests/all_config.test.yaml deleted file mode 100644 index 6c902f9..0000000 --- a/tests/all_config.test.yaml +++ /dev/null @@ -1,10 +0,0 @@ -test_metadata: - type: config - name: all config - description: testing EKS cluster will all possible config options - -cluster_name: ${EnvironmentName}-Cluster -eks_version: 1.11 - -extra_tags: - Cluster: ${EnvironmentName}-Cluster diff --git a/tests/basic.test.yaml b/tests/basic.test.yaml new file mode 100644 index 0000000..4f10f7e --- /dev/null +++ b/tests/basic.test.yaml @@ -0,0 +1,25 @@ +test_metadata: + type: config + name: basic + description: test with some basic configuration + +cluster_name: ${EnvironmentName}-Cluster +eks_version: 1.11 + +extra_tags: + Cluster: ${EnvironmentName}-Cluster + +userdata: | + echo "this is in the userdata" + printenv + +iam: + services: + - ec2 + - ssm + policies: + ssm_get_secrets: + actions: + - ssm:GetParameters + - ssm:GetParametersByPath + - ssm:DescribeParameters diff --git a/tests/default.test.yaml b/tests/default.test.yaml new file mode 100644 index 0000000..0db09b7 --- /dev/null +++ b/tests/default.test.yaml @@ -0,0 +1,4 @@ +test_metadata: + type: config + name: default + description: test with default config diff --git a/tests/spot_array.test.yaml b/tests/spot_array.test.yaml new file mode 100644 index 0000000..ed5dbe8 --- /dev/null +++ b/tests/spot_array.test.yaml @@ -0,0 +1,10 @@ +test_metadata: + type: config + name: spot array + description: test spot instance types with deafult price using the array format + +spot: + enabled: true + instances: + - t2.small + - c4.large diff --git a/tests/spot_hash.test.yaml b/tests/spot_hash.test.yaml new file mode 100644 index 0000000..03c77c3 --- /dev/null +++ b/tests/spot_hash.test.yaml @@ -0,0 +1,10 @@ +test_metadata: + type: config + name: spot hash + description: test spot instance types with a price using the hash format + +spot: + enabled: true + instances: + t2.small: '0.02' + c4.large: '0.08'