Skip to content

Commit

Permalink
Use separate EBS Volumes for /var, /home folders (EKS 1.24+)
Browse files Browse the repository at this point in the history
  • Loading branch information
@fred-vogt-dod
fred-vogt-dod committed May 16, 2023
1 parent 93d760a commit 195fe68
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 67 deletions.
80 changes: 53 additions & 27 deletions amazon-eks-al2.pkr.hcl
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,3 @@
locals {
timestamp = regex_replace(timestamp(), "[- TZ:]", "")

target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}"
}

data "amazon-ami" "this" {
filters = {
architecture = var.source_ami_arch
Expand All @@ -21,36 +15,68 @@ data "amazon-ami" "this" {
region = var.aws_region
}

source "amazon-ebs" "this" {
ami_block_device_mappings {
delete_on_termination = true
device_name = "/dev/sdb"
volume_size = var.data_volume_size
volume_type = "gp2"
encrypted = true
locals {
timestamp = regex_replace(timestamp(), "[- TZ:]", "")

target_ami_name = "${var.ami_name_prefix}-${var.eks_version}-v${local.timestamp}"

block_device_mappings = {
"/" = {
device_name = "/dev/xvda"
volume_size = var.root_volume_size
}
"/home" = {
device_name = "/dev/sdf"
volume_size = var.home_volume_size
}
"/var" = {
device_name = "/dev/sdg"
volume_size = var.var_volume_size
}
"/var/log" = {
device_name = "/dev/sdh"
volume_size = var.varlog_volume_size
}
"/var/log/audit" = {
device_name = "/dev/sdi"
volume_size = var.varlogaudit_volume_size
}
"/var/lib/containerd" = {
device_name = "/dev/sdj"
volume_size = var.varlibcontainerd_volume_size
}
}
}

source "amazon-ebs" "this" {
ami_description = "EKS Kubernetes Worker AMI with AmazonLinux2 image"
ami_name = local.target_ami_name
ami_virtualization_type = "hvm"
instance_type = var.instance_type

launch_block_device_mappings {
delete_on_termination = true
device_name = "/dev/xvda"
volume_size = var.root_volume_size
volume_type = "gp2"
encrypted = true
kms_key_id = var.kms_key_id
dynamic "ami_block_device_mappings" {
for_each = local.block_device_mappings

content {
device_name = ami_block_device_mappings.value.device_name
volume_size = ami_block_device_mappings.value.volume_size
delete_on_termination = true
volume_type = "gp3"
encrypted = true
}
}

launch_block_device_mappings {
delete_on_termination = true
device_name = "/dev/sdb"
volume_size = var.data_volume_size
volume_type = "gp2"
encrypted = true
kms_key_id = var.kms_key_id
dynamic "launch_block_device_mappings" {
for_each = local.block_device_mappings

content {
device_name = launch_block_device_mappings.value.device_name
volume_size = launch_block_device_mappings.value.volume_size
delete_on_termination = true
volume_type = "gp3"
encrypted = true
kms_key_id = var.kms_key_id
}
}

encrypt_boot = var.encrypt_boot
Expand Down
2 changes: 1 addition & 1 deletion scripts/cis-docker.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ echo "1.1.2 - ensure that the version of Docker is up to date"
yum -y update docker

echo "1.2.1 - ensure a separate partition for containers has been created"
grep '/var/lib/docker\s' /proc/mounts
#grep '/var/lib/docker\s' /proc/mounts

echo "1.2.2 - ensure only trusted users are allowed to control Docker daemon"
getent group docker
Expand Down
70 changes: 41 additions & 29 deletions scripts/partition-disks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,21 +17,32 @@ set -o errexit
# None
################################################################
migrate_and_mount_disk() {
local disk_name=$1
local device_name=$1
local folder_path=$2
local mount_options=$3
local temp_path="/mnt${folder_path}"
local old_path="${folder_path}-old"

# install an ext4 filesystem to the disk
mkfs -t ext4 ${disk_name}
# AWS EC2 API Block Device Mapping name to Linux NVME device name
disk_name="/dev/$(readlink "$device_name")"

# partition the disk (single data partition)
parted -a optimal -s $disk_name \
mklabel gpt \
mkpart data xfs 0% 90%

# wait for the disk to settle
sleep 5

# install an xfs filesystem to the disk
mkfs -t xfs "${disk_name}p1"

# check if the folder already exists
if [ -d "${folder_path}" ]; then
FILE=$(ls -A ${folder_path})
>&2 echo $FILE
mkdir -p ${temp_path}
mount ${disk_name} ${temp_path}
mount "${disk_name}p1" ${temp_path}
# Empty folder give error on /*
if [ ! -z "$FILE" ]; then
cp -Rax ${folder_path}/* ${temp_path}
Expand All @@ -42,7 +53,7 @@ migrate_and_mount_disk() {
mkdir -p ${folder_path}

# add the mount point to fstab and mount the disk
echo "UUID=$(blkid -s UUID -o value ${disk_name}) ${folder_path} ext4 ${mount_options} 0 1" >> /etc/fstab
echo "UUID=$(blkid -s UUID -o value "${disk_name}p1") ${folder_path} xfs ${mount_options} 0 1" >> /etc/fstab
mount -a

# if selinux is enabled restore the objects on it
Expand All @@ -51,27 +62,28 @@ migrate_and_mount_disk() {
fi
}

disk_name='/dev/nvme1n1'

# partition the disk
parted -a optimal -s $disk_name \
mklabel gpt \
mkpart var ext4 0% 20% \
mkpart varlog ext4 20% 40% \
mkpart varlogaudit ext4 40% 60% \
mkpart home ext4 60% 70% \
mkpart varlibdocker ext4 70% 90%

# wait for the disks to settle
sleep 5

# migrate and mount the existing
migrate_and_mount_disk "${disk_name}p1" /var defaults,nofail,nodev
migrate_and_mount_disk "${disk_name}p2" /var/log defaults,nofail,nodev,nosuid
migrate_and_mount_disk "${disk_name}p3" /var/log/audit defaults,nofail,nodev,nosuid
migrate_and_mount_disk "${disk_name}p4" /home defaults,nofail,nodev,nosuid

# Create folder instead of starting/stopping docker daemon
mkdir -p /var/lib/docker
chown -R root:docker /var/lib/docker
migrate_and_mount_disk "${disk_name}p5" /var/lib/docker defaults,nofail
# migrate and mount the existing folders to dedicated EBS Volumes
migrate_and_mount_disk "/dev/sdf" "/home" defaults,nofail,nodev,nosuid
migrate_and_mount_disk "/dev/sdg" "/var" defaults,nofail,nodev
migrate_and_mount_disk "/dev/sdh" "/var/log" defaults,nofail,nodev,nosuid
migrate_and_mount_disk "/dev/sdi" "/var/log/audit" defaults,nofail,nodev,nosuid
migrate_and_mount_disk "/dev/sdj" "/var/lib/containerd" defaults,nofail

# Resize on instance launch
cloud_init_script="/var/lib/cloud/scripts/per-boot/resize-disks.sh"
cat > "$cloud_init_script" <<EOF
#!/usr/bin/env bash
set -x
lsblk
growpart "/dev/\$(readlink "/dev/sdf")" 1; xfs_growfs '/home'
growpart "/dev/\$(readlink "/dev/sdg")" 1; xfs_growfs '/var'
growpart "/dev/\$(readlink "/dev/sdh")" 1; xfs_growfs '/var/log'
growpart "/dev/\$(readlink "/dev/sdi")" 1; xfs_growfs '/var/log/audit'
growpart "/dev/\$(readlink "/dev/sdj")" 1; xfs_growfs '/var/lib/containerd'
df -Th | grep -E 'Filesystem|xfs'
EOF
chmod +x "$cloud_init_script"
44 changes: 34 additions & 10 deletions variables.pkr.hcl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,45 @@ variable "aws_region" {
default = "us-west-2"
}

variable "data_volume_size" {
description = "Size of the AMI data EBS volume"
type = number
default = 50
variable "eks_version" {
description = "The EKS cluster version associated with the AMI created"
type = string
default = "1.22"
}

variable "root_volume_size" {
description = "Size of the AMI root EBS volume"
type = number
default = 4
}

variable "home_volume_size" {
description = "Size of the AMI /home EBS volume"
type = number
default = 1
}

variable "var_volume_size" {
description = "Size of the AMI /var EBS volume"
type = number
default = 4
}

variable "varlog_volume_size" {
description = "Size of the AMI /var/log EBS volume"
type = number
default = 1
}

variable "varlogaudit_volume_size" {
description = "Size of the AMI /var/log/audit EBS volume"
type = number
default = 1
}

variable "varlibcontainerd_volume_size" {
description = "Size of the AMI /var/lib/containerd EBS volume"
type = number
default = 10
}

Expand All @@ -34,12 +64,6 @@ variable "region_kms_key_ids" {
default = null
}

variable "eks_version" {
description = "The EKS cluster version associated with the AMI created"
type = string
default = "1.22"
}

variable "http_proxy" {
description = "The HTTP proxy to set on the AMI created"
type = string
Expand Down

0 comments on commit 195fe68

Please sign in to comment.