3166 aws cleanup

.github/cleanup-old-images.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+#set -x
+
+# AWS wrapper without passing a region (AWS(N)o(R)egion)
+# Redefined here so that we can source this file alone if needed.
+# The other copy is in cleanup-old-images.sh script.
+AWSNR() {
+  if [ -z "$AWS_PROFILE" ]; then
+    aws "$@"
+  else
+    aws --profile "$AWS_PROFILE" "$@"
+  fi
+}
+
+amiDeleteIfNotInVersionList() {
+  local reg=$1
+  local img=$2
+  shift 2
+  local versionList=("$@")
+
+  # TODO:
+  # Make the script stop when things fail (it didn't?). Like so:
+  # $(dosomething | tail -1 | tee /dev/fd/2)
+
+  # get all image tags
+  mapfile -t imgTags < <(AWSNR --region "$reg" ec2 describe-images --image-ids "$img" --query 'Images[].Tags[]' --output text)
+  TagExists=false
+  for tag in "${imgTags[@]}"; do
+    for tagToCheck in "${versionList[@]}"; do
+      if [[ $tag == "KairosVersion"*"$tagToCheck" ]]; then
+        echo "[$reg] AMI $img has the '$tagToCheck' tag. Skipping cleanup."
+        TagExists=true
+        break 2
+      fi
+    done
+  done
+
+  if [ "$TagExists" = false ]; then
+      AWSNR --region "$reg" ec2 deregister-image --image-id "$img"
+      echo "[$reg] AMI $img deleted because it does not match any of the versions: '${versionList[*]}'."
+  fi
+}
+
+snapshotDeleteIfNotInVersionList() {
+  local reg=$1
+  local snapshot=$2
+  shift 2
+  local versionList=("$@")
+
+  # Get all snapshot tags
+  mapfile -t snapshotTags < <(AWSNR --region "$reg" ec2 describe-snapshots --snapshot-ids "$snapshot" --query 'Snapshots[].Tags[]' --output text)
+  TagExists=false
+  for tag in "${snapshotTags[@]}"; do
+    for tagToCheck in "${versionList[@]}"; do
+      if [[ $tag == "KairosVersion"*"$tagToCheck" ]]; then
+        echo "[$reg] Snapshot $snapshot has the '$tagToCheck' tag. Skipping cleanup."
+        TagExists=true
+        break
+      fi
+    done
+  done
+
+  if [ "$TagExists" = false ]; then
+    (AWSNR --region "$reg" ec2 delete-snapshot --snapshot-id "$snapshot" && \
+      echo "[$reg] Snapshot $snapshot deleted because it does not match any of the versions: '${versionList[*]}'.") || true
+  fi
+}
+
+s3ObjectDeleteIfNotInVersionList() {
+  local bucket=$1
+  local key=$2
+  shift 2
+  local versionList=("$@")
+
+  # Get all S3 object tags
+  mapfile -t s3Tags < <(AWSNR s3api get-object-tagging --bucket "$bucket" --key "$key" --query 'TagSet[]' --output text)
+
+  TagExists=false
+  for tag in "${s3Tags[@]}"; do
+    for tagToCheck in "${versionList[@]}"; do
+      if [[ $tag == "KairosVersion"*"$tagToCheck" ]]; then
+        echo "S3 object '$key' in bucket '$bucket' has the '$tagToCheck' tag. Skipping cleanup."
+        TagExists=true
+        break 2
+      fi
+    done
+  done
+
+  if [ "$TagExists" = false ]; then
+    AWSNR s3api delete-object --bucket "$bucket" --key "$key"
+    echo "S3 object $key in bucket $bucket deleted because it does not match any of the versions: '${versionList[*]}'."
+  fi
+}
+
+getHighest4StableVersions() {
+  local reg=$1
+  local kairosVersions
+  local stableVersions=()
+  local sortedVersions
+  local highest4StableVersions
+
+  # Get all Kairos versions
+  mapfile -t kairosVersions < <(AWSNR --region "$reg" ec2 describe-images --owners self --query "Images[].Tags[?Key=='KairosVersion'].Value" --output text)
+
+  # Filter out non-stable versions (those containing '-rc')
+  for version in "${kairosVersions[@]}"; do
+    if [[ ! $version =~ -rc ]]; then
+      stableVersions+=("$version")
+    fi
+  done
+
+  # Sort the stable versions and keep only the highest 4
+  IFS=$'\n' mapfile -t sortedVersions < <(sort -V -r <<<"${stableVersions[*]}")
+  unset IFS
+  highest4StableVersions=("${sortedVersions[@]:0:4}")
+
+  # Return the highest 4 stable versions
+  echo "${highest4StableVersions[@]}"
+}
+
+cleanupOldVersionsRegion() {
+  local reg=$1
+  shift 1
+  local versionList=("$@")
+
+  # Cleanup AMIs
+  mapfile -t allAmis < <(AWSNR --region "$reg" ec2 describe-images --owners self --query 'Images[].ImageId' --output text | tr '\t' '\n')
+  for img in "${allAmis[@]}"; do
+    amiDeleteIfNotInVersionList "$reg" "$img" "${versionList[@]}"
+  done
+
+  # Cleanup Snapshots
+  mapfile -t allSnapshots < <(AWSNR --region "$reg" ec2 describe-snapshots --owner-ids self --query 'Snapshots[].SnapshotId' --output text | tr '\t' '\n')
+  for snapshot in "${allSnapshots[@]}"; do
+    snapshotDeleteIfNotInVersionList "$reg" "$snapshot" "${versionList[@]}"
+  done
+}
+
+cleanupOldVersions() {
+  mapfile -t highest4StableVersions < <(getHighest4StableVersions "$AWS_REGION")
+  echo "Highest 4 stable versions (in region $AWS_REGION): ${highest4StableVersions[*]}"
+
+  mapfile -t regions < <(AWSNR ec2 describe-regions | jq -r '.Regions[].RegionName')
+  for reg in "${regions[@]}"; do
+    cleanupOldVersionsRegion "$reg" "${highest4StableVersions[@]}"
+  done
+
+  # Cleanup S3 Objects
+  mapfile -t allS3Objects < <(AWSNR s3api list-objects-v2 --bucket "$AWS_S3_BUCKET" --query 'Contents[].Key' --output text| tr '\t' '\n')
+  for s3Object in "${allS3Objects[@]}"; do
+    s3ObjectDeleteIfNotInVersionList "$AWS_S3_BUCKET" "$s3Object" "${highest4StableVersions[@]}"
+  done
+}

.github/upload-image-to-aws.sh

@@ -10,10 +10,14 @@
 set -e
 set -o pipefail
 
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=/dev/null
+source "$SCRIPT_DIR/cleanup-old-images.sh"
+
 checkArguments() {
-  if [ $# -lt 1 ]; then
-    echo "Error: You need to specify the cloud image to upload."
-    echo "Usage: $0 <cloud-image>"
+  if [ $# -lt 2 ]; then
+    echo "Error: You need to specify the cloud image to upload and the Kairos version (to tag resources)."
+    echo "Usage: $0 <cloud-image> <kairos-version>"
     exit 1
   fi
 
@@ -45,6 +49,15 @@ AWS() {
   fi
 }
 
+# AWS wrapper without passing a region (AWS(N)o(R)egion)
+AWSNR() {
+  if [ -z "$AWS_PROFILE" ]; then
+    aws "$@"
+  else
+    aws --profile "$AWS_PROFILE" "$@"
+  fi
+}
+
 # https://docs.aws.amazon.com/vm-import/latest/userguide/required-permissions.html#vmimport-role
 ensureVmImportRole() {
   (AWS iam list-roles | jq -r '.Roles[] | select(.RoleName | contains("vmimport")) | .RoleName' | grep -q "vmimport" && echo "vmimport role found. All good.") || {
@@ -115,13 +128,16 @@ uploadImageToS3() {
   local file
   local baseName
   file="$1"
+  kairosVersion="$2"
   baseName=$(basename "$file")
 
   if AWS s3 ls "$AWS_S3_BUCKET/$baseName" > /dev/null 2>&1; then
     echo "File '$baseName' already exists in S3 bucket '$AWS_S3_BUCKET'."
   else
     echo "File '$baseName' does not exist in S3 bucket '$AWS_S3_BUCKET'. Uploading now."
     AWS s3 cp "$1" "s3://$AWS_S3_BUCKET/$baseName"
+
+    AWS s3api put-object-tagging --bucket "$AWS_S3_BUCKET" --key "$baseName" --tagging "TagSet=[{Key=KairosVersion,Value=$2}]"
   fi
 }
 
@@ -149,6 +165,7 @@ waitForSnapshotCompletion() {
 
 importAsSnapshot() {
   local file="$1"
+  local kairosVersion="$2"
   local snapshotID
 
   snapshotID=$(AWS ec2 describe-snapshots --filters "Name=tag:SourceFile,Values=$file" --query "Snapshots[0].SnapshotId" --output text)
@@ -174,14 +191,16 @@ EOF
 
   snapshotID=$(waitForSnapshotCompletion "$taskID" | tail -1 | tee /dev/fd/2)
   echo "Adding tag to the snapshot with ID: $snapshotID"
-  AWS ec2 create-tags --resources "$snapshotID" --tags "Key=SourceFile,Value=$file"
+  AWS ec2 create-tags --resources "$snapshotID" \
+    --tags Key=Name,Value="${file}" Key=SourceFile,Value="${file}" Key=KairosVersion,Value="${kairosVersion}"
 
   echo "$snapshotID" # Return the snapshot ID so that we can grab it with `tail -1`
 }
 
 checkImageExistsOrCreate() {
   local imageName="$1"
   local snapshotID="$2"
+  local kairosVersion="$3"
   local imageID
 
   # Check if the image already exists
@@ -205,12 +224,15 @@ checkImageExistsOrCreate() {
       --query 'ImageId' \
       --output text)
 
+    AWS ec2 create-tags --resources "$imageID" \
+      --tags Key=KairosVersion,Value="$kairosVersion" Key=Name,Value="$imageName" Key=Project,Value=Kairos
+
     echo "Image '$imageName' created with Image ID: $imageID"
   fi
 
   waitAMI "$imageID" "$AWS_REGION"
   makeAMIpublic "$imageID" "$AWS_REGION"
-  copyToAllRegions "$imageID" "$imageName" "$description"
+  copyToAllRegions "$imageID" "$imageName" "$description" "$kairosVersion"
 }
 
 # Function to wait for the AMI to become available
@@ -220,7 +242,7 @@ waitAMI() {
 
   echo "[$region] Waiting for AMI $amiID to be available"
   while true; do
-    status=$(aws --profile "$AWS_PROFILE" ec2 describe-images --region "$region" --image-ids "$amiID" --query "Images[0].State" --output text 2>/dev/null)
+    status=$(AWSNR ec2 describe-images --region "$region" --image-ids "$amiID" --query "Images[0].State" --output text 2>/dev/null)
     if [[ "$status" == "available" ]]; then
       echo "[$region] AMI $amiID is now available!"
       break
@@ -238,29 +260,29 @@ makeAMIpublic() {
   local region="$2"
 
   echo "[$region] calling DisableImageBlockPublicAccess"
-  aws --profile "$AWS_PROFILE" --region "$region" ec2 disable-image-block-public-access > /dev/null 2>&1
+  AWSNR --region "$region" ec2 disable-image-block-public-access > /dev/null 2>&1
   echo "[$region] Making image '$imageID' public..."
-  aws --profile "$AWS_PROFILE" --region "$region" ec2 modify-image-attribute --image-id "$imageID" --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
+  AWSNR --region "$region" ec2 modify-image-attribute --image-id "$imageID" --launch-permission "{\"Add\":[{\"Group\":\"all\"}]}"
   echo "[$region] Image '$imageID' is now public."
 }
 
 copyToAllRegions() {
   local imageID="$1"
   local imageName="$2"
   local description="$3"
+  local kairosVersion="$4"
 
   echo "Copying AMI '$imageName ($imageID)' to all regions"
   mapfile -t regions < <(AWS ec2 describe-regions | jq -r '.Regions[].RegionName')
   for reg in "${regions[@]}"; do
     # If the current region is the same as the region we are trying to copy, just ignore, the AMI is already there
-    if [[ "${AWS_REGION}" == "${reg}" ]]
-      then
+    if [[ "${AWS_REGION}" == "${reg}" ]]; then
         continue
     fi
     (
       echo "[$reg] Copying AMI '$imageName' to region $reg"
       # Check if the image already exists in this region
-      amiCopyID=$(aws --profile "$AWS_PROFILE" --region "$reg" ec2 describe-images --filters "Name=name,Values=$imageName" --query 'Images[0].ImageId' --output text)
+      amiCopyID=$(AWSNR --region "$reg" ec2 describe-images --filters "Name=name,Values=$imageName" --query 'Images[0].ImageId' --output text)
       if [ "$amiCopyID" != "None" ]; then
         echo "[$reg] Image '$imageName' already exists with Image ID: $amiCopyID"
       else
@@ -274,14 +296,25 @@ copyToAllRegions() {
         )
 
         echo "[$reg] Tagging Copied AMI ${amiCopyID}"
-        AWS ec2 create-tags --resources "${imageID}" --tags \
-          --tags Key=Name,Value="${imageName}" Key=Project,Value=Kairos
       fi
 
       waitAMI "${amiCopyID}" "${reg}"
-      makeAMIpublic "${amiCopyID}" "${reg}"
 
-      echo "[$reg] AMI Copied: ${amiCopyID}"
+      snapshotCopyID=$(AWSNR ec2 describe-images \
+        --image-ids "$amiCopyID" \
+        --region "$reg" \
+        --query 'Images[0].BlockDeviceMappings[0].Ebs.SnapshotId' \
+        --output text)
+      AWSNR --region "$reg" ec2 create-tags \
+        --resources "$snapshotCopyID" \
+        --tags Key=Name,Value="$imageName" Key=SourceFile,Value="$imageName" Key=KairosVersion,Value="$kairosVersion"
+
+      AWSNR --region "$reg" ec2 create-tags \
+        --resources "$amiCopyID" \
+        --tags Key=Name,Value="$imageName" Key=Project,Value=Kairos Key=KairosVersion,Value="$kairosVersion"
+      makeAMIpublic "$amiCopyID" "$reg"
+
+      echo "[$reg] AMI Copied: $amiCopyID"
     ) &
   done
 
@@ -290,11 +323,19 @@ copyToAllRegions() {
 
 # ----- Main script -----
 baseName=$(basename "$1")
+kairosVersion="$2"
 checkEnvVars
 checkArguments "$@"
+
+echo
+echo "Performing cleanup of old versions"
+cleanupOldVersions
+echo "Done cleaning up"
+echo
+
 # This is an one-off operation and require additional permissions which we don't need to give to CI.
 #ensureVmImportRole
-uploadImageToS3 "$1"
-output=$(importAsSnapshot "$baseName" | tee /dev/fd/2)
+uploadImageToS3 "$1" "$kairosVersion"
+output=$(importAsSnapshot "$baseName" "$kairosVersion"| tee /dev/fd/2)
 snapshotID=$(echo "$output" | tail -1)
-checkImageExistsOrCreate "$baseName" "$snapshotID"
+checkImageExistsOrCreate "$baseName" "$snapshotID" "$kairosVersion"