From bf8a92f5a0d7465a99e0f5b9e5138faed6845b10 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 30 May 2024 08:28:42 -0600 Subject: [PATCH 1/3] updates to scripting and documentation for generating AWS AMI images for arm64 --- docs/kubernetes-eks.md | 2 +- docs/third-party-envs.md | 39 ++++++++++++------- .../aws/ami/packer_build.json | 7 ++-- .../aws/ami/packer_vars.json.example | 1 + .../aws/ami/scripts/Malcolm_AMI_Setup.sh | 4 +- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md index 45c303b6d..e724cb867 100644 --- a/docs/kubernetes-eks.md +++ b/docs/kubernetes-eks.md @@ -31,7 +31,7 @@ This document assumes good working knowledge of Amazon Web Services (AWS) and Am aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml ``` 1. Create a [node group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) - * `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm, but users' needs may vary (see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm) + * For x86-64 instances `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm; or , for arm64 instances, `m6gd.2xlarge`, `m6g.2xlarge`, `m7g.2xlarge`, and `t4g.2xlarge`; but users' needs may vary (see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm) * set the nodes to run on the VPC's public subnets 1. [Deploy `metrics-server`](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) ```bash diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md index 34a8f9499..0c883445a 100644 --- a/docs/third-party-envs.md +++ b/docs/third-party-envs.md @@ -8,7 +8,7 @@ ## Generating a Malcolm Amazon Machine Image (AMI) for Use on Amazon Web Services (AWS) -This section outlines the process of using [packer](https://www.packer.io/)'s [Amazon AMI Builder](https://developer.hashicorp.com/packer/plugins/builders/amazon) to create an [EBS-backed](https://developer.hashicorp.com/packer/plugins/builders/amazon/ebs) Malcolm AMI. This section assumes you have good working knowledge of [Amazon Web Services (AWS)](https://docs.aws.amazon.com/index.html). +This section outlines the process of using [packer](https://www.packer.io/)'s [Amazon AMI Builder](https://developer.hashicorp.com/packer/plugins/builders/amazon) to create an [EBS-backed](https://developer.hashicorp.com/packer/plugins/builders/amazon/ebs) Malcolm AMI for either the x86-64 or arm64 CPU architecture. This section assumes you have good working knowledge of [Amazon Web Services (AWS)](https://docs.aws.amazon.com/index.html). ### Prerequisites @@ -30,7 +30,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro $ cp ./packer_vars.json.example ./packer_vars.json ``` 1. Edit `packer_vars.json` - * set `aws_access_key`, `aws_secret_key`, `vpc_region`, and other variables as needed + * set `aws_access_key`, `aws_secret_key`, `vpc_region`, `instance_arch`, and other variables as needed 1. Validate the packer configuration ```bash $ packer validate packer_build.json @@ -43,20 +43,26 @@ The files referenced in this section can be found in [scripts/third-party-enviro amazon-ebs: output will be in this color. ==> amazon-ebs: Prevalidating any provided VPC information - ==> amazon-ebs: Prevalidating AMI Name: malcolm-amzn2_v1-2023-05-30T21-12-22Z - amazon-ebs: Found Image ID: ami-0bef6cc322bfff646 + ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z + amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx ... ==> amazon-ebs: Waiting for AMI to become ready... ==> amazon-ebs: Skipping Enable AMI deprecation... + ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)... + ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx + ==> amazon-ebs: Creating AMI tags + amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.05.0" + amazon-ebs: Adding tag: "source_ami_name": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2" + ==> amazon-ebs: Creating snapshot tags ==> amazon-ebs: Terminating the source AWS instance... ==> amazon-ebs: Cleaning up any extra volumes... ==> amazon-ebs: No volumes to clean up, skipping ==> amazon-ebs: Deleting temporary keypair... - Build 'amazon-ebs' finished after 3 minutes 47 seconds. + Build 'amazon-ebs' finished after 23 minutes 58 seconds. - ==> Wait completed after 3 minutes 47 seconds + ==> Wait completed after 23 minutes 58 seconds ==> Builds finished. The artifacts of successful builds are: --> amazon-ebs: AMIs were created: @@ -70,10 +76,10 @@ The files referenced in this section can be found in [scripts/third-party-enviro { "Images": [ { - "Architecture": "x86_64", - "CreationDate": "2023-05-31T17:07:42.000Z", + "Architecture": "arm64", + "CreationDate": "2024-05-30T14:02:21.000Z", "ImageId": "ami-xxxxxxxxxxxxxxxxx", - "ImageLocation": "xxxxxxxxxxxx/malcolm-v23.05.1-2023-05-31T16-58-00Z", + "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z", "ImageType": "machine", "Public": false, "OwnerId": "xxxxxxxxxxxx", @@ -94,27 +100,32 @@ The files referenced in this section can be found in [scripts/third-party-enviro ], "EnaSupport": true, "Hypervisor": "xen", - "Name": "malcolm-v23.05.1-2023-05-31T16-58-00Z", + "Name": "malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z", "RootDeviceName": "/dev/xvda", "RootDeviceType": "ebs", "SriovNetSupport": "simple", "Tags": [ { "Key": "Malcolm", - "Value": "idaholab/Malcolm/v23.05.1" + "Value": "idaholab/Malcolm/v24.05.0" }, { "Key": "source_ami_name", - "Value": "amzn2-ami-kernel-5.10-hvm-2.0.20230515.0-x86_64-gp2" + "Value": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2" } ], - "VirtualizationType": "hvm" + "VirtualizationType": "hvm", + "BootMode": "uefi", + "SourceInstanceId": "i-xxxxxxxxxxxxxxxxx", + "DeregistrationProtection": "disabled" } ] } ``` 1. Launch an instance from the new AMI - * `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm + * for x86-64 instances `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm + * for arm64 instances, `m6gd.2xlarge`, `m6g.2xlarge`, `m7g.2xlarge`, and `t4g.2xlarge` seem to be good instance types for Malcolm + * see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm 1. SSH into the instance 1. Run `~/Malcolm/scripts/configure` to configure Malcolm 1. Run `~/Malcolm/scripts/auth_setup` to set up authentication for Malcolm diff --git a/scripts/third-party-environments/aws/ami/packer_build.json b/scripts/third-party-environments/aws/ami/packer_build.json index 03d900881..2d8f2c0d8 100644 --- a/scripts/third-party-environments/aws/ami/packer_build.json +++ b/scripts/third-party-environments/aws/ami/packer_build.json @@ -4,6 +4,7 @@ "aws_secret_key": "${aws_secret_key}", "vpc_region": "${vpc_region}", "instance_type": "${instance_type}", + "instance_arch": "${instance_arch}", "ssh_username": "${ssh_username}", "vpc_tag_name_filter": "${vpc_tag_name_filter}", "vpc_subnet_tag_name_filter": "${vpc_subnet_tag_name_filter}", @@ -36,9 +37,9 @@ }, "source_ami_filter": { "filters": { - "architecture": "x86_64", + "architecture": "{{user `instance_arch`}}", "virtualization-type": "hvm", - "name": "amzn2-ami-kernel-5.10-hvm-*-x86_64-gp2", + "name": "amzn2-ami-kernel-5.10-hvm-*-{{user `instance_arch`}}-gp2", "root-device-type": "ebs" }, "owners": [ @@ -68,7 +69,7 @@ "source_ami_name": "{{ .SourceAMIName }}", "Malcolm": "{{user `malcolm_repo`}}/{{user `malcolm_tag`}}" }, - "ami_name": "malcolm-{{user `malcolm_tag`}}-{{isotime | clean_resource_name}}" + "ami_name": "malcolm-{{user `malcolm_tag`}}-{{user `instance_arch`}}-{{isotime | clean_resource_name}}" } ], "provisioners": [ diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example index fe86f2ca4..131e09c6a 100644 --- a/scripts/third-party-environments/aws/ami/packer_vars.json.example +++ b/scripts/third-party-environments/aws/ami/packer_vars.json.example @@ -2,6 +2,7 @@ "aws_access_key": "XXXXXXXXXXXXXXXXXXXX", "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "instance_type": "t2.micro", + "instance_arch": "x86_64", "malcolm_tag": "v24.05.0", "malcolm_repo": "idaholab/Malcolm", "malcolm_uid": "1000", diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh index dc019b92a..45cc9a653 100755 --- a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh +++ b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh @@ -32,7 +32,7 @@ fi # -u UID (user UID, e.g., 1000) VERBOSE_FLAG= MALCOLM_REPO=${MALCOLM_REPO:-idaholab/Malcolm} -MALCOLM_TAG=${MALCOLM_TAG:-v23.10.0} +MALCOLM_TAG=${MALCOLM_TAG:-v24.05.0} [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" ) while getopts 'vr:t:u:' OPTION; do case "$OPTION" in @@ -70,6 +70,7 @@ MALCOLM_USER="$(id -nu $MALCOLM_UID)" MALCOLM_USER_GROUP="$(id -gn $MALCOLM_UID)" MALCOLM_USER_HOME="$(getent passwd "$MALCOLM_USER" | cut -d: -f6)" MALCOLM_URL="https://codeload.github.com/$MALCOLM_REPO/tar.gz/$MALCOLM_TAG" +IMAGE_ARCH_SUFFIX="$(uname -m | sed 's/^x86_64$//' | sed 's/^arm64$/-arm64/' | sed 's/^aarch64$/-arm64/')" ################################################################################### # InstallEssentialPackages @@ -215,6 +216,7 @@ function InstallMalcolm { if [[ -s ./Malcolm/docker-compose.yml ]]; then pushd ./Malcolm >/dev/null 2>&1 for ENVEXAMPLE in ./config/*.example; do ENVFILE="${ENVEXAMPLE%.*}"; cp "$ENVEXAMPLE" "$ENVFILE"; done + sed -i "s@\(/malcolm/.*\):\(.*\)@\1:\2${IMAGE_ARCH_SUFFIX}@g" docker-compose.yml echo "Pulling Docker images..." >&2 docker-compose --profile malcolm pull >/dev/null 2>&1 rm -f ./config/*.env From 85c61a429a8ca550d483869e4a539d5582867dcc Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 30 May 2024 08:39:58 -0600 Subject: [PATCH 2/3] do mkdir on arkime directories outside of check for maxmind API token --- Dockerfiles/arkime.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index a582654d5..2023eedf8 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -179,11 +179,11 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic # see https://dev.maxmind.com/geoip/geoipupdate/#Direct_Downloads # see https://github.com/arkime/arkime/issues/1350 # see https://github.com/arkime/arkime/issues/1352 -RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \ +RUN mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \ + [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \ cd /tmp && \ curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \ tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \ - mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \ mv -v "GeoLite2-$DB.mmdb" $ARKIME_DIR/etc/; \ rm -f "GeoLite2-$DB*"; \ done; \ From 52ee55305ee97cc5ff21d7fabef2d2b82c3dc6a9 Mon Sep 17 00:00:00 2001 From: Seth Grover Date: Thu, 30 May 2024 08:45:05 -0600 Subject: [PATCH 3/3] do mkdir on arkime directories outside of check for maxmind API token --- Dockerfiles/arkime.Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile index 2023eedf8..87932d1e9 100644 --- a/Dockerfiles/arkime.Dockerfile +++ b/Dockerfiles/arkime.Dockerfile @@ -179,8 +179,7 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic # see https://dev.maxmind.com/geoip/geoipupdate/#Direct_Downloads # see https://github.com/arkime/arkime/issues/1350 # see https://github.com/arkime/arkime/issues/1352 -RUN mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \ - [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \ +RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do \ cd /tmp && \ curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \ tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \ @@ -202,7 +201,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \ chown root:${PGROUP} /sbin/ethtool && \ setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \ - mkdir -p /var/run/arkime && \ + mkdir -p /var/run/arkime $ARKIME_DIR/logs && \ chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/rules $ARKIME_DIR/logs /var/run/arkime #Update Path ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}"