Merge branch 'development' of https://github.com/mmguero-dev/Malcolm

cisagov · May 30, 2024 · ac111cc · ac111cc
2 parents f54cfd8 + 52ee553
commit ac111cc
Show file tree

Hide file tree

Showing 6 changed files with 35 additions and 21 deletions.
diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
@@ -183,7 +183,6 @@ RUN [ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ] && for DB in ASN Country City; do
       cd /tmp && \
       curl -s -S -L -o "GeoLite2-$DB.mmdb.tar.gz" "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-$DB&license_key=$MAXMIND_GEOIP_DB_LICENSE_KEY&suffix=tar.gz" && \
       tar xf "GeoLite2-$DB.mmdb.tar.gz" --wildcards --no-anchored '*.mmdb' --strip=1 && \
-      mkdir -p $ARKIME_DIR/etc/ $ARKIME_DIR/rules/ $ARKIME_DIR/logs/ && \
       mv -v "GeoLite2-$DB.mmdb" $ARKIME_DIR/etc/; \
       rm -f "GeoLite2-$DB*"; \
     done; \
@@ -202,7 +201,7 @@ RUN groupadd --gid $DEFAULT_GID $PGROUP && \
       setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip CAP_IPC_LOCK+eip' $ARKIME_DIR/bin/capture && \
     chown root:${PGROUP} /sbin/ethtool && \
       setcap 'CAP_NET_RAW+eip CAP_NET_ADMIN+eip' /sbin/ethtool && \
-    mkdir -p /var/run/arkime && \
+    mkdir -p /var/run/arkime $ARKIME_DIR/logs && \
     chown -R $PUSER:$PGROUP $ARKIME_DIR/etc $ARKIME_DIR/rules $ARKIME_DIR/logs /var/run/arkime
 #Update Path
 ENV PATH="/opt:$ARKIME_DIR/bin:${PATH}"

diff --git a/docs/kubernetes-eks.md b/docs/kubernetes-eks.md
@@ -31,7 +31,7 @@ This document assumes good working knowledge of Amazon Web Services (AWS) and Am
     aws eks update-kubeconfig --region us-east-1 --name cluster-name --kubeconfig malcolmeks.yaml
     ```
 1. Create a [node group](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html)
-    * `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm, but users' needs may vary (see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm)
+    * For x86-64 instances `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm; or , for arm64 instances, `m6gd.2xlarge`, `m6g.2xlarge`, `m7g.2xlarge`, and `t4g.2xlarge`; but users' needs may vary (see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm)
     * set the nodes to run on the VPC's public subnets
 1. [Deploy `metrics-server`](https://docs.aws.amazon.com/eks/latest/userguide/metrics-server.html) 
     ```bash

diff --git a/docs/third-party-envs.md b/docs/third-party-envs.md
@@ -8,7 +8,7 @@
 
 ## <a name="AWSAMI"></a>Generating a Malcolm Amazon Machine Image (AMI) for Use on Amazon Web Services (AWS)
 
-This section outlines the process of using [packer](https://www.packer.io/)'s [Amazon AMI Builder](https://developer.hashicorp.com/packer/plugins/builders/amazon) to create an [EBS-backed](https://developer.hashicorp.com/packer/plugins/builders/amazon/ebs) Malcolm AMI. This section assumes you have good working knowledge of [Amazon Web Services (AWS)](https://docs.aws.amazon.com/index.html).
+This section outlines the process of using [packer](https://www.packer.io/)'s [Amazon AMI Builder](https://developer.hashicorp.com/packer/plugins/builders/amazon) to create an [EBS-backed](https://developer.hashicorp.com/packer/plugins/builders/amazon/ebs) Malcolm AMI for either the x86-64 or arm64 CPU architecture. This section assumes you have good working knowledge of [Amazon Web Services (AWS)](https://docs.aws.amazon.com/index.html).
 
 ### <a name="AWSAMIPrerequisites"></a> Prerequisites
 
@@ -30,7 +30,7 @@ The files referenced in this section can be found in [scripts/third-party-enviro
     $ cp ./packer_vars.json.example ./packer_vars.json
     ```
 1. Edit `packer_vars.json` 
-    * set `aws_access_key`, `aws_secret_key`, `vpc_region`, and other variables as needed
+    * set `aws_access_key`, `aws_secret_key`, `vpc_region`, `instance_arch`, and other variables as needed
 1. Validate the packer configuration
     ```bash
     $ packer validate packer_build.json
@@ -43,20 +43,26 @@ The files referenced in this section can be found in [scripts/third-party-enviro
     amazon-ebs: output will be in this color.
 
     ==> amazon-ebs: Prevalidating any provided VPC information
-    ==> amazon-ebs: Prevalidating AMI Name: malcolm-amzn2_v1-2023-05-30T21-12-22Z
-        amazon-ebs: Found Image ID: ami-0bef6cc322bfff646
+    ==> amazon-ebs: Prevalidating AMI Name: malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z
+        amazon-ebs: Found Image ID: ami-xxxxxxxxxxxxxxxxx
 
     ...
 
     ==> amazon-ebs: Waiting for AMI to become ready...
     ==> amazon-ebs: Skipping Enable AMI deprecation...
+    ==> amazon-ebs: Adding tags to AMI (ami-xxxxxxxxxxxxxxxxx)...
+    ==> amazon-ebs: Tagging snapshot: snap-xxxxxxxxxxxxxxxxx
+    ==> amazon-ebs: Creating AMI tags
+        amazon-ebs: Adding tag: "Malcolm": "idaholab/Malcolm/v24.05.0"
+        amazon-ebs: Adding tag: "source_ami_name": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2"
+    ==> amazon-ebs: Creating snapshot tags
     ==> amazon-ebs: Terminating the source AWS instance...
     ==> amazon-ebs: Cleaning up any extra volumes...
     ==> amazon-ebs: No volumes to clean up, skipping
     ==> amazon-ebs: Deleting temporary keypair...
-    Build 'amazon-ebs' finished after 3 minutes 47 seconds.
+    Build 'amazon-ebs' finished after 23 minutes 58 seconds.
 
-    ==> Wait completed after 3 minutes 47 seconds
+    ==> Wait completed after 23 minutes 58 seconds
 
     ==> Builds finished. The artifacts of successful builds are:
     --> amazon-ebs: AMIs were created:
@@ -70,10 +76,10 @@ The files referenced in this section can be found in [scripts/third-party-enviro
     {
         "Images": [
             {
-                "Architecture": "x86_64",
-                "CreationDate": "2023-05-31T17:07:42.000Z",
+                "Architecture": "arm64",
+                "CreationDate": "2024-05-30T14:02:21.000Z",
                 "ImageId": "ami-xxxxxxxxxxxxxxxxx",
-                "ImageLocation": "xxxxxxxxxxxx/malcolm-v23.05.1-2023-05-31T16-58-00Z",
+                "ImageLocation": "xxxxxxxxxxxx/malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z",
                 "ImageType": "machine",
                 "Public": false,
                 "OwnerId": "xxxxxxxxxxxx",
@@ -94,27 +100,32 @@ The files referenced in this section can be found in [scripts/third-party-enviro
                 ],
                 "EnaSupport": true,
                 "Hypervisor": "xen",
-                "Name": "malcolm-v23.05.1-2023-05-31T16-58-00Z",
+                "Name": "malcolm-v24.05.0-arm64-2024-05-30T13-57-31Z",
                 "RootDeviceName": "/dev/xvda",
                 "RootDeviceType": "ebs",
                 "SriovNetSupport": "simple",
                 "Tags": [
                     {
                         "Key": "Malcolm",
-                        "Value": "idaholab/Malcolm/v23.05.1"
+                        "Value": "idaholab/Malcolm/v24.05.0"
                     },
                     {
                         "Key": "source_ami_name",
-                        "Value": "amzn2-ami-kernel-5.10-hvm-2.0.20230515.0-x86_64-gp2"
+                        "Value": "amzn2-ami-kernel-5.10-hvm-2.0.20240521.0-arm64-gp2"
                     }
                 ],
-                "VirtualizationType": "hvm"
+                "VirtualizationType": "hvm",
+                "BootMode": "uefi",
+                "SourceInstanceId": "i-xxxxxxxxxxxxxxxxx",
+                "DeregistrationProtection": "disabled"
             }
         ]
     }
     ```
 1. Launch an instance from the new AMI
-    * `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm
+    * for x86-64 instances `c4.4xlarge`, `t2.2xlarge`, and `t3a.2xlarge` seem to be good instance types for Malcolm
+    * for arm64 instances, `m6gd.2xlarge`, `m6g.2xlarge`, `m7g.2xlarge`, and `t4g.2xlarge` seem to be good instance types for Malcolm
+    * see [recommended system requirements](system-requirements.md#SystemRequirements) for Malcolm
 1. SSH into the instance
 1. Run `~/Malcolm/scripts/configure` to configure Malcolm
 1. Run `~/Malcolm/scripts/auth_setup` to set up authentication for Malcolm

diff --git a/scripts/third-party-environments/aws/ami/packer_build.json b/scripts/third-party-environments/aws/ami/packer_build.json
@@ -4,6 +4,7 @@
     "aws_secret_key": "${aws_secret_key}",
     "vpc_region": "${vpc_region}",
     "instance_type": "${instance_type}",
+    "instance_arch": "${instance_arch}",
     "ssh_username": "${ssh_username}",
     "vpc_tag_name_filter": "${vpc_tag_name_filter}",
     "vpc_subnet_tag_name_filter": "${vpc_subnet_tag_name_filter}",
@@ -36,9 +37,9 @@
       },
       "source_ami_filter": {
         "filters": {
-          "architecture": "x86_64",
+          "architecture": "{{user `instance_arch`}}",
           "virtualization-type": "hvm",
-          "name": "amzn2-ami-kernel-5.10-hvm-*-x86_64-gp2",
+          "name": "amzn2-ami-kernel-5.10-hvm-*-{{user `instance_arch`}}-gp2",
           "root-device-type": "ebs"
         },
         "owners": [
@@ -68,7 +69,7 @@
         "source_ami_name": "{{ .SourceAMIName }}",
         "Malcolm": "{{user `malcolm_repo`}}/{{user `malcolm_tag`}}"
       },
-      "ami_name": "malcolm-{{user `malcolm_tag`}}-{{isotime | clean_resource_name}}"
+      "ami_name": "malcolm-{{user `malcolm_tag`}}-{{user `instance_arch`}}-{{isotime | clean_resource_name}}"
     }
   ],
   "provisioners": [

diff --git a/scripts/third-party-environments/aws/ami/packer_vars.json.example b/scripts/third-party-environments/aws/ami/packer_vars.json.example
@@ -2,6 +2,7 @@
   "aws_access_key": "XXXXXXXXXXXXXXXXXXXX",
   "aws_secret_key": "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
   "instance_type": "t2.micro",
+  "instance_arch": "x86_64",
   "malcolm_tag": "v24.05.0",
   "malcolm_repo": "cisagov/Malcolm",
   "malcolm_uid": "1000",

diff --git a/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh b/scripts/third-party-environments/aws/ami/scripts/Malcolm_AMI_Setup.sh
@@ -32,7 +32,7 @@ fi
 # -u UID      (user UID, e.g., 1000)
 VERBOSE_FLAG=
 MALCOLM_REPO=${MALCOLM_REPO:-cisagov/Malcolm}
-MALCOLM_TAG=${MALCOLM_TAG:-v23.10.0}
+MALCOLM_TAG=${MALCOLM_TAG:-v24.05.0}
 [[ -z "$MALCOLM_UID" ]] && ( [[ $EUID -eq 0 ]] && MALCOLM_UID=1000 || MALCOLM_UID="$(id -u)" )
 while getopts 'vr:t:u:' OPTION; do
   case "$OPTION" in
@@ -70,6 +70,7 @@ MALCOLM_USER="$(id -nu $MALCOLM_UID)"
 MALCOLM_USER_GROUP="$(id -gn $MALCOLM_UID)"
 MALCOLM_USER_HOME="$(getent passwd "$MALCOLM_USER" | cut -d: -f6)"
 MALCOLM_URL="https://codeload.github.com/$MALCOLM_REPO/tar.gz/$MALCOLM_TAG"
+IMAGE_ARCH_SUFFIX="$(uname -m | sed 's/^x86_64$//' | sed 's/^arm64$/-arm64/' | sed 's/^aarch64$/-arm64/')"
 
 ###################################################################################
 # InstallEssentialPackages
@@ -215,6 +216,7 @@ function InstallMalcolm {
     if [[ -s ./Malcolm/docker-compose.yml ]]; then
         pushd ./Malcolm >/dev/null 2>&1
         for ENVEXAMPLE in ./config/*.example; do ENVFILE="${ENVEXAMPLE%.*}"; cp "$ENVEXAMPLE" "$ENVFILE"; done
+        sed -i "s@\(/malcolm/.*\):\(.*\)@\1:\2${IMAGE_ARCH_SUFFIX}@g" docker-compose.yml
         echo "Pulling Docker images..." >&2
         docker-compose --profile malcolm pull >/dev/null 2>&1
         rm -f ./config/*.env