feat(core): update to work with Deadline 10.4.0

integ/components/deadline/common/scripts/bastion/utils/configure-deadline.sh

@@ -30,7 +30,13 @@ else
   # Non-TLS connections can connect to the repository directly
   sudo $DEADLINE/deadlinecommand SetIniFileSetting ProxyUseSSL False
   sudo $DEADLINE/deadlinecommand SetIniFileSetting ProxySSLCA ""
-  sudo $DEADLINE/deadlinecommand ChangeRepository Remote $ENDPOINT >/dev/null
+  ALL_USERS_OPTION=
+  if $DEADLINE/deadlinecommand --help | grep "^ChangeRepository " | grep -q "<Save For All Users>"; then
+    # Deadline 10.4 added an option to "Save For All Users".
+    # This is the default behavior in prior versions.
+    ALL_USERS_OPTION="True"
+  fi
+  sudo $DEADLINE/deadlinecommand ChangeRepository Remote $ENDPOINT '' '' $ALL_USERS_OPTION >/dev/null
 fi
 
 exit 0

integ/components/deadline/deadline_01_repository/bin/deadline_01_repository.ts

@@ -5,7 +5,11 @@
 
 import { App, Stack, Aspects } from 'aws-cdk-lib';
 import { AutoScalingGroupRequireImdsv2Aspect } from 'aws-cdk-lib/aws-autoscaling';
-import { InstanceRequireImdsv2Aspect, LaunchTemplateRequireImdsv2Aspect } from 'aws-cdk-lib/aws-ec2';
+import {
+  InstanceRequireImdsv2Aspect,
+  LaunchTemplateRequireImdsv2Aspect,
+  MachineImage,
+} from 'aws-cdk-lib/aws-ec2';
 import {
   Stage,
   ThinkboxDockerRecipes,
@@ -49,7 +53,14 @@ const structs: Array<StorageStruct> = [
   }),
 ];
 
-new RepositoryTestingTier(app, 'RFDKInteg-DL-TestingTier' + integStackTag, { env, integStackTag, structs });
+new RepositoryTestingTier(app, 'RFDKInteg-DL-TestingTier' + integStackTag, {
+  env,
+  integStackTag,
+  structs,
+  // Currently we test using MongoDB 3.6, which doesn't run on the
+  // Amazon Linux 2023 image that we use to test Deadline 10.4.0.
+  bastionMachineImageOverride: MachineImage.latestAmazonLinux2(),
+});
 
 // Adds IAM Policy to Instance and ASG Roles
 Aspects.of(app).add(new SSMInstancePolicyAspect());

integ/components/deadline/deadline_03_workerFleetHttp/scripts/bastion/testing/WF-submit-jobs-to-sets.sh

@@ -6,19 +6,34 @@
 # Script to submit test Deadline jobs to be picked up by worker nodes based on their assigned group or pool
 #
 # Input:
+#   WORKER_OS: The operating system that the Worker node is running on. Will be either "Linux" or "Windows".
 #   JOB_NAME: Name for the test config/job submitted to Deadline. Will be either "group" or "pool"
 #   ARG: Command line arg added to `deadlinecommand` to submit the job to only the specific group/pool for the test case
 # Output:
 #   A count of the number of entries in the output for `deadlinecommand GetSlavesRenderingJob` for the test job
 
 set -euo pipefail
 
-JOB_NAME=$1
-ARG=$2
+WORKER_OS=$1
+JOB_NAME=$2
+ARG=$3
 DEADLINE="/opt/Thinkbox/Deadline10/bin"
 
+case ${WORKER_OS^^} in
+  LINUX)
+    JOB_EXECUTABLE="/usr/bin/sleep"
+    ;;
+  WINDOWS)
+    JOB_EXECUTABLE="timeout.exe"
+    ;;
+  *)
+    echo "Error: unknown WORKER_OS from command line arguments: ${WORKER_OS}"
+    exit 1
+    ;;
+esac
+
 # Send a sleep command to the render queue; based on the arg passed in, this job will be assigned to the test group or pool
-JOB_ID=$($DEADLINE/deadlinecommand --prettyjson SubmitCommandLineJob -executable "/usr/bin/sleep" -arguments "10" -frames "1-10" -name $JOB_NAME $ARG)
+JOB_ID=$($DEADLINE/deadlinecommand --prettyjson SubmitCommandLineJob -executable "$JOB_EXECUTABLE" -arguments "10" -frames "1-10" -name $JOB_NAME $ARG)
 # We then pull the jobId from the output of `SubmitCommandLineJob`
 JOB_ID=$(jq -r '.result' <<< "$JOB_ID")
 

integ/components/deadline/deadline_03_workerFleetHttp/test/deadline_03_workerFleetHttp.test.ts

@@ -15,8 +15,8 @@ const bastionRegex = /bastionId/;
 const rqRegex = /renderQueueEndpointWF(\d)/;
 
 const testCases: Array<Array<any>> = [
-  [ 'Linux Worker HTTP mode', 1 ],
-  [ 'Windows Worker HTTP mode', 2 ],
+  [ 'Linux Worker HTTP mode', 1, 'Linux' ],
+  [ 'Windows Worker HTTP mode', 2, 'Windows' ],
 ];
 let bastionId: any;
 let renderQueueEndpoints: Array<string> = [];
@@ -45,7 +45,7 @@ beforeAll( async () => {
   });
 });
 
-describe.each(testCases)('Deadline WorkerFleet tests (%s)', (_, id) => {
+describe.each(testCases)('Deadline WorkerFleet tests (%s)', (_, id, workerOS) => {
   describe('Worker node tests', () => {
 
     // Before testing the render queue, send a command to configure the Deadline client to use that endpoint
@@ -139,7 +139,7 @@ describe.each(testCases)('Deadline WorkerFleet tests (%s)', (_, id) => {
             'sudo -i',
             'su - ec2-user >/dev/null',
             'cd ~ec2-user',
-            `./testScripts/WF-submit-jobs-to-sets.sh "${name}" "${arg}"`,
+            `./testScripts/WF-submit-jobs-to-sets.sh "${workerOS}" "${name}" "${arg}"`,
           ],
         },
       };

integ/components/deadline/deadline_04_workerFleetHttps/scripts/bastion/testing/WFS-submit-jobs-to-sets.sh

@@ -6,19 +6,34 @@
 # Script to submit test Deadline jobs to be picked up by worker nodes based on their assigned group or pool
 #
 # Input:
+#   WORKER_OS: The operating system that the Worker node is running on. Will be either "Linux" or "Windows".
 #   JOB_NAME: Name for the test config/job submitted to Deadline. Will be either "group" or "pool"
 #   ARG: Command line arg added to `deadlinecommand` to submit the job to only the specific group/pool for the test case
 # Output:
 #   A count of the number of entries in the output for `deadlinecommand GetSlavesRenderingJob` for the test job
 
 set -euo pipefail
 
-JOB_NAME=$1
-ARG=$2
+WORKER_OS=$1
+JOB_NAME=$2
+ARG=$3
 DEADLINE="/opt/Thinkbox/Deadline10/bin"
 
+case ${WORKER_OS^^} in
+  LINUX)
+    JOB_EXECUTABLE="/usr/bin/sleep"
+    ;;
+  WINDOWS)
+    JOB_EXECUTABLE="timeout.exe"
+    ;;
+  *)
+    echo "Error: unknown WORKER_OS from command line arguments: ${WORKER_OS}"
+    exit 1
+    ;;
+esac
+
 # Send a sleep command to the render queue; based on the arg passed in, this job will be assigned to the test group or pool
-JOB_ID=$($DEADLINE/deadlinecommand --prettyjson SubmitCommandLineJob -executable "/usr/bin/sleep" -arguments "10" -frames "1-10" -name $JOB_NAME $ARG)
+JOB_ID=$($DEADLINE/deadlinecommand --prettyjson SubmitCommandLineJob -executable "$JOB_EXECUTABLE" -arguments "10" -frames "1-10" -name $JOB_NAME $ARG)
 # We then pull the jobId from the output of `SubmitCommandLineJob`
 JOB_ID=$(jq -r '.result' <<< "$JOB_ID")
 

integ/components/deadline/deadline_04_workerFleetHttps/test/deadline_04_workerFleetHttps.test.ts

@@ -19,8 +19,8 @@ const rqRegex = /renderQueueEndpointWFS(\d)/;
 const certRegex = /CertSecretARNWFS(\d)/;
 
 const testCases: Array<Array<any>> = [
-  [ 'Linux Worker HTTPS (TLS) mode', 1 ],
-  [ 'Windows Worker HTTPS (TLS) mode', 2 ],
+  [ 'Linux Worker HTTPS (TLS) mode', 1, 'Linux' ],
+  [ 'Windows Worker HTTPS (TLS) mode', 2, 'Windows' ],
 ];
 let bastionId: any;
 let renderQueueEndpoints: Array<string> = [];
@@ -54,7 +54,7 @@ beforeAll( async () => {
   });
 });
 
-describe.each(testCases)('Deadline WorkerFleetHttps tests (%s)', (_, id) => {
+describe.each(testCases)('Deadline WorkerFleetHttps tests (%s)', (_, id, workerOS) => {
 
   beforeAll( async () => {
     if(secretARNs[id]) {
@@ -191,7 +191,7 @@ describe.each(testCases)('Deadline WorkerFleetHttps tests (%s)', (_, id) => {
             'sudo -i',
             'su - ec2-user >/dev/null',
             'cd ~ec2-user',
-            `./testScripts/WFS-submit-jobs-to-sets.sh "${name}" "${arg}"`,
+            `./testScripts/WFS-submit-jobs-to-sets.sh "${workerOS}" "${name}" "${arg}"`,
           ],
         },
       };

integ/lib/testing-tier.ts

@@ -7,7 +7,9 @@ import * as path from 'path';
 import { CfnOutput, Duration, Stack, StackProps } from 'aws-cdk-lib';
 import {
   BastionHostLinux,
+  IMachineImage,
   InstanceType,
+  MachineImage,
   Port,
   Vpc,
 } from 'aws-cdk-lib/aws-ec2';
@@ -16,7 +18,7 @@ import {
   SessionManagerHelper,
   X509CertificatePem,
 } from 'aws-rfdk';
-import { RenderQueue } from 'aws-rfdk/deadline';
+import { RenderQueue, Version } from 'aws-rfdk/deadline';
 import { Construct } from 'constructs';
 import { NetworkTier } from '../components/_infrastructure/lib/network-tier';
 import { IRenderFarmDb } from './storage-struct';
@@ -39,6 +41,12 @@ export interface TestingTierProps extends StackProps {
    * The unique suffix given to all stacks in the testing app
    */
   readonly integStackTag: string;
+
+  /**
+   * The machine image to use for the Bastion instance.
+   * Defaults to an image that's suitable for running Deadline.
+   */
+  readonly bastionMachineImageOverride?: IMachineImage;
 }
 
 /**
@@ -79,6 +87,7 @@ export abstract class TestingTier extends Stack {
       vpc: this.vpc,
       subnetSelection: { subnetGroupName: NetworkTier.subnetConfig.testRunner.name },
       instanceType: new InstanceType('t3.small'),
+      machineImage: props.bastionMachineImageOverride ?? this.getMachineImageForDeadlineVersion(this.deadlineVersion),
     });
     if (process.env.DEV_MODE?.toLowerCase() === 'true') {
       SessionManagerHelper.grantPermissionsTo(this.testInstance);
@@ -88,7 +97,6 @@ export abstract class TestingTier extends Stack {
     new CfnOutput(this, 'bastionId', {
       value: this.testInstance.instanceId,
     });
-
   }
 
   /**
@@ -179,7 +187,6 @@ export abstract class TestingTier extends Stack {
       'cd ~ec2-user',
       `cp ${installerPath} ./deadline-client-installer.run`,
       'chmod +x *.run',
-      'sudo yum install -y lsb',
       'sudo ./deadline-client-installer.run --mode unattended',
       `rm -f ${installerPath}`,
       'rm -f ./deadline-client-installer.run',
@@ -274,4 +281,17 @@ export abstract class TestingTier extends Stack {
     this.testInstance.instance.userData.addCommands( ...userDataCommands );
     this.testInstance.instance.userData.addSignalOnExitCommand( this.testInstance.instance );
   }
+
+  /**
+   * Return an Amazon Linux machine image that can run the specified version of Deadline.
+   */
+  private getMachineImageForDeadlineVersion(deadlineVersion: string): IMachineImage {
+    const REMOVED_SUPPORT_FOR_AMAZON_LINUX_2 = new Version([10, 4, 0, 0]);
+
+    if (Version.parse(deadlineVersion).isLessThan(REMOVED_SUPPORT_FOR_AMAZON_LINUX_2)) {
+      return MachineImage.latestAmazonLinux2();
+    } else {
+      return MachineImage.latestAmazonLinux2023();
+    }
+  }
 }

integ/test-config.sh

@@ -54,6 +54,7 @@ export LINUX_DEADLINE_AMI_ID
 export WINDOWS_DEADLINE_AMI_ID
 
 # Configure test suites to include in end-to-end test
+#   - To skip a test, set it to true
 export SKIP_deadline_01_repository_TEST
 export SKIP_deadline_02_renderQueue_TEST
 export SKIP_deadline_03_workerFleetHttp_TEST

packages/aws-rfdk/lib/core/lib/deployment-instance.ts

@@ -172,7 +172,7 @@ export class DeploymentInstance extends Construct implements IScriptHost, IConne
     this.asg = new AutoScalingGroup(this, 'ASG', {
       instanceType: props.instanceType ?? InstanceType.of(InstanceClass.T3, InstanceSize.SMALL),
       keyName: props.keyName,
-      machineImage: props.machineImage ?? MachineImage.latestAmazonLinux2(),
+      machineImage: props.machineImage ?? MachineImage.latestAmazonLinux2023(),
       minCapacity: 1,
       maxCapacity: 1,
       securityGroup: props.securityGroup,

packages/aws-rfdk/lib/core/lib/mongodb-instance.ts

@@ -585,6 +585,8 @@ export class MongoDbInstance extends Construct implements IMongoDb, IGrantable {
       'sudo chmod 750 /etc/mongod_certs/', // Directory needs to be executable.
       // mongod user id might, potentially change on reboot. Make sure we own all mongo data
       `sudo chown mongod.mongod -R ${MongoDbInstance.MONGO_DEVICE_MOUNT_POINT}`,
+      // We need yaml for some of our MongoDB configuration scripts
+      'sudo yum install -y python3-PyYAML',
       // Configure mongod
       'bash ./setMongoLimits.sh',
       `bash ./setStoragePath.sh "${MongoDbInstance.MONGO_DEVICE_MOUNT_POINT}"`,

packages/aws-rfdk/lib/core/scripts/bash/configureCloudWatchAgent.sh

@@ -31,17 +31,25 @@ function install_agent {
   TMPDIR=$(mktemp -d)
   pushd $TMPDIR 2>&1 > /dev/null
 
+  # Set the gpg home directory, where it stores configuration files.
+  # By default, this is in the user's home directory. This might not exist, causing errors.
+  # We don't need these files after installation, so we use a temporary directory.
+  export GNUPGHOME=${TMPDIR}
+
   # Download CloudWatch agent installer
   aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm amazon-cloudwatch-agent.rpm
 
   if [ "$SKIP_VERIFICATION" = false ]
   then
     aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key assets/amazon-cloudwatch-agent.gpg amazon-cloudwatch-agent.gpg
-    GPG_IMPORT_OUT=$(gpg --no-default-keyring --keyring ./keyring.gpg --import amazon-cloudwatch-agent.gpg 2>&1)
+    GPG_IMPORT_OUT=$(gpg --no-default-keyring --keyring ${TMPDIR}/keyring.gpg --import amazon-cloudwatch-agent.gpg 2>&1)
     GPG_KEY=$(echo "${GPG_IMPORT_OUT}" | grep -Eow 'key [0-9A-F]+' | awk '{print $2}')
-    GPG_FINGERPRINT_OUT=$(gpg --no-default-keyring --keyring ./keyring.gpg --fingerprint ${GPG_KEY} 2>&1)
-    GPG_FINGERPRINT=$(echo "${GPG_FINGERPRINT_OUT}" | tr -d '[:blank:]' | grep -Eo 'fingerprint=[0-9A-F]{40}')
-    if test "${GPG_FINGERPRINT}" != "fingerprint=937616F3450B7D806CBD9725D58167303B789C72"
+    GPG_FINGERPRINT_OUT=$(gpg --no-default-keyring --keyring ${TMPDIR}/keyring.gpg --fingerprint --with-colons ${GPG_KEY} 2>&1)
+    # The "--with-colons" above requests output in a machine-readable colon separated format.
+    # For a description of the format, see https://github.com/gpg/gnupg/blob/master/doc/DETAILS
+    # We're looking for the "fpr" or "Fingerprint" record, which has the fingerprint in field 10.
+    GPG_FINGERPRINT=$(echo "${GPG_FINGERPRINT_OUT}" | grep '^fpr:' | cut -d ':' -f10)
+    if test "${GPG_FINGERPRINT}" != "937616F3450B7D806CBD9725D58167303B789C72"
     then
         # Key failed to verify. Alert AWS!!
         echo "ERROR: Key failed to verify."
@@ -51,7 +59,7 @@ function install_agent {
     fi
 
     aws s3api get-object --region $region --bucket amazoncloudwatch-agent-$region --key amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm.sig
-    if ! gpg --no-default-keyring --keyring ./keyring.gpg --verify amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm 2>&1
+    if ! gpg --no-default-keyring --keyring ${TMPDIR}/keyring.gpg --verify amazon-cloudwatch-agent.rpm.sig amazon-cloudwatch-agent.rpm 2>&1
     then
         # CloudWatch agent installer failed to verify. Alert AWS!!
         echo "ERROR: Agent installer failed to verify"

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/secretsFunction.sh

@@ -6,8 +6,8 @@
 function get_secret_string() {
   SECRET_ID=$1
   AWS_REGION=$(echo ${SECRET_ID} | cut -d: -f4)
-  PYTHON_SCRIPT="import json,sys; d=json.load(sys.stdin); print d[\"SecretString\"];"
+  PYTHON_SCRIPT="import json,sys; d=json.load(sys.stdin); print(d[\"SecretString\"]);"
   set +x
-  export RET_VALUE=$(aws --region ${AWS_REGION} secretsmanager get-secret-value --secret-id "${SECRET_ID}" | python -c "${PYTHON_SCRIPT}" )
+  export RET_VALUE=$(aws --region ${AWS_REGION} secretsmanager get-secret-value --secret-id "${SECRET_ID}" | python3 -c "${PYTHON_SCRIPT}" )
   set -x
 }

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setLiveConfiguration.sh

@@ -8,7 +8,7 @@
 
 set -xeufo pipefail
 
-cat /etc/mongod.conf | python ./setupMongodLiveConfig.py > ./mongod.conf.new
+cat /etc/mongod.conf | python3 ./setupMongodLiveConfig.py > ./mongod.conf.new
 sudo mv ./mongod.conf.new /etc/mongod.conf
 # Make sure mongod user can read the config file
 sudo chmod 640 /etc/mongod.conf

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setMongoNoAuth.sh

@@ -3,7 +3,7 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-cat /etc/mongod.conf | python ./setupMongodNoAuth.py > ./mongod.conf.new
+cat /etc/mongod.conf | python3 ./setupMongodNoAuth.py > ./mongod.conf.new
 sudo mv ./mongod.conf.new /etc/mongod.conf
 # Make sure mongod user can read the config file
 sudo chmod 640 /etc/mongod.conf

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setStoragePath.sh

@@ -16,7 +16,7 @@ fi
 
 STORAGE_PATH=$1
 
-cat /etc/mongod.conf | python ./setupMongodStorage.py "${STORAGE_PATH}" > ./mongod.conf.new
+cat /etc/mongod.conf | python3 ./setupMongodStorage.py "${STORAGE_PATH}" > ./mongod.conf.new
 sudo mv ./mongod.conf.new /etc/mongod.conf
 # Make sure mongod user can read the config file
 sudo chmod 640 /etc/mongod.conf

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setupMongodLiveConfig.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/bin/env python3
 
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
@@ -98,7 +98,7 @@ def main():
   mongod_conf = yaml.load(sys.stdin)
   modify_security(mongod_conf)
   modify_net_options(mongod_conf)
-  print yaml.dump(mongod_conf, default_flow_style=False)
+  print(yaml.dump(mongod_conf, default_flow_style=False))
 
 if __name__ == '__main__':
   main()

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setupMongodNoAuth.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/bin/env python3
 
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
@@ -81,7 +81,7 @@ def main():
   mongod_conf = yaml.load(sys.stdin)
   modify_security(mongod_conf)
   modify_network(mongod_conf)
-  print yaml.dump(mongod_conf, default_flow_style=False)
+  print(yaml.dump(mongod_conf, default_flow_style=False))
 
 if __name__ == '__main__':
   main()

packages/aws-rfdk/lib/core/scripts/mongodb/3.6/setupMongodStorage.py

@@ -1,4 +1,4 @@
-#!/bin/env python
+#!/bin/env python3
 
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0