Skip to content

Commit

Permalink
Fix Lambda python version for previous versions of ParallelCluster.
Browse files Browse the repository at this point in the history 
PC 3.11.1 updated the Lambda layer to use Python 3.12.
Previous version required Python 3.9.

Set the Lambda Python runtime based on the ParallelCluster version so that
the application continues to work on older versions of PC.

This bug was introduced in #270.

Related to #270

Resolves #280
  • Loading branch information
@cartalla
cartalla committed Nov 4, 2024
1 parent b9eb695 commit db2f38d
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 12 deletions.
24 changes: 12 additions & 12 deletions source/cdk/cdk_slurm_stack.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
import boto3
from botocore.exceptions import ClientError
import config_schema
from config_schema import get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION
from config_schema import get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION, get_PARALLEL_CLUSTER_MUNGE_VERSION, get_PARALLEL_CLUSTER_PYTHON_VERSION, get_PC_SLURM_VERSION, get_SLURM_VERSION
from constructs import Construct
from copy import copy, deepcopy
from hashlib import sha512
Expand Down Expand Up @@ -1427,7 +1427,7 @@ def create_parallel_cluster_lambdas(self):
aws_lambda.Architecture.X86_64,
],
compatible_runtimes = [
aws_lambda.Runtime.PYTHON_3_12,
get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version']))
],
)

Expand All @@ -1437,7 +1437,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-CreateBuildFiles",
description="Create ParallelCluster build configuration files",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(2),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1499,7 +1499,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-CreateParallelClusterConfig",
description="Create ParallelCluster config",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1547,7 +1547,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-CreateParallelCluster",
description="Create ParallelCluster",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1846,7 +1846,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-CreateHeadNodeARecord",
description="Create head node A record",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1893,7 +1893,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-UpdateHeadNode",
description="Update head node",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1935,7 +1935,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-ConfigUsersGroupsJson",
description="Configure users and groups json file",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -1983,7 +1983,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-ConfigExternalLoginNodes",
description="Configure external login nodes",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -2030,7 +2030,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-DeconfigUsersGroupsJson",
description="Deconfigure RES users and groups json file",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -2072,7 +2072,7 @@ def create_parallel_cluster_lambdas(self):
function_name=f"{self.stack_name}-DeconfigExternalLoginNodes",
description="Deconfigure external login nodes",
memory_size=2048,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.X86_64,
timeout=Duration.minutes(15),
log_retention=logs.RetentionDays.INFINITE,
Expand Down Expand Up @@ -2114,7 +2114,7 @@ def create_callSlurmRestApiLambda(self):
function_name=f"{self.stack_name}-CallSlurmRestApiLambda",
description="Example showing how to call Slurm REST API",
memory_size=128,
runtime=aws_lambda.Runtime.PYTHON_3_12,
runtime=get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parse_version(self.config['slurm']['ParallelClusterConfig']['Version'])),
architecture=aws_lambda.Architecture.ARM_64,
timeout=Duration.minutes(1),
log_retention=logs.RetentionDays.INFINITE,
Expand Down
11 changes: 11 additions & 0 deletions source/cdk/config_schema.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""

from aws_cdk import (
aws_lambda as aws_lambda
)
import boto3
from botocore.client import ClientError
import json
Expand Down Expand Up @@ -318,6 +321,14 @@ def PARALLEL_CLUSTER_SUPPORTS_CUSTOM_MUNGE_KEY(parallel_cluster_version):
def PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT(parallel_cluster_version):
return parallel_cluster_version >= PARALLEL_CLUSTER_SUPPORTS_HOME_MOUNT_VERSION

# Version 3.11.0

def get_PARALLEL_CLUSTER_LAMBDA_LAYER_PYTHON_VERSION(parallel_cluster_version):
if parallel_cluster_version < parse_version('3.11.1'):
return aws_lambda.Runtime.PYTHON_3_9
else:
return aws_lambda.Runtime.PYTHON_3_12

# Determine all AWS regions available on the account.
default_region = environ.get("AWS_DEFAULT_REGION", "us-east-1")
ec2_client = boto3.client("ec2", region_name=default_region)
Expand Down

0 comments on commit db2f38d

Please sign in to comment.