reroganize commands

3.test_cases/3.MPT/Makefile

@@ -38,9 +38,7 @@ install-pcluster: # Install pcluster cli command using pip
 	python3 -m pip install --upgrade "aws-parallelcluster==3.6.0"
 
 deploy-s3: # Deploy S3 bucket using CloudFormation
-	aws cloudformation create-stack --stack-name s3-$(NAME) --template-body file://cloudformation/s3-bucket.yaml \
-		--parameters ParameterKey=S3BucketName,ParameterValue=$(S3BucketName) \
-		--region $(REGION) --capabilities=CAPABILITY_IAM
+	bash scripts/deploy-s3.sh
 
 deploy-vpc: # Deploy VPC using CloudFormation
 	aws cloudformation create-stack --stack-name vpc-$(NAME)  --template-body file://cloudformation/VPC-Large-Scale-singleAZ.yaml \

3.test_cases/3.MPT/config.env

@@ -0,0 +1,22 @@
+function set_options(){
+  if [ "${DRYRUN:-}" == "" ]; then  # Execute command when DRYRUN is unset
+    set -euxo pipefail
+  fi
+}
+# You can dry run each command like
+# DRYRUN=1 make run-container
+function run(){
+  if [ "${DRYRUN:-}" == "" ]; then  # Execute command when DRYRUN is unset
+     "$@"
+  else # Dry-run otherwise
+     {
+       set +x
+       echo -n "[DRYRUN]"
+       echo "$@"
+     } >&2
+  fi
+}
+NAME=llm-foundry
+ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)
+S3_BUCKET_NAME=${NAME}-${ACCOUNT_ID}-bucket
+REGION=ap-northeast-1

3.test_cases/3.MPT/scripts/build-image.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
-set -euxo pipefail
-
-docker build -t llm-foundry .
+. config.env
+set_options
+run docker build -t llm-foundry .

3.test_cases/3.MPT/scripts/deploy-s3.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+. config.env
+
+set_options
+run aws cloudformation create-stack --stack-name s3-${NAME} \
+    --template-body file://../../1.architectures/0.s3/0.private-bucket.yaml \
+    --parameters ParameterKey=S3BucketName,ParameterValue=${S3_BUCKET_NAME} \
+    --region ${REGION} --capabilities=CAPABILITY_IAM

3.test_cases/3.MPT/scripts/run-benchmarks.sh

@@ -1,18 +1,17 @@
 #!/usr/bin/env bash
-set -euxo pipefail
-
+. config.env
 # Prepare dataset
+set_options
 
 export PROFILE_FILE=/report/profile_file
-docker run \
+run docker run \
   --gpus all --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
   --rm \
   -i \
   -v $(pwd)/report:/report \
   --name llm-foundry \
   llm-foundry \
   /bin/bash -s <<EOF
-set -euxo pipefail
 cd /llm-foundry/scripts
 
 # Convert C4 dataset to StreamingDataset format
@@ -33,5 +32,6 @@ nsys profile -w true -t cuda,nvtx,osrt,cudnn,cublas \
   max_duration=3ba \
   eval_interval=0 \
   save_folder=mpt-7b \
+  device_train_microbatch_size=8 \
   global_train_batch_size=256
 EOF

3.test_cases/3.MPT/scripts/run-container.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
-set -euxo pipefail
+. config.env
 
-docker run \
+set_options
+
+run docker run \
   --gpus all --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 \
   --rm \
   --name llm-foundry \