Merge pull request #143 from aws-samples/bionemo

Bionemo
aws-samples · Feb 24, 2024 · d35b7c9 · d35b7c9
2 parents 235c3c0 + 99e4438
commit d35b7c9
Show file tree

Hide file tree

Showing 6 changed files with 369 additions and 0 deletions.
diff --git a/3.test_cases/14.bionemo/0.Dockerfile b/3.test_cases/14.bionemo/0.Dockerfile
@@ -0,0 +1,102 @@
+FROM nvcr.io/nvidia/clara/bionemo-framework:latest
+
+ARG EFA_INSTALLER_VERSION=1.30.0
+ARG AWS_OFI_NCCL_VERSION=v1.7.4-aws
+ARG NCCL_TESTS_VERSION=master
+ARG NCCL_VERSION=v2.18.6-1
+RUN apt-get update -y
+RUN apt-get remove -y --allow-change-held-packages \
+    libmlx5-1 ibverbs-utils libibverbs-dev libibverbs1 libnccl2 libnccl-dev
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-unauthenticated \
+    git \
+    gcc \
+    vim \
+    kmod \
+    openssh-client \
+    openssh-server \
+    build-essential \
+    curl \
+    autoconf \
+    libtool \
+    gdb \
+    automake \
+    python3-distutils \
+    cmake \
+    apt-utils \
+    devscripts \
+    debhelper \
+    libsubunit-dev \
+    check \
+    pkg-config
+
+RUN mkdir -p /var/run/sshd
+RUN sed -i 's/[ #]\(.*StrictHostKeyChecking \).*/ \1no/g' /etc/ssh/ssh_config && \
+    echo "    UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config && \
+    sed -i 's/#\(StrictModes \).*/\1no/g' /etc/ssh/sshd_config
+ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:/opt/amazon/openmpi/lib:/opt/nccl/build/lib:/opt/amazon/efa/lib:/opt/aws-ofi-nccl/install/lib:/usr/local/lib:$LD_LIBRARY_PATH
+ENV PATH /opt/amazon/openmpi/bin/:/opt/amazon/efa/bin:/usr/bin:/usr/local/bin:$PATH
+RUN curl https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
+ && python3 /tmp/get-pip.py \
+    && pip3 install awscli pynvml
+
+#################################################
+# Install NVIDIA GDRCopy
+RUN git clone https://github.com/NVIDIA/gdrcopy.git /opt/gdrcopy \
+    && cd /opt/gdrcopy \
+    && make lib_install install \
+    && cd /opt/gdrcopy/tests \
+    && make \
+    && make install \
+    && mv gdrcopy_copylat gdrcopy_copybw gdrcopy_sanity gdrcopy_apiperf /usr/bin/
+
+#################################################
+## Install EFA installer
+RUN cd $HOME \
+    && curl -O https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz \
+    && tar -xf $HOME/aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz \
+    && cd aws-efa-installer \
+    && ./efa_installer.sh -y -g -d --skip-kmod --skip-limit-conf --no-verify \
+    && rm -rf $HOME/aws-efa-installer
+
+###################################################
+## Install AWS-OFI-NCCL plugin
+RUN apt-get install libtool autoconf cmake nasm unzip pigz parallel nfs-common build-essential hwloc libhwloc-dev libjemalloc2 libnuma-dev numactl libjemalloc-dev preload htop iftop liblapack-dev libgfortran5 ipcalc wget curl devscripts debhelper check libsubunit-dev fakeroot pkg-config dkms -y
+RUN export OPAL_PREFIX="" \
+    && git clone https://github.com/aws/aws-ofi-nccl.git /opt/aws-ofi-nccl \
+    && cd /opt/aws-ofi-nccl \
+    && git checkout ${AWS_OFI_NCCL_VERSION} \
+    && ./autogen.sh \
+    && ./configure --prefix=/opt/aws-ofi-nccl \
+    --with-libfabric=/opt/amazon/efa \
+    --with-cuda=/usr/local/cuda \
+    --with-nccl=/opt/nccl/build \
+    --with-mpi=/opt/amazon/openmpi \
+    --enable-platform-aws
+    && make && make install
+
+###################################################
+## Install NCCL-tests
+RUN git clone https://github.com/NVIDIA/nccl-tests.git /opt/nccl-tests \
+    && cd /opt/nccl-tests \
+    && git checkout ${NCCL_TESTS_VERSION} \
+    && make MPI=1 \
+    MPI_HOME=/opt/amazon/openmpi/ \
+    CUDA_HOME=/usr/local/cuda \
+    NCCL_HOME=/opt/nccl/build \
+    NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_86,code=sm_86 -gencode=arch=compute_90,code=sm_90"
+
+
+
+RUN rm -rf /var/lib/apt/lists/*
+ENV LD_PRELOAD /opt/nccl/build/lib/libnccl.so
+
+
+##############################################
+## BioNemo dependencies
+COPY requirements.txt /workspace/
+RUN pip3 install -r /workspace/requirements.txt
+
+COPY prepare_uniref50.py /workspace/bionemo
+
+WORKDIR /workspace/bionemo/
diff --git a/3.test_cases/14.bionemo/1.uniref50.slurm b/3.test_cases/14.bionemo/1.uniref50.slurm
@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --nodes=1                    # number of nodes
+#SBATCH --ntasks-per-node=1          # n tasks per machine (one task per gpu) <required>
+#SBATCH --cpus-per-task=16
+#SBATCH --exclusive                   # exclusive node access
+#SBATCH --output slurm-uniref-%j.out
+
+
+###########################
+###### User Variables #####
+###########################
+
+# default variables for Enroot
+: "${IMAGE:=$(pwd)/${ENROOT_IMAGE}}"
+: "${DATA_PATH:=/fsx}"
+: "${FSX_MOUNT:=$DATA_PATH:$DATA_PATH}"
+
+declare -a ARGS=(
+    --container-image $IMAGE
+    --container-mount-home
+    --container-mounts $FSX_MOUNT
+)
+
+srun -l "${ARGS[@]}"  python3 /workspace/bionemo/prepare_uniref50.py
diff --git a/3.test_cases/14.bionemo/2.esm1nv_pretrain.slurm b/3.test_cases/14.bionemo/2.esm1nv_pretrain.slurm
@@ -0,0 +1,82 @@
+#!/bin/bash
+#SBATCH --nodes=4                    # number of nodes
+#SBATCH --ntasks-per-node=8          # n tasks per machine (one task per gpu) <required>
+#SBATCH --gpus-per-node=8
+#SBATCH --exclusive                   # exclusive node access
+#SBATCH --output slurm-esm1nv-train-%j.out
+
+export FI_EFA_USE_HUGE_PAGE=0
+
+
+###########################
+###### User Variables #####
+###########################
+
+# default variables for Enroot
+: "${IMAGE:=$(pwd)/${ENROOT_IMAGE}}"
+: "${DATA_PATH:=/fsx}"
+: "${FSX_MOUNT:=$DATA_PATH:$DATA_PATH}"
+
+declare -a ARGS=(
+    --container-image $IMAGE
+    --container-mount-home
+    --container-mounts $FSX_MOUNT
+)
+
+
+# Training parameters
+# =========================
+MICRO_BATCH_SIZE=256 # micro batch size per GPU, for best efficiency should be set to occupy ~85% of GPU memory. Suggested value for A100 80GB is 256
+ACCUMULATE_GRAD_BATCHES=1 # gradient accumulation
+TENSOR_MODEL_PARALLEL_SIZE=1 # tensor model parallel size
+VAL_CHECK_INTERVAL=500 # how often validation step is performed, including downstream task validation
+MAX_STEPS=1000000 # duration of training as the number of training steps
+# =========================
+
+
+# Logging
+# =========================
+PROJECT_NAME="esm1nv_pretraining" # project name, will be used for logging
+EXP_TAG="-small" # any additional experiment info, can be empty
+EXP_NAME="esm1nv_batch${MICRO_BATCH_SIZE}_gradacc${ACCUMULATE_GRAD_BATCHES}_nodes${SLURM_JOB_NUM_NODES}${EXP_TAG}"
+CREATE_WANDB_LOGGER=False # set to False if you don't want to log results with WandB
+WANDB_LOGGER_OFFLINE=False # set to True if there are issues uploading to WandB during training
+# =========================
+
+# Mounts
+# =========================
+DATA_PATH=/fsx/processed # Directory with data for model training and downstream task validation
+TRAIN_FILES='x_OP_000..049_CL_' # Range for the train dataset
+TEST_FILES='x_OP_000..049_CL_'  # Range for the test dataset
+VAL_FILES='x_OP_000..049_CL_'   # Range for the val dataset
+RESULTS_PATH=/fsx/esm1nv-train/${PROJECT_NAME}/${EXP_NAME}/results # directory to store logs, checkpoints and results
+
+mkdir -p ${RESULTS_PATH}}
+
+
+# Necessary Exports
+# =========================
+export HYDRA_FULL_ERROR=1
+# =========================
+
+srun -l "${ARGS[@]}"  python3 /workspace/bionemo/examples/protein/esm1nv/pretrain.py \
+    --config-path=/workspace/bionemo/examples/protein/esm1nv/conf \
+    --config-name=pretrain_small \
+    exp_manager.exp_dir=${RESULTS_PATH} \
+    exp_manager.create_wandb_logger=${CREATE_WANDB_LOGGER} \
+    exp_manager.wandb_logger_kwargs.name=${EXP_NAME} \
+    exp_manager.wandb_logger_kwargs.project=${PROJECT_NAME} \
+    ++exp_manager.wandb_logger_kwargs.offline=${WANDB_LOGGER_OFFLINE} \
+    trainer.num_nodes=${SLURM_JOB_NUM_NODES} \
+    trainer.devices=${SLURM_GPUS_PER_NODE} \
+    trainer.max_steps=${MAX_STEPS} \
+    trainer.accumulate_grad_batches=${ACCUMULATE_GRAD_BATCHES} \
+    trainer.val_check_interval=${VAL_CHECK_INTERVAL} \
+    model.micro_batch_size=${MICRO_BATCH_SIZE} \
+    model.tensor_model_parallel_size=${TENSOR_MODEL_PARALLEL_SIZE} \
+    model.data.dataset_path=${DATA_PATH} \
+    model.data.dataset.train=${TRAIN_FILES} \
+    model.data.dataset.val=${VAL_FILES} \
+    model.data.dataset.test=${TEST_FILES} \
+    model.data.index_mapping_dir=${DATA_PATH} \
+    ++model.dwnstr_task_validation.enabled=False
diff --git a/3.test_cases/14.bionemo/README.md b/3.test_cases/14.bionemo/README.md
@@ -0,0 +1,147 @@
+# Train Evolutionary Scale Models (ESM) with BioNemo
+
+NVIDIA BioNeMo is a domain-specific machine learning framework for training and using foundation models for biology. This includes models for analyzing proteins, small molecules, and other biological molecules. NVIDIA first announced it in [September 2022](https://nvidianews.nvidia.com/news/nvidia-launches-large-language-model-cloud-services-to-advance-ai-and-digital-biology) and released a more comprehensive version on DGX cloud at [GTC 2023](https://nvidianews.nvidia.com/news/nvidia-unveils-large-language-models-and-generative-ai-services-to-advance-life-sciences-r-d). The GTC 2023 release included two main capabilities:
+1. A NeMo-based training framework to enable ML teams to create training and inference jobs via Python scripts. submitted via DGX-hosted notebooks
+2. A web application that enabled scientists to create inference jobs and visualize output data.
+
+|Num|                                    BioNeMo Model Support                                     |
+|:-:|:--------------------------------------------------------------------------------------------:|
+| 1 |      [ESM-1nv](https://docs.nvidia.com/bionemo-framework/latest/models/esm1-nv.html)         |
+| 2 |      [ESM-2nv](https://docs.nvidia.com/bionemo-framework/latest/models/esm2-nv.html)         |
+| 3 |      [MegaMolBART](https://docs.nvidia.com/bionemo-framework/latest/models/megamolbart.html) |
+| 4 |      [DiffDock](https://docs.nvidia.com/bionemo-framework/latest/models/diffdock.html)       |
+| 5 |      [EquiDock](https://docs.nvidia.com/bionemo-framework/latest/models/equidock.html)       |
+| 6 |      [ProtT5nv](https://docs.nvidia.com/bionemo-framework/latest/models/prott5nv.html)       |
+
+
+This project provides a guide to run [Nvidia's BioNemo](https://docs.nvidia.com/bionemo-framework/latest/index.html) on AWS ParallelCluster and pretrain the popular [ESM models](https://github.com/facebookresearch/esm) specifically the [ESM1nv](https://docs.nvidia.com/bionemo-framework/latest/notebooks/model_training_esm1nv.html) model.
+
+
+## 0. Prerequisites
+
+0. You have access to the bionemo container. To get the access to BioNeMo, visit the [information website](https://www.nvidia.com/en-us/clara/bionemo/).
+
+1. Have a slurm based AWS ParallelCluster created with a FSx for Lustre filesystem mounted. Below we are presenting instructions for a cluster with compute nodes instantiated with an Ubuntu based AMI.
+
+## 1. Install Nvidia Container CLI
+
+### 1.1 If you have created your cluster with the AWS ParallelCluster Base AMI or [DLAMI](https://aws.amazon.com/machine-learning/amis/) or your custom AMI, please make sure `libnvidia-container cli` is installed. You can follow the instructions below to install it.   
+
+### 1.2 To install libnvidia-container cli:
+We need [libnvidia-container cli](https://github.com/NVIDIA/libnvidia-container) to train models in an Nvidia container. We follow the instructions [here](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). This installation needs to be done in each compute node.
+
+```
+curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
+  && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
+    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list \
+  && \
+    sudo apt-get update \
+  && sudo apt-get install libnvidia-container1 \
+  && sudo apt-get install libnvidia-container-tools
+```
+### 1.3 You can set the Nemo Multimodal version and others as environment variables:
+
+SSH into the head node of your cluster and run:
+
+```
+export PYTHON_VERSION=3.10
+# We are using Python version 3.10 in this work. For a different Python version select the right Miniconda file from https://repo.anaconda.com/miniconda/
+export MINICONDA_INSTALLER=Miniconda3-py310_23.5.2-0-Linux-x86_64
+export TARGET_PATH=/apps/bionemo-src   # Must be a shared filesystem. This is where Nemo launcher scripts will reside.
+export DOCKER_IMAGE_NAME=bionemo
+export TAG=latest
+export ENROOT_IMAGE=/apps/${DOCKER_IMAGE_NAME}
+export DATASET_PATH=/fsx/
+```
+
+## 1.4. Pull this github repo
+
+```bash
+cd /apps/
+git clone https://github.com/aws-samples/awsome-distributed-training.git
+cp -r /apps/awsome-distributed-training/3.test_cases/14.bionemo/* ./apps/
+```
+
+## 2. Pull Image
+
+```bash
+cd /apps/
+docker pull nvcr.io/nvidia/clara/bionemo-framework:latest
+```
+
+## 3. Create Conda env
+We need a conda environment that has the necessary dependencies for submitting multiple arrays of slurm jobs via [HYDRA](https://github.com/facebookresearch/hydra) which NeMo uses to configuring both NeMo models and the PyTorch Lightning Trainer. 
+```
+# Miniconda is already installed if you are using the DLAMI but needs installation with Base AMI
+
+wget -O miniconda.sh "https://repo.anaconda.com/miniconda/${MINICONDA_INSTALLER}.sh" \
+    && bash miniconda.sh -b -p /apps/.conda \
+          &&  /apps/.conda/bin/conda init bash  
+
+source ~/.bashrc    
+conda create --name bionemo python=${PYTHON_VERSION}
+
+source activate bionemo
+
+pip3 install -r requirements.txt
+
+```
+All package versions in the above `requirements.txt` file is recommended from Nvidia. An older version of the package `opencv-python-headless==4.8.0.74` has to be installed to avoid this [error](https://github.com/rom1504/img2dataset/issues/355) with [img2dataset](https://github.com/rom1504/img2dataset) package.
+
+
+
+## 4. Build customized docker image
+To achieve target performance of Nemo-Multimodal with EFA on P5 and P4de instances, we provide a customized 
+`3.test_cases/14.nemo-multimodal/0.Dockerfile` and we can build a image like below:
+
+```
+docker build -t ${DOCKER_IMAGE_NAME}:${TAG} -f 0.Dockerfile .
+```
+
+## 5. Convert image
+Convert the Docker container image to an [Enroot](https://github.com/NVIDIA/enroot) squash file that will be stored in `/apps`. This step takes a few minutes.
+```
+enroot import -o ${ENROOT_IMAGE}.sqsh dockerd://${DOCKER_IMAGE_NAME}
+
+```
+
+## 6. Download and preprocess data
+We will use the popular [UniRef50](https://www.uniprot.org/help/uniref) dataset for pretraining. We will use BioNemo's in-built functionality to download and pre-process data. To this end, we provide `prepare_uniref50.py` file to do so. You can edit the above to download and process [UniRef90]((https://www.uniprot.org/help/uniref)). To run the above python code on your slurm cluster in the BioNemo cluster execute the following:
+
+```bash
+sbatch 1.uniref50.slurm
+```
+
+This will download raw data in `/fsx/raw/` and save pre-processed `train, validation and test` csv files in `/fsx/processed/`. The log files for submitted jobs are written to the local directory. To check the status of the datasets download job, you can tail the log file:
+
+```bash
+tail -f slurm-uniref-<slurm_job_id>.out
+```
+
+
+
+## 7. Pretrain ESM models
+Now we are ready to submit distributed training jobs to pretrain `ESM1nv` models. We provide the `2.esm1nv_pretrain.slurm` script to run training 4 `p4de.24xlarge` nodes with `8xA100 80 GB` GPUs. Make sure data paths and model configuration is correct if you are running on custom data. To kick off distributed training execute:
+
+```bash
+sbatch 2.esm1nv_pretrain.slurm
+
+```
+
+Before kicking off training, first train, validation and test datasets are indexed and dataloaders are created and then you should see an example output like below:
+
+```bash
+Epoch 0:   3%|▎         | 34103/1100000 [5:28:58<171:22:21,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.510, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34106/1100000 [5:29:00<171:22:19,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34109/1100000 [5:29:02<171:22:09,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+Epoch 0:   3%|▎         | 34112/1100000 [5:29:03<171:22:00,  1.73it/s, loss=2.52, v_num=, reduced_train_loss=2.520, global_step=3.1e+4, consumed_samples=2.54e+8, val_loss=2.510]
+```
+
+## 8. Run container on Head Node [Troubleshooting]
+Once the above image is pulled, you can run the container on the head node like below. This step could be used for troubleshooting purposes. Here we are running the container just to be able to copy launcher scripts on the host machine. If you need to run the container on the compute nodes, you would need to add `--gpus all` flag to the run command. It is recommended to have the docker run flags like below, as recommended by Nvidia PyTorch containers, otherwise you may potentially run into an error like [this](https://github.com/NVIDIA/Megatron-LM/issues/516)
+
+```
+ docker run -it nvcr.io/nvidia/clara/bionemo-framework:latest bash
+```
+
diff --git a/3.test_cases/14.bionemo/prepare_uniref50.py b/3.test_cases/14.bionemo/prepare_uniref50.py
@@ -0,0 +1,3 @@
+from bionemo.data import UniRef50Preprocess
+data = UniRef50Preprocess(root_directory='/fsx')
+data.prepare_dataset(source='uniprot')
diff --git a/3.test_cases/14.bionemo/requirements.txt b/3.test_cases/14.bionemo/requirements.txt
@@ -0,0 +1,11 @@
+dask==2024.2.0
+huggingface_hub>=0.13.0,<0.14.0
+hydra-core>=1.2.0,<1.3
+img2dataset
+omegaconf>=2.2,<2.3
+pynvml==11.4.1
+requests==2.31.0
+tqdm==4.62.3
+zstandard==0.15.2
+opencv-python-headless==4.8.0.74
+numba==0.59.0