Merge pull request #159 from ENCODE-DCC/dev

v2.1.3

.circleci/config.yml

@@ -9,41 +9,42 @@ defaults:
 
 machine_defaults: &machine_defaults
   machine:
-    image: ubuntu-1604:202007-01
+    image: ubuntu-2004:202010-01
   working_directory: ~/caper
 
 
 update_apt: &update_apt
   name: Update apt
   command: |
-    sudo apt-get update
+    sudo apt-get update -y
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
 
 
 install_python3: &install_python3
-  name: Install python3, pip3
+  name: Install python3, pip3, java
   command: |
-    sudo apt-get install software-properties-common git wget curl -y
-    sudo add-apt-repository ppa:deadsnakes/ppa -y
-    sudo apt-get update && sudo apt-get install python3.6 -y
-    sudo wget --no-check-certificate https://bootstrap.pypa.io/get-pip.py
-    sudo python3.6 get-pip.py
-    sudo ln -s /usr/bin/python3.6 /usr/local/bin/python3
+    sudo apt-get install -y software-properties-common git wget curl python3 python3-pip default-jre
 
 
 install_singularity: &install_singularity
   name: Install Singularity (container)
   command: |
-    sudo wget -O- http://neuro.debian.net/lists/xenial.us-ca.full | sudo tee /etc/apt/sources.list.d/neurodebian.sources.list
-    # key server is so unstable. commented out for possible later use.
-    #sudo apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9
-    sudo apt-get update && sudo apt-get install singularity-container --allow-unauthenticated
+    sudo apt-get install -y alien
+    sudo wget https://kojipkgs.fedoraproject.org//packages/singularity/3.8.5/2.el8/x86_64/singularity-3.8.5-2.el8.x86_64.rpm
+    sudo alien -d singularity-3.8.5-2.el8.x86_64.rpm
+    sudo apt-get install -y ./singularity_3.8.5-3_amd64.deb
+    sudo apt-get install -y squashfs-tools
+    singularity --version
 
 
 install_py3_packages: &install_py3_packages
   name: Install Python packages
   command: |
-    sudo pip3 install pyOpenSSL pytest requests dateparser filelock autouri miniwdl pyhocon numpy pandas scikit-learn matplotlib "six>=1.13.0" "PyYAML==3.11"
-    sudo pip3 install --upgrade pyasn1-modules
+    sudo python3 -m pip install --upgrade pip
+    sudo pip3 install PyYAML --ignore-installed
+    sudo pip3 install pyOpenSSL pytest requests dateparser filelock autouri miniwdl pyhocon numpy pandas scikit-learn matplotlib six
+    #sudo pip3 install pyOpenSSL pytest requests dateparser filelock autouri miniwdl pyhocon numpy pandas scikit-learn matplotlib "six>=1.13.0" "PyYAML==3.11"
+    #sudo pip3 install --upgrade pyasn1-modules
 
 
 install_gcs_lib: &install_gcs_lib
@@ -61,14 +62,6 @@ install_aws_lib: &install_aws_lib
     sudo pip3 install boto3 awscli
 
 
-install_java: &install_java
-  name: Install openjdk-11
-  command: |
-    sudo add-apt-repository ppa:openjdk-r/ppa -y
-    sudo apt-get update && sudo apt-get install openjdk-11-jdk -y
-    # automatically set 11 as default java
-    sudo update-java-alternatives -a
-
 jobs:
   pytest:
     <<: *machine_defaults
@@ -80,7 +73,6 @@ jobs:
       - run: *install_py3_packages
       - run: *install_gcs_lib
       - run: *install_aws_lib
-      - run: *install_java
       - run:
           no_output_timeout: 60m
           command: |

README.md

@@ -24,6 +24,7 @@ See [this](scripts/aws_caper_server/README.md) for details.
 1) Make sure that you have Java (>= 11) and Python>=3.6 installed on your system and `pip` to install Caper.
 
 	```bash
+        $ pip install pip --upgrade
 	$ pip install caper
 	```
 

caper/__init__.py

@@ -2,4 +2,4 @@
 from .caper_runner import CaperRunner
 
 __all__ = ['CaperClient', 'CaperClientSubmit', 'CaperRunner']
-__version__ = '2.1.2'
+__version__ = '2.1.3'

caper/caper_init.py

@@ -216,7 +216,6 @@
 # Parallel environement is required, ask your administrator to create one
 # If your cluster doesn't support PE then edit 'sge-resource-param'
 # to fit your cluster's configuration.
-sge-pe=
 """
     + CONF_CONTENTS_SGE_PARAM
     + CONF_CONTENTS_LOCAL_HASH_STRAT

scripts/gcp_caper_server/create_instance.sh

@@ -27,8 +27,9 @@ if [[ $# -lt 1 ]]; then
   echo "  -z, --zone: Zone. Check available zones: gcloud compute zones list. us-central1-a by default."
   echo "  -m, --machine-type: Machine type. Check available machine-types: gcloud compute machine-types list. n1-standard-4 by default."
   echo "  -b, --boot-disk-size: Boot disk size. Use a suffix for unit. e.g. GB and MB. 100GB by default."
+  echo "  -u, --username: Username (super user) used for transferring key file to the instance. ubuntu by default."
   echo "  --boot-disk-type: Boot disk type. pd-standard (Standard persistent disk) by default."
-  echo "  --image: Image. Check available images: gcloud compute images list. ubuntu-1804-bionic-v20200716 by default."
+  echo "  --image: Image. Check available images: gcloud compute images list. ubuntu-2004-focal-v20220118 by default."
   echo "  --image-project: Image project. ubuntu-os-cloud by default."
   echo "  --tags: Tags to apply to the new instance. caper-server by default."
   echo "  --startup-script: Startup script CONTENTS (NOT A FILE). These command lines should sudo-install screen, Java, PostgreSQL, Python3 and pip3. DO NOT INSTALL CAPER HERE. some apt-get command lines by default."
@@ -95,6 +96,11 @@ while [[ $# -gt 0 ]]; do
       shift
       shift
       ;;
+    -u|--username)
+      USERNAME="$2"
+      shift
+      shift
+      ;;
     --boot-disk-type)
       BOOT_DISK_TYPE="$2"
       shift
@@ -174,11 +180,14 @@ fi
 if [[ -z "$BOOT_DISK_SIZE" ]]; then
   BOOT_DISK_SIZE=100GB
 fi
+if [[ -z "$USERNAME" ]]; then
+  USERNAME=ubuntu
+fi
 if [[ -z "$BOOT_DISK_TYPE" ]]; then
   BOOT_DISK_TYPE=pd-standard
 fi
 if [[ -z "$IMAGE" ]]; then
-  IMAGE=ubuntu-1804-bionic-v20200908
+  IMAGE=ubuntu-2004-focal-v20220118
 fi
 if [[ -z "$IMAGE_PROJECT" ]]; then
   IMAGE_PROJECT=ubuntu-os-cloud
@@ -189,7 +198,7 @@ fi
 if [[ -z "$STARTUP_SCRIPT" ]]; then
   STARTUP_SCRIPT="""
 sudo apt-get update
-sudo apt-get -y install screen python3 python3-pip default-jre postgresql postgresql-contrib
+sudo apt-get -y install screen python3 python3-pip default-jre postgresql postgresql-contrib acl
 """
 fi
 
@@ -224,16 +233,16 @@ REMOTE_KEY_FILE="$CAPER_CONF_DIR/service_account_key.json"
 
 # prepend more init commands to the startup-script
 STARTUP_SCRIPT="""#!/bin/bash
-### make caper's work directory
+### make caper's directories
 sudo mkdir -p $CAPER_CONF_DIR
-sudo chmod 777 -R $CAPER_CONF_DIR
-sudo setfacl -d -m u::rwX $CAPER_CONF_DIR
-sudo setfacl -d -m g::rwX $CAPER_CONF_DIR
-sudo setfacl -d -m o::rwX $CAPER_CONF_DIR
-
-### make caper's out/localization directory
 sudo mkdir -p $CAPER_CONF_DIR/local_loc_dir $CAPER_CONF_DIR/local_out_dir
 
+### set default permission on caper's directories
+sudo chmod 777 -R $CAPER_CONF_DIR
+sudo setfacl -R -d -m u::rwX $CAPER_CONF_DIR
+sudo setfacl -R -d -m g::rwX $CAPER_CONF_DIR
+sudo setfacl -R -d -m o::rwX $CAPER_CONF_DIR
+
 ### make caper conf file
 cat <<EOF > $GLOBAL_CAPER_CONF_FILE
 # caper
@@ -319,28 +328,32 @@ done
 
 echo "$(date): If key file transfer fails for several times then manually transfer it to $REMOTE_KEY_FILE on the instance."
 echo "$(date): Transferring service account key file to the instance..."
-until gcloud --project "$GCP_PRJ" compute scp "$GCP_SERVICE_ACCOUNT_KEY_JSON_FILE" root@"$INSTANCE_NAME":"$REMOTE_KEY_FILE" --zone="$ZONE"; do
+until gcloud --project "$GCP_PRJ" compute scp "$GCP_SERVICE_ACCOUNT_KEY_JSON_FILE" "$USERNAME"@"$INSTANCE_NAME":"$REMOTE_KEY_FILE" --zone="$ZONE"; do
   echo "$(date): Key file transfer failed. Retrying in 20 seconds..."
   sleep 20
 done
 echo "$(date): Transferred a key file to instance successfully."
 
 echo "$(date): Waiting for the instance finishing up installing Caper..."
-until gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" root@"$INSTANCE_NAME" --command="caper -v"; do
+until gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" "$USERNAME"@"$INSTANCE_NAME" --command="caper -v"; do
   echo "$(date): Caper has not been installed yet. Retrying in 40 seconds..."
   sleep 40
 done
 echo "$(date): Finished installing Caper on the instance. Ready to run Caper server."
 
 echo "$(date): Spinning up Caper server..."
-gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" root@"$INSTANCE_NAME" --command="cd $CAPER_CONF_DIR && screen -dmS caper_server bash -c \"caper server > caper_server.log 2>&1\""
+gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" "$USERNAME"@"$INSTANCE_NAME" --command="cd $CAPER_CONF_DIR && sudo screen -dmS caper_server bash -c \"sudo caper server > caper_server.log 2>&1\""
 sleep 60
-until gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" root@"$INSTANCE_NAME" --command="caper list"; do
+until gcloud --project "$GCP_PRJ" compute ssh --zone="$ZONE" "$USERNAME"@"$INSTANCE_NAME" --command="caper list"; do
   echo "$(date): Caper server has not been started yet. Retrying in 60 seconds..."
   sleep 60
 done
-echo "$(date): Caper server is up and ready to take submissions. You can find Caper server's log file at $CAPER_CONF_DIR/caper_server.log. Cromwell's STDERR will be written to cromwell.out."
-
+echo
+echo "$(date): Caper server is up and ready to take submissions."
+echo "$(date): You can find Caper server log file at $CAPER_CONF_DIR/caper_server.log."
+echo "$(date): Cromwell's STDERR will be written to $CAPER_CONF_DIR/cromwell.out*."
+echo
 echo "$(date): Use the following command line to SSH to the instance."
 echo
 echo "gcloud beta compute ssh --zone $ZONE $INSTANCE_NAME --project $GCP_PRJ"
+echo

setup.py

@@ -69,5 +69,6 @@ def find_meta(meta):
         'scikit-learn>=0.19.2',
         'matplotlib>=1.5',
         'six>=1.13.0',
+        'pyyaml<=5.5',
     ],
 )