Add venv build to github actions (gwastro#3567)

* Add venv build to github actions

* Fix

* Trying to use updated congif files

* Try this

* Add dummy bank file

* One simple thing takes forever

* Try this

* Enable secure variables now

* Test actually pushing

* Debug

* REVERT THIS

* Try this?

* Revert "Try this?"

This reverts commit 344c3f2.

* Is this it??

* Revert "REVERT THIS"

This reverts commit 190b65c.

* Put back to only run on release

* Update docker_build_dist.sh

.github/workflows/build_venv.yml

@@ -0,0 +1,28 @@
+name: docker build
+
+on: [push]
+
+jobs:
+  build: 
+    runs-on: ubuntu-20.04
+    steps: 
+      - 
+        uses: actions/checkout@v1
+      - 
+        name: "Set up Python"
+        uses: actions/setup-python@v1
+        with: 
+          python-version: 2.7
+      - 
+        env:
+          OSG_ACCESS: "${{secrets.OSG_ACCESS}}"
+        name: "Preparing ssh stuff"
+        run: "bash -e tools/docker_build_prepssh.sh"
+      - 
+        env:
+          DOCKER_IMG: ligo/lalsuite-dev:el7
+          PYCBC_CONTAINER: pycbc_rhel_virtualenv
+          DOCKER_SECURE_ENV_VARS: true
+        name: "Creating the virtual environment"
+        run: "bash -e tools/docker_build_and_test.sh"
+

tools/docker_build_and_test.sh

@@ -4,11 +4,13 @@ set -e
 
 echo -e "\\n>> [`date`] Starting docker container ${DOCKER_IMG}"
 
-if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then
+if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then
   cp -R ~/.ssh .
 fi
 
-sudo docker run --name buildvm -v `pwd`:/pycbc:rw ${DOCKER_IMG} /bin/bash -c "bash /pycbc/tools/docker_build_dist.sh --pycbc-container=${PYCBC_CONTAINER} --pull-request=${TRAVIS_PULL_REQUEST} --commit=${TRAVIS_COMMIT} --secure=${TRAVIS_SECURE_ENV_VARS} --tag=${TRAVIS_TAG}"
+SOURCE_TAG=`git show-ref | grep ${GITHUB_SHA} | grep -E -o "refs/tags.{0,100}" | cut -c11-`
+
+sudo docker run --name buildvm -v `pwd`:/pycbc:rw ${DOCKER_IMG} /bin/bash -c "bash /pycbc/tools/docker_build_dist.sh --pycbc-container=${PYCBC_CONTAINER} --pycbc-code=${GITHUB_REF} --secure=${DOCKER_SECURE_ENV_VARS} --tag=${SOURCE_TAG}"
 
 echo -e "\\n>> [`date`] Docker exited"
 

tools/docker_build_dist.sh

@@ -5,39 +5,30 @@ set -e
 for i in $*; do
   case $i in
     --pycbc-container=*) PYCBC_CONTAINER="`echo $i|sed 's/^--pycbc-container=//'`";;
-    --pull-request=*) TRAVIS_PULL_REQUEST="`echo $i|sed 's/^--pull-request=//'`";;
-    --commit=*) TRAVIS_COMMIT="`echo $i|sed 's/^--commit=//'`";;
-    --secure=*) TRAVIS_SECURE_ENV_VARS="`echo $i|sed 's/^--secure=//'`";;
-    --tag=*) TRAVIS_TAG="`echo $i|sed 's/^--tag=//'`";;
+    --pycbc-code=*) PYCBC_CODE="`echo $i|sed 's/^--pycbc-code=//'`";;
+    --secure=*) DOCKER_SECURE_ENV_VARS="`echo $i|sed 's/^--secure=//'`";;
+    --tag=*) SOURCE_TAG="`echo $i|sed 's/^--tag=//'`";;
     *) echo -e "unknown option '$i', valid are:\n$usage">&2; exit 1;;
   esac
 done
 
-# determine the pycbc git branch and origin
-if test x$TRAVIS_PULL_REQUEST = "xfalse" ; then
-    PYCBC_CODE="--pycbc-commit=${TRAVIS_COMMIT}"
-else
-    PYCBC_CODE="--pycbc-fetch-ref=refs/pull/${TRAVIS_PULL_REQUEST}/merge"
-fi
 
 # set the lalsuite checkout to use
 
-if [ "x$TRAVIS_TAG" == "x" ] ; then
-  TRAVIS_TAG="master"
+if [ "x$SOURCE_TAG" == "x" ] ; then
+  SOURCE_TAG="master"
   RSYNC_OPTIONS="--delete"
 else
   RSYNC_OPTIONS=""
 fi
 
 echo -e "\\n>> [`date`] Inside container ${PYCBC_CONTAINER}"
-echo -e "\\n>> [`date`] Release tag is ${TRAVIS_TAG}"
+echo -e "\\n>> [`date`] Release tag is ${SOURCE_TAG}"
 echo -e "\\n>> [`date`] Using PyCBC code ${PYCBC_CODE}"
-echo -e "\\n>> [`date`] Travis pull request is ${TRAVIS_PULL_REQUEST}"
-echo -e "\\n>> [`date`] Travis commit is ${TRAVIS_COMMIT}"
-echo -e "\\n>> [`date`] Travis secure env is ${TRAVIS_SECURE_ENV_VARS}"
-echo -e "\\n>> [`date`] Travis tag is ${TRAVIS_TAG}"
+echo -e "\\n>> [`date`] Travis secure env is ${DOCKER_SECURE_ENV_VARS}"
+echo -e "\\n>> [`date`] Travis tag is ${SOURCE_TAG}"
 
-if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then
+if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then
   mkdir -p ~/.ssh
   cp /pycbc/.ssh/* ~/.ssh
   chmod 600 ~/.ssh/id_rsa
@@ -61,10 +52,10 @@ if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then
   CVMFS_PATH=/cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/${ENV_OS}/virtualenv
   mkdir -p ${CVMFS_PATH}
 
-  VENV_PATH=${CVMFS_PATH}/pycbc-${TRAVIS_TAG}
+  VENV_PATH=${CVMFS_PATH}/pycbc-${SOURCE_TAG}
   virtualenv ${VENV_PATH}
   echo 'export PYTHONUSERBASE=${VIRTUAL_ENV}/.local' >> ${VENV_PATH}/bin/activate
-  echo "export XDG_CACHE_HOME=\${HOME}/cvmfs-pycbc-${TRAVIS_TAG}/.cache" >> ${VENV_PATH}/bin/activate
+  echo "export XDG_CACHE_HOME=\${HOME}/cvmfs-pycbc-${SOURCE_TAG}/.cache" >> ${VENV_PATH}/bin/activate
   source ${VENV_PATH}/bin/activate
   mkdir -p ${VIRTUAL_ENV}/.local
   echo -e "[easy_install]\\nzip_ok = false\\n" > ~/.pydistutils.cfg
@@ -111,24 +102,21 @@ EOF
   echo -e "\\n>> [`date`] Running test_coinc_search_workflow.sh"
   mkdir -p /pycbc/workflow-test
   pushd /pycbc/workflow-test
-  /pycbc/tools/test_coinc_search_workflow.sh ${VENV_PATH} ${TRAVIS_TAG}
+  /pycbc/tools/test_coinc_search_workflow.sh ${VENV_PATH} ${SOURCE_TAG}
   popd
 
-  if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then
+  if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then
     echo -e "\\n>> [`date`] Setting virtual environment permissions for deployment"
     find ${VENV_PATH} -type d -exec chmod go+rx {} \;
     chmod -R go+r ${VENV_PATH}
 
     echo -e "\\n>> [`date`] Deploying virtual environment ${VENV_PATH}"
-    echo -e "\\n>> [`date`] Deploying virtual environment to sugwg-condor.phy.syr.edu"
-    ssh [email protected] "mkdir -p /home/pycbc/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}"
-    rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ [email protected]:/home/pycbc/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}/
-    if [ "x${TRAVIS_TAG}" != "xmaster" ] ; then
-      echo -e "\\n>> [`date`] Deploying release ${TRAVIS_TAG} to CVMFS"
+    if [ "x${SOURCE_TAG}" != "xmaster" ] ; then
+      echo -e "\\n>> [`date`] Deploying release ${SOURCE_TAG} to CVMFS"
       # remove lalsuite source and deploy on cvmfs
       rm -rf ${VENV_PATH}/src/lalsuite
-      ssh [email protected] "mkdir -p /home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}"
-      rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ [email protected]:/home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}/
+      ssh [email protected] "mkdir -p /home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${SOURCE_TAG}"
+      rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ [email protected]:/home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${SOURCE_TAG}/
       ssh [email protected] osg-oasis-update
     fi
     echo -e "\\n>> [`date`] virtualenv deployment complete"

tools/docker_build_prepssh.sh

@@ -0,0 +1,9 @@
+mkdir -p ~/.ssh
+touch ~/.ssh/id_rsa ~/.ssh/ldg_user ~/.ssh/ldg_token
+chmod 600 ~/.ssh/id_rsa ~/.ssh/ldg_user ~/.ssh/ldg_token
+echo "${OSG_ACCESS}" > ~/.ssh/id_rsa
+echo -e "Host sugwg-test1.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ;
+echo -e "Host sugwg-condor.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ;
+echo -e "Host oasis-login.opensciencegrid.org\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ;
+echo -e "Host code.pycbc.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ;
+chmod 600 ~/.ssh/id_rsa ~/.ssh/config ~/.ssh/ldg_user ~/.ssh/ldg_token

tools/test_coinc_search_workflow.sh

@@ -30,17 +30,22 @@ export LIGO_DATAFIND_SERVER='128.230.190.43:80'
 mkdir $WORKFLOW_NAME
 pushd $WORKFLOW_NAME
 
+# Doesn't need to be a valid bank file, just needs to exist
+echo "DUMMY BANK FILE" > bank.hdf
+echo "DUMMY STAT FILE" > statHL.hdf
+echo "DUMMY STAT FILE" > statLV.hdf
+echo "DUMMY STAT FILE" > statHV.hdf
+echo "DUMMY STAT FILE" > statHLV.hdf
+
 echo -e "\\n>> [`date`] Building test workflow $WORKFLOWNAME"
 
 pycbc_make_coinc_search_workflow \
 --workflow-name ${WORKFLOW_NAME} --output-dir output \
 --config-files \
-${CONFIG_PATH}/analysis.ini \
-${CONFIG_PATH}/data_O1.ini \
-${CONFIG_PATH}/plotting.ini \
-${CONFIG_PATH}/injections_minimal.ini \
-${CONFIG_PATH}/executables.ini \
-${CONFIG_PATH}/gps_times_O1_analysis_1.ini \
+/pycbc/examples/search/analysis.ini \
+/pycbc/examples/search/plotting.ini \
+/pycbc/examples/search/injections_minimal.ini \
+/pycbc/examples/search/executables.ini \
 --config-overrides \
 "results_page:output-path:../../../html"