From 12efe46e7b8c6a15e63883f6bd8b4f22d0bf9a80 Mon Sep 17 00:00:00 2001 From: Ian Harry Date: Thu, 17 Dec 2020 18:31:51 +0000 Subject: [PATCH] Add venv build to github actions (#3567) * Add venv build to github actions * Fix * Trying to use updated congif files * Try this * Add dummy bank file * One simple thing takes forever * Try this * Enable secure variables now * Test actually pushing * Debug * REVERT THIS * Try this? * Revert "Try this?" This reverts commit 344c3f2722f529ce7d039e8b48603c2e7a576361. * Is this it?? * Revert "REVERT THIS" This reverts commit 190b65c5d73318eeae02e1af121b17c87b9a7d77. * Put back to only run on release * Update docker_build_dist.sh --- .github/workflows/build_venv.yml | 28 ++++++++++++++++++ tools/docker_build_and_test.sh | 6 ++-- tools/docker_build_dist.sh | 46 +++++++++++------------------ tools/docker_build_prepssh.sh | 9 ++++++ tools/test_coinc_search_workflow.sh | 17 +++++++---- 5 files changed, 69 insertions(+), 37 deletions(-) create mode 100644 .github/workflows/build_venv.yml create mode 100644 tools/docker_build_prepssh.sh diff --git a/.github/workflows/build_venv.yml b/.github/workflows/build_venv.yml new file mode 100644 index 00000000000..579cec3bfa1 --- /dev/null +++ b/.github/workflows/build_venv.yml @@ -0,0 +1,28 @@ +name: docker build + +on: [push] + +jobs: + build: + runs-on: ubuntu-20.04 + steps: + - + uses: actions/checkout@v1 + - + name: "Set up Python" + uses: actions/setup-python@v1 + with: + python-version: 2.7 + - + env: + OSG_ACCESS: "${{secrets.OSG_ACCESS}}" + name: "Preparing ssh stuff" + run: "bash -e tools/docker_build_prepssh.sh" + - + env: + DOCKER_IMG: ligo/lalsuite-dev:el7 + PYCBC_CONTAINER: pycbc_rhel_virtualenv + DOCKER_SECURE_ENV_VARS: true + name: "Creating the virtual environment" + run: "bash -e tools/docker_build_and_test.sh" + diff --git a/tools/docker_build_and_test.sh b/tools/docker_build_and_test.sh index bc6e74c7bb4..52847de08e7 100755 --- a/tools/docker_build_and_test.sh +++ b/tools/docker_build_and_test.sh @@ -4,11 +4,13 @@ set -e echo -e "\\n>> [`date`] Starting docker container ${DOCKER_IMG}" -if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then +if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then cp -R ~/.ssh . fi -sudo docker run --name buildvm -v `pwd`:/pycbc:rw ${DOCKER_IMG} /bin/bash -c "bash /pycbc/tools/docker_build_dist.sh --pycbc-container=${PYCBC_CONTAINER} --pull-request=${TRAVIS_PULL_REQUEST} --commit=${TRAVIS_COMMIT} --secure=${TRAVIS_SECURE_ENV_VARS} --tag=${TRAVIS_TAG}" +SOURCE_TAG=`git show-ref | grep ${GITHUB_SHA} | grep -E -o "refs/tags.{0,100}" | cut -c11-` + +sudo docker run --name buildvm -v `pwd`:/pycbc:rw ${DOCKER_IMG} /bin/bash -c "bash /pycbc/tools/docker_build_dist.sh --pycbc-container=${PYCBC_CONTAINER} --pycbc-code=${GITHUB_REF} --secure=${DOCKER_SECURE_ENV_VARS} --tag=${SOURCE_TAG}" echo -e "\\n>> [`date`] Docker exited" diff --git a/tools/docker_build_dist.sh b/tools/docker_build_dist.sh index 31f776a2e54..79a5bc7ec92 100755 --- a/tools/docker_build_dist.sh +++ b/tools/docker_build_dist.sh @@ -5,39 +5,30 @@ set -e for i in $*; do case $i in --pycbc-container=*) PYCBC_CONTAINER="`echo $i|sed 's/^--pycbc-container=//'`";; - --pull-request=*) TRAVIS_PULL_REQUEST="`echo $i|sed 's/^--pull-request=//'`";; - --commit=*) TRAVIS_COMMIT="`echo $i|sed 's/^--commit=//'`";; - --secure=*) TRAVIS_SECURE_ENV_VARS="`echo $i|sed 's/^--secure=//'`";; - --tag=*) TRAVIS_TAG="`echo $i|sed 's/^--tag=//'`";; + --pycbc-code=*) PYCBC_CODE="`echo $i|sed 's/^--pycbc-code=//'`";; + --secure=*) DOCKER_SECURE_ENV_VARS="`echo $i|sed 's/^--secure=//'`";; + --tag=*) SOURCE_TAG="`echo $i|sed 's/^--tag=//'`";; *) echo -e "unknown option '$i', valid are:\n$usage">&2; exit 1;; esac done -# determine the pycbc git branch and origin -if test x$TRAVIS_PULL_REQUEST = "xfalse" ; then - PYCBC_CODE="--pycbc-commit=${TRAVIS_COMMIT}" -else - PYCBC_CODE="--pycbc-fetch-ref=refs/pull/${TRAVIS_PULL_REQUEST}/merge" -fi # set the lalsuite checkout to use -if [ "x$TRAVIS_TAG" == "x" ] ; then - TRAVIS_TAG="master" +if [ "x$SOURCE_TAG" == "x" ] ; then + SOURCE_TAG="master" RSYNC_OPTIONS="--delete" else RSYNC_OPTIONS="" fi echo -e "\\n>> [`date`] Inside container ${PYCBC_CONTAINER}" -echo -e "\\n>> [`date`] Release tag is ${TRAVIS_TAG}" +echo -e "\\n>> [`date`] Release tag is ${SOURCE_TAG}" echo -e "\\n>> [`date`] Using PyCBC code ${PYCBC_CODE}" -echo -e "\\n>> [`date`] Travis pull request is ${TRAVIS_PULL_REQUEST}" -echo -e "\\n>> [`date`] Travis commit is ${TRAVIS_COMMIT}" -echo -e "\\n>> [`date`] Travis secure env is ${TRAVIS_SECURE_ENV_VARS}" -echo -e "\\n>> [`date`] Travis tag is ${TRAVIS_TAG}" +echo -e "\\n>> [`date`] Travis secure env is ${DOCKER_SECURE_ENV_VARS}" +echo -e "\\n>> [`date`] Travis tag is ${SOURCE_TAG}" -if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then +if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then mkdir -p ~/.ssh cp /pycbc/.ssh/* ~/.ssh chmod 600 ~/.ssh/id_rsa @@ -61,10 +52,10 @@ if [ "x${PYCBC_CONTAINER}" == "xpycbc_rhel_virtualenv" ]; then CVMFS_PATH=/cvmfs/oasis.opensciencegrid.org/ligo/sw/pycbc/${ENV_OS}/virtualenv mkdir -p ${CVMFS_PATH} - VENV_PATH=${CVMFS_PATH}/pycbc-${TRAVIS_TAG} + VENV_PATH=${CVMFS_PATH}/pycbc-${SOURCE_TAG} virtualenv ${VENV_PATH} echo 'export PYTHONUSERBASE=${VIRTUAL_ENV}/.local' >> ${VENV_PATH}/bin/activate - echo "export XDG_CACHE_HOME=\${HOME}/cvmfs-pycbc-${TRAVIS_TAG}/.cache" >> ${VENV_PATH}/bin/activate + echo "export XDG_CACHE_HOME=\${HOME}/cvmfs-pycbc-${SOURCE_TAG}/.cache" >> ${VENV_PATH}/bin/activate source ${VENV_PATH}/bin/activate mkdir -p ${VIRTUAL_ENV}/.local echo -e "[easy_install]\\nzip_ok = false\\n" > ~/.pydistutils.cfg @@ -111,24 +102,21 @@ EOF echo -e "\\n>> [`date`] Running test_coinc_search_workflow.sh" mkdir -p /pycbc/workflow-test pushd /pycbc/workflow-test - /pycbc/tools/test_coinc_search_workflow.sh ${VENV_PATH} ${TRAVIS_TAG} + /pycbc/tools/test_coinc_search_workflow.sh ${VENV_PATH} ${SOURCE_TAG} popd - if [ "x${TRAVIS_SECURE_ENV_VARS}" == "xtrue" ] ; then + if [ "x${DOCKER_SECURE_ENV_VARS}" == "xtrue" ] ; then echo -e "\\n>> [`date`] Setting virtual environment permissions for deployment" find ${VENV_PATH} -type d -exec chmod go+rx {} \; chmod -R go+r ${VENV_PATH} echo -e "\\n>> [`date`] Deploying virtual environment ${VENV_PATH}" - echo -e "\\n>> [`date`] Deploying virtual environment to sugwg-condor.phy.syr.edu" - ssh pycbc@sugwg-condor.phy.syr.edu "mkdir -p /home/pycbc/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}" - rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ pycbc@sugwg-condor.phy.syr.edu:/home/pycbc/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}/ - if [ "x${TRAVIS_TAG}" != "xmaster" ] ; then - echo -e "\\n>> [`date`] Deploying release ${TRAVIS_TAG} to CVMFS" + if [ "x${SOURCE_TAG}" != "xmaster" ] ; then + echo -e "\\n>> [`date`] Deploying release ${SOURCE_TAG} to CVMFS" # remove lalsuite source and deploy on cvmfs rm -rf ${VENV_PATH}/src/lalsuite - ssh ouser.ligo@oasis-login.opensciencegrid.org "mkdir -p /home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}" - rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ ouser.ligo@oasis-login.opensciencegrid.org:/home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${TRAVIS_TAG}/ + ssh ouser.ligo@oasis-login.opensciencegrid.org "mkdir -p /home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${SOURCE_TAG}" + rsync --rsh=ssh $RSYNC_OPTIONS -qraz ${VENV_PATH}/ ouser.ligo@oasis-login.opensciencegrid.org:/home/login/ouser.ligo/ligo/deploy/sw/pycbc/${ENV_OS}/virtualenv/pycbc-${SOURCE_TAG}/ ssh ouser.ligo@oasis-login.opensciencegrid.org osg-oasis-update fi echo -e "\\n>> [`date`] virtualenv deployment complete" diff --git a/tools/docker_build_prepssh.sh b/tools/docker_build_prepssh.sh new file mode 100644 index 00000000000..864f385e7b6 --- /dev/null +++ b/tools/docker_build_prepssh.sh @@ -0,0 +1,9 @@ +mkdir -p ~/.ssh +touch ~/.ssh/id_rsa ~/.ssh/ldg_user ~/.ssh/ldg_token +chmod 600 ~/.ssh/id_rsa ~/.ssh/ldg_user ~/.ssh/ldg_token +echo "${OSG_ACCESS}" > ~/.ssh/id_rsa +echo -e "Host sugwg-test1.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ; +echo -e "Host sugwg-condor.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ; +echo -e "Host oasis-login.opensciencegrid.org\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ; +echo -e "Host code.pycbc.phy.syr.edu\n\tStrictHostKeyChecking no\n" >> ~/.ssh/config ; +chmod 600 ~/.ssh/id_rsa ~/.ssh/config ~/.ssh/ldg_user ~/.ssh/ldg_token diff --git a/tools/test_coinc_search_workflow.sh b/tools/test_coinc_search_workflow.sh index 0a65ce85a34..fd7e28c6b55 100755 --- a/tools/test_coinc_search_workflow.sh +++ b/tools/test_coinc_search_workflow.sh @@ -30,17 +30,22 @@ export LIGO_DATAFIND_SERVER='128.230.190.43:80' mkdir $WORKFLOW_NAME pushd $WORKFLOW_NAME +# Doesn't need to be a valid bank file, just needs to exist +echo "DUMMY BANK FILE" > bank.hdf +echo "DUMMY STAT FILE" > statHL.hdf +echo "DUMMY STAT FILE" > statLV.hdf +echo "DUMMY STAT FILE" > statHV.hdf +echo "DUMMY STAT FILE" > statHLV.hdf + echo -e "\\n>> [`date`] Building test workflow $WORKFLOWNAME" pycbc_make_coinc_search_workflow \ --workflow-name ${WORKFLOW_NAME} --output-dir output \ --config-files \ -${CONFIG_PATH}/analysis.ini \ -${CONFIG_PATH}/data_O1.ini \ -${CONFIG_PATH}/plotting.ini \ -${CONFIG_PATH}/injections_minimal.ini \ -${CONFIG_PATH}/executables.ini \ -${CONFIG_PATH}/gps_times_O1_analysis_1.ini \ +/pycbc/examples/search/analysis.ini \ +/pycbc/examples/search/plotting.ini \ +/pycbc/examples/search/injections_minimal.ini \ +/pycbc/examples/search/executables.ini \ --config-overrides \ "results_page:output-path:../../../html"