Skip to content

Merge pull request #2016 from cBioPortal/supp_headers #295

Merge pull request #2016 from cBioPortal/supp_headers

Merge pull request #2016 from cBioPortal/supp_headers #295

Workflow file for this run

name: Zip studies on push to master branch
on:
# Trigger the workflow on push,
# but only for the master branch
push:
branches:
- master
paths:
- 'public/**'
jobs:
zip_studies:
if: github.repository == 'cbioportal/datahub'
runs-on: ubuntu-latest
steps:
- id: files
uses: jitterbit/get-changed-files@v1
- run: |
echo "##[set-output name=unique_studies_list;]$(echo "${{ steps.files.outputs.all }}" | awk 'BEGIN { RS=" "; FS="/"; ORS=" " } { set[$2]++ } END { for (dir in set) { print dir } }')"
exit 0
id: unique_studies
- uses: actions/checkout@v2
- run: |
# detecting modified studies and archiving them
# create a list of all studies under 'public'
echo creating ${STUDY_LIST_FILENAME}
echo "[" > ${STUDY_LIST_FILENAME}
first_study=1
find public -type d -mindepth 1 -maxdepth 1 | sort > ${STUDY_PATH_FILENAME}
while read studypath ; do
if [ ${first_study} -ne 1 ] ; then
echo "," >> ${STUDY_LIST_FILENAME}
fi
studyname="${studypath:7}"
echo -n "\"${studyname}\"" >> ${STUDY_LIST_FILENAME}
first_study=0
done < ${STUDY_PATH_FILENAME}
rm -f ${STUDY_PATH_FILENAME}
echo "" >> ${STUDY_LIST_FILENAME}
echo "]" >> ${STUDY_LIST_FILENAME}
# copy study list file to S3
echo copying ${STUDY_LIST_FILENAME} to S3
aws s3 cp --acl public-read ${STUDY_LIST_FILENAME} s3://${AWS_S3_BUCKET}/
rm -f ${STUDY_LIST_FILENAME}
# first save .git
tar -cvf ${SAVED_GIT_REPO_FILENAME} .git > /dev/null
mkdir studies_to_upload
cd public
for changed_study in ${{ steps.unique_studies.outputs.unique_studies_list }}; do
if [ ! -d "${changed_study}" ]; then
continue
fi
echo "Pulling study: ${changed_study}."
git lfs pull --include="${changed_study}"
echo "Compressing this study: ${changed_study}."
tar -czvf ../studies_to_upload/${changed_study}.tar.gz ${changed_study}
echo "Copy file to S3: ../studies_to_upload/${changed_study}.tar.gz"
aws s3 cp --acl public-read ../studies_to_upload/${changed_study}.tar.gz s3://${AWS_S3_BUCKET}/
echo "Remove this directory: ${changed_study}."
rm -rf "${changed_study}"
echo "Remove this file: ../studies_to_upload/${changed_study}.tar.gz."
rm ../studies_to_upload/${changed_study}.tar.gz
echo "Remove .git"
cd ..
rm -rf .git
# add original .git back
tar -xf ${SAVED_GIT_REPO_FILENAME} > /dev/null
cd public
done
cd ..
env:
STUDY_PATH_FILENAME: study_paths.txt
STUDY_LIST_FILENAME: study_list.json
SAVED_GIT_REPO_FILENAME: save_git.tar
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
AWS_EC2_METADATA_DISABLED: "true" # fixes aws cli error (code 255); needed since ubuntu-latest v2.262.1