Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial setup of Payu environment #1

Open
wants to merge 13 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 29 additions & 12 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,34 @@ jobs:
build:
name: Build environment
runs-on: ubuntu-latest
environment: Gadi
steps:
- name: Build environment
uses: appleboy/[email protected]
- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
host: gadi.nci.org.au
username: ${{secrets.GADI_USER}}
key: ${{secrets.DEPLOY_KEY}}
command_timeout: 120m
script: |
export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
cd $JOB_LOG_DIR
qsub -N build_${{ inputs.environment }} -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -q copyq -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/build.sh
hosts: ${{ secrets.HOST }}
private-key: ${{ secrets.SSH_KEY }}

- name: Build environment
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
set -e
export SCRIPT_DIR="${{ secrets.REPO_PATH }}/scripts"
export CONDA_ENVIRONMENT="${{ inputs.environment }}"
export ADMIN_DIR="${{ vars.ADMIN_DIR }}"
export CONDA_BASE="${{ vars.CONDA_BASE }}"
export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}"
export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}"
PROJECT=${{ vars.PROJECT }}
STORAGE=${{ vars.STORAGE }}

source "${SCRIPT_DIR}"/install_config.sh
cd "${JOB_LOG_DIR}"

qsub -N build_"${CONDA_ENVIRONMENT}" -lncpus=1,mem=20GB,walltime=2:00:00,jobfs=50GB,storage="${STORAGE}" \
-v SCRIPT_DIR,CONDA_ENVIRONMENT,ADMIN_DIR,CONDA_BASE,APPS_USERS_GROUP,APPS_OWNERS_GROUP \
-P "${PROJECT}" -q copyq -Wblock=true -Wumask=037 \
"${SCRIPT_DIR}"/build.sh

EOT
34 changes: 23 additions & 11 deletions .github/workflows/deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,28 @@ jobs:
deploy:
name: Deploy environment
runs-on: ubuntu-latest
environment: Gadi
steps:
- name: Deploy environment
uses: appleboy/[email protected]
- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
host: gadi.nci.org.au
username: ${{secrets.GADI_USER}}
key: ${{secrets.DEPLOY_KEY}}
script: |
export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
export CONDA_ENVIRONMENT=${{ inputs.environment }}
eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
eval $( grep BUILD_STAGE_DIR $SCRIPT_DIR/install_config.sh )
"${SCRIPT_DIR}"/deploy.sh
hosts: ${{ secrets.HOST }}
private-key: ${{ secrets.SSH_KEY }}

- name: Deploy environment
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
set -e
export SCRIPT_DIR="${{ secrets.REPO_PATH }}/scripts"
export CONDA_ENVIRONMENT="${{ inputs.environment }}"
export ADMIN_DIR="${{ vars.ADMIN_DIR }}"
export CONDA_BASE="${{ vars.CONDA_BASE }}"
export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}"
export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}"

source "${SCRIPT_DIR}"/install_config.sh

"${SCRIPT_DIR}"/deploy.sh

EOT
2 changes: 1 addition & 1 deletion .github/workflows/get_changed_env.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
for line in $DIFF; do
if [[ $line =~ environments ]]; then
env_name=$( basename ${line%/*} )
if ! [[ $json =~ $env_name ]]; then
if ! [[ $json =~ $env_name ]] && [ -d "environments/$env_name" ]; then
json="$json{\"environment\":\"$env_name\"},"
fi
fi
Expand Down
47 changes: 31 additions & 16 deletions .github/workflows/manual_trigger.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,27 +10,42 @@ on:
jobs:
setup:
runs-on: ubuntu-latest
environment: Gadi
steps:
- name: Checkout repository
uses: actions/checkout@v4

- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
hosts: |
${{ secrets.HOST }}
${{ secrets.HOST_DATA }}
private-key: ${{ secrets.SSH_KEY }}

- name: Sync repository to Gadi
uses: up9cloud/[email protected]
env:
HOST: gadi.nci.org.au
TARGET: ${{secrets.GADI_REPO_PATH}}
KEY: ${{secrets.DEPLOY_KEY}}
USER: ${{secrets.GADI_USER}}
run: |
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-avz --delete --exclude=/.git/ --exclude=/.github/ \
./ \
${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }}

- name: Create Admin dirs on Gadi
uses: appleboy/[email protected]
with:
host: gadi.nci.org.au
username: ${{secrets.GADI_USER}}
key: ${{secrets.DEPLOY_KEY}}
script: |
source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh
source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh
mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
set -e
export ADMIN_DIR="${{ vars.ADMIN_DIR }}"
export CONDA_BASE="${{ vars.CONDA_BASE }}"
export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}"
export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}"

source "${{ secrets.REPO_PATH }}/scripts/install_config.sh"
source "${{ secrets.REPO_PATH }}/scripts/functions.sh"
mkdir -p "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}"
set_admin_perms "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}"

EOT

build:
needs: setup
Expand Down
124 changes: 104 additions & 20 deletions .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
@@ -1,51 +1,135 @@
name: Build and test conda env

on: pull_request

jobs:
generate_matrix:
uses: ./.github/workflows/get_changed_env.yml
setup:

build_base_image:
runs-on: ubuntu-latest
needs: generate_matrix
if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
container:
image: quay.io/singularity/singularity:v3.11.4
options: --privileged
outputs:
built-container: ${{ steps.changed-container-def.outputs.containerdef_any_changed }}
steps:
- name: Checkout repository
### Latest at time of writing
uses: actions/checkout@v4

- name: Check if container definition has changed
id: changed-container-def
uses: tj-actions/changed-files@v41
uses: tj-actions/changed-files@48d8f15b2aaa3d255ca5af3eba4870f807ce6b3c # v43.0.2
with:
files_yaml: |
containerdef:
- container/container.def

- name: Build container if definition has changed
if: steps.changed-container-def.outputs.containerdef_any_changed == 'true'
run: |
echo ${{ needs.generate_matrix.outputs.matrix }}
sudo -E singularity build container/base.sif container/container.def

- name: Upload base container
if: steps.changed-container-def.outputs.containerdef_any_changed == 'true'
uses: actions/upload-artifact@v4
with:
name: base-container
path: container/base.sif

setup:
runs-on: ubuntu-latest
needs: [ generate_matrix, build_base_image ]
if: ${{ needs.generate_matrix.outputs.matrix != '{"include":[]}' }}
environment: Gadi
steps:
- name: Check Gadi Github Environment configuration
run: |
vars_unset=false
if [ -z "${{ vars.ADMIN_DIR }}" ]; then
echo "::error::vars.ADMIN_DIR is unset. ADMIN_DIR is the base directory which will contain the staging and logs sub-directories"
vars_unset=true
fi
if [ -z "${{ vars.CONDA_BASE }}" ]; then
echo "::error::vars.CONDA_BASE is unset. CONDA_BASE is the base deployment directory which will contain apps and module sub-directories"
vars_unset=true
fi
if [ -z "${{ vars.PROJECT }}" ]; then
echo "::error::vars.PROJECT is unset. PROJECT is the project code passed to build+test qsub calls"
vars_unset=true
fi
if [ -z "${{ vars.STORAGE }}" ]; then
echo "::error::vars.STORAGE is unset. STORAGE is the storage flags passed to build+test qsub calls"
vars_unset=true
fi
if [ -z "${{ vars.APPS_USERS_GROUP }}" ]; then
echo "::error::vars.APPS_USERS_GROUP is unset. APPS_USERS_GROUP is used for setting file permissions"
vars_unset=true
fi
if [ -z "${{ vars.APPS_OWNERS_GROUP }}" ]; then
echo "::error::APPS_OWNERS_GROUP is unset. APPS_OWNERS_GROUP is used for setting file permissions"
vars_unset=true
fi
if [ -z "${{ secrets.REPO_PATH }}" ]; then
echo "::error::secrets.REPO_PATH is unset. REPO_PATH is the path where this repository is synced to"
vars_unset=true
fi

if [ "$vars_unset" == "true" ]; then
echo "::error::Required vars in Gadi Github Environment are unset."
exit 1
fi

- name: Checkout repository
uses: actions/checkout@v4
Comment on lines +87 to +88
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One of the things that I'm not too sure about when using the relative uses: ./.github/workflows/thingo.yml as opposed to the version-specific uses: access-nri/model-release-condaenv/.github/workflows/thingo.yml@main is that in the case where it is on.pull_request - the checkout that you do here means that the version of the deploy workflow that you use here would be the pull request version, rather than the @main version, for example.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the relative paths were useful to the use the test the changes in this PR. But maybe if the build and test workflows work, the workflow changes should be separated into a different PR and merged first, with the versioned values @main?

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah that will work :)


- name: Download base container
if: needs.build_base_image.outputs.built-container == 'true'
uses: actions/download-artifact@v4
with:
name: base-container
path: container

- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
hosts: |
${{ secrets.HOST }}
${{ secrets.HOST_DATA }}
private-key: ${{ secrets.SSH_KEY }}

- name: Create parent repository directories
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
mkdir -p ${{ secrets.REPO_PATH }}
EOT

- name: Sync repository to Gadi
### Latest at time of writing
uses: up9cloud/[email protected]
env:
HOST: gadi.nci.org.au
TARGET: ${{secrets.GADI_REPO_PATH}}
KEY: ${{secrets.DEPLOY_KEY}}
USER: ${{secrets.GADI_USER}}
run: |
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-avz --delete --exclude=/.git/ --exclude=/.github/ \
./ \
${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }}

- name: Create Admin dirs on Gadi
uses: appleboy/[email protected]
with:
host: gadi.nci.org.au
username: ${{secrets.GADI_USER}}
key: ${{secrets.DEPLOY_KEY}}
script: |
source ${{secrets.GADI_REPO_PATH}}/scripts/install_config.sh
source ${{secrets.GADI_REPO_PATH}}/scripts/functions.sh
mkdir -p $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
set_admin_perms $ADMIN_DIR $JOB_LOG_DIR $BUILD_STAGE_DIR
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
set -e
export ADMIN_DIR="${{ vars.ADMIN_DIR }}"
export CONDA_BASE="${{ vars.CONDA_BASE }}"
export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}"
export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}"

source "${{ secrets.REPO_PATH }}/scripts/install_config.sh"
source "${{ secrets.REPO_PATH }}/scripts/functions.sh"
mkdir -p "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}"
set_admin_perms "${ADMIN_DIR}" "${JOB_LOG_DIR}" "${BUILD_STAGE_DIR}"
EOT

build:
needs: [ generate_matrix, setup ]
Expand Down
23 changes: 15 additions & 8 deletions .github/workflows/push_to_main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,26 @@ jobs:

setup:
runs-on: ubuntu-latest
environment: Gadi
steps:
- name: Checkout repository
### Latest at time of writing
uses: actions/checkout@v4

- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
hosts: ${{ secrets.HOST_DATA }}
private-key: ${{ secrets.SSH_KEY }}

- name: Sync repository to Gadi
### Latest at time of writing
uses: up9cloud/[email protected]
env:
HOST: gadi.nci.org.au
TARGET: ${{secrets.GADI_REPO_PATH}}
KEY: ${{secrets.DEPLOY_KEY}}
USER: ${{secrets.GADI_USER}}

run: |
rsync -e 'ssh -i ${{ steps.ssh.outputs.private-key-path }}' \
-avz --delete --exclude=/.git/ --exclude=/.github/ \
./ \
${{ secrets.USER }}@${{ secrets.HOST_DATA }}:${{ secrets.REPO_PATH }}

deploy:
needs: [ generate_matrix, setup ]
uses: ./.github/workflows/deploy.yml
Expand Down
40 changes: 29 additions & 11 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,16 +9,34 @@ jobs:
test:
name: Test environment
runs-on: ubuntu-latest
environment: Gadi
steps:
- name: Test environment
uses: appleboy/[email protected]
- name: Setup SSH
uses: access-nri/actions/.github/actions/setup-ssh@main
id: ssh
with:
host: gadi.nci.org.au
username: ${{secrets.GADI_USER}}
key: ${{secrets.DEPLOY_KEY}}
script: |
export SCRIPT_DIR=${{secrets.GADI_REPO_PATH}}/scripts
eval $( grep ADMIN_DIR $SCRIPT_DIR/install_config.sh )
eval $( grep JOB_LOG_DIR $SCRIPT_DIR/install_config.sh )
cd $JOB_LOG_DIR
qsub -N test_${{ inputs.environment }} -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage=gdata/v45+scratch/v45+gdata/hh5+scratch/hh5 -v SCRIPT_DIR,CONDA_ENVIRONMENT=${{ inputs.environment }} -P kr06 -Wblock=true -Wumask=037 "${SCRIPT_DIR}"/test.sh
hosts: ${{ secrets.HOST }}
private-key: ${{ secrets.SSH_KEY }}

- name: Test environment
run: |
ssh ${{ secrets.USER }}@${{ secrets.HOST }} -i ${{ steps.ssh.outputs.private-key-path }} /bin/bash <<'EOT'
set -e
export SCRIPT_DIR="${{secrets.REPO_PATH}}/scripts"
export CONDA_ENVIRONMENT="${{ inputs.environment }}"
export ADMIN_DIR="${{ vars.ADMIN_DIR }}"
export CONDA_BASE="${{ vars.CONDA_BASE }}"
export APPS_USERS_GROUP="${{ vars.APPS_USERS_GROUP }}"
export APPS_OWNERS_GROUP="${{ vars.APPS_OWNERS_GROUP }}"
PROJECT=${{ vars.PROJECT }}
STORAGE=${{ vars.STORAGE }}

source "${SCRIPT_DIR}"/install_config.sh
cd "${JOB_LOG_DIR}"

qsub -N test_"${CONDA_ENVIRONMENT}" -lncpus=4,mem=20GB,walltime=0:20:00,jobfs=50GB,storage="${STORAGE}" \
-v SCRIPT_DIR,CONDA_ENVIRONMENT,ADMIN_DIR,CONDA_BASE,APPS_USERS_GROUP,APPS_OWNERS_GROUP \
-P "${PROJECT}" -Wblock=true -Wumask=037 \
"${SCRIPT_DIR}"/test.sh

EOT
Loading