Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify building workflow - build directly in GHA #32

Merged
merged 2 commits into from
Mar 26, 2024
Merged

Simplify building workflow - build directly in GHA #32

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
aa9cd1c
CI: simplify building workflow - build directly in GHA
mrakitin Mar 22, 2024
4f795c8
CLN: remove commented-out lines
mrakitin Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 55 additions & 72 deletions .github/workflows/package.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,51 +20,47 @@ jobs:
shell: bash -l {0}

steps:
- name: Checkout the code
uses: actions/checkout@v4

- name: Install Python for YAML CLI tools
uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Install YAML CLI tools
run: |
python3 -m pip install shyaml

- name: Set env vars
run: |
set -vxeuo pipefail

export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo
echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV

export DATETIME_STRING=$(date +%Y%m%d%H%M%S)
echo "DATETIME_STRING=${DATETIME_STRING}" >> $GITHUB_ENV

- name: Set non-secret environment variables
run: |
export PYTHONVER=$(echo ${{ matrix.python-version }} | sed 's/\.//g')
echo "PYTHONVER=${PYTHONVER}" >> $GITHUB_ENV

export CONDA_PACK_TEMPLATE_DIR=${HOME}/conda-pack-template
echo "CONDA_PACK_TEMPLATE_DIR=${CONDA_PACK_TEMPLATE_DIR}" >> $GITHUB_ENV

- name: checkout the code
uses: actions/checkout@v3
env_name=$(cat configs/config-py${PYTHONVER}.yml | shyaml get-value env_name)
export CONDA_PACK_ENV_NAME=${env_name}
echo "CONDA_PACK_ENV_NAME=${CONDA_PACK_ENV_NAME}" >> $GITHUB_ENV

- name: checkout NSLS-II/conda-pack-template
run: |
git clone https://github.com/NSLS-II/conda-pack-template ${CONDA_PACK_TEMPLATE_DIR}
cd ${CONDA_PACK_TEMPLATE_DIR}
env | sort -u

- uses: conda-incubator/setup-miniconda@v2
- name: Setup umamba
uses: mamba-org/setup-micromamba@v1
with:
activate-environment: ${{ env.REPOSITORY_NAME }}
auto-update-conda: true
miniconda-version: "latest"
python-version: ${{ matrix.python-version }}

- name: pip-install dependencies
run: |
# For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html.
set -vxeuo pipefail
python3 -m pip install -r ${CONDA_PACK_TEMPLATE_DIR}/requirements.txt
python3 -m pip install shyaml

- name: pip-install dev-dependencies
run: |
# For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html.
set -vxeuo pipefail
python3 -m pip install -r ${CONDA_PACK_TEMPLATE_DIR}/requirements-dev.txt
environment-file: envs/env-py${{ env.PYTHONVER }}.yml
log-level: info

- name: check env
- name: Check env
run: |
# For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html.
set -vxeuo pipefail
Expand All @@ -76,67 +72,54 @@ jobs:
conda config --show
printenv | sort

- name: render config
- name: Export files
run: |
# For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html.
set -vxeo pipefail

env_name=$(cat configs/config-py${PYTHONVER}.yml | shyaml get-value env_name)
export CONDA_PACK_ENV_NAME="${env_name}"
echo "CONDA_PACK_ENV_NAME=${CONDA_PACK_ENV_NAME}" >> $GITHUB_ENV

dockerfile=$(python3 ${CONDA_PACK_TEMPLATE_DIR}/render.py -c configs/config-py${PYTHONVER}.yml -f Dockerfile.j2)
runner=$(python3 ${CONDA_PACK_TEMPLATE_DIR}/render.py -c configs/config-py${PYTHONVER}.yml -f runner.sh.j2)
echo "${dockerfile}"
echo "${runner}"
ls -la
cat "${dockerfile}"
cat "${runner}"

export CONDA_PACK_TEMPLATE_RUNNER="${runner}"
echo "CONDA_PACK_TEMPLATE_RUNNER=${CONDA_PACK_TEMPLATE_RUNNER}" >> $GITHUB_ENV

- name: run a build with Docker/Podman
export ARTIFACTS_DIR="$HOME/artifacts"
echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> $GITHUB_ENV
if [ ! -d "${ARTIFACTS_DIR}" ]; then
mkdir -v -p "${ARTIFACTS_DIR}"
fi

# conda env export -n ${CONDA_PACK_ENV_NAME} -f ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.yml -c conda-forge --override-channels
conda env export -f ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.yml
# Per https://conda.github.io/conda-pack/cli.html:
conda-pack -o ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.tar.gz --ignore-missing-files --ignore-editable-packages
openssl sha256 ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.tar.gz > ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}-sha256sum.txt
openssl md5 ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.tar.gz > ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}-md5sum.txt
chmod -v 664 ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}[.-]*

- name: Contents of the env .yml file
run: |
# For reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html.
set -vxeo pipefail
cp -v ${CONDA_PACK_TEMPLATE_DIR}/export.sh .
bash ${CONDA_PACK_TEMPLATE_RUNNER}
ls -laF
cp -v Dockerfile ${CONDA_PACK_TEMPLATE_RUNNER} artifacts/
sudo chown -v $USER: artifacts/*
ls -laF artifacts/

- name: contents of the env .yml file
run: |
cat artifacts/${CONDA_PACK_ENV_NAME}.yml
cat ${ARTIFACTS_DIR}/${CONDA_PACK_ENV_NAME}.yml

- name: checksum files
- name: Checksum files
run: |
cat artifacts/*sum.txt
cat ${ARTIFACTS_DIR}/*sum.txt

# https://github.com/actions/upload-artifact
- name: upload artifacts for branch
- name: Upload artifacts for the env .yml file
uses: actions/upload-artifact@v4
with:
name: ${{ env.CONDA_PACK_ENV_NAME }}.yml
path: ${{ env.ARTIFACTS_DIR }}/${{ env.CONDA_PACK_ENV_NAME }}.yml
retention-days: 60

- name: Upload artifacts for branch
if: |
github.ref != 'refs/heads/main'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: test-${{ env.CONDA_PACK_ENV_NAME }}-${{ env.DATETIME_STRING }}
path: artifacts
path: ${{ env.ARTIFACTS_DIR }}
retention-days: 14

- name: upload artifacts for release
- name: Upload artifacts for release
if: |
github.ref == 'refs/heads/main'
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: ${{ env.CONDA_PACK_ENV_NAME }}
path: artifacts
retention-days: 60

- name: upload artifacts for the env .yml file
uses: actions/upload-artifact@v3
with:
name: ${{ env.CONDA_PACK_ENV_NAME }}.yml
path: artifacts/${{ env.CONDA_PACK_ENV_NAME}}.yml
path: ${{ env.ARTIFACTS_DIR }}
retention-days: 60
Loading