diff --git a/.github/workflows/builder.yml b/.github/workflows/builder.yml new file mode 100644 index 0000000..a000381 --- /dev/null +++ b/.github/workflows/builder.yml @@ -0,0 +1,63 @@ +name: singularity-deploy + +on: + push: + branches: + - "main" + +jobs: + release: + name: Create Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Define Repository Name and Release Version + run: | + repo=$(echo "${GITHUB_REPOSITORY/\//-}") + release=$(cat VERSION) + echo "reponame=$repo" >> $GITHUB_ENV + echo "release_tag=$release" >> $GITHUB_ENV + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ env.release_tag }} + release_name: Release ${{ env.release_tag }} + draft: false + prerelease: false + - uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.6.4 + - name: Build the singularity container + run: | + repo=$(echo "${GITHUB_REPOSITORY/\//-}") + + # For each Singularity* container, build based on the prefix (tag) + for recipe in $(ls Singularity*); do + echo "Building $recipe" + tag=$(echo "${recipe/Singularity\./}") + # If we find empty, use latest + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + container="$repo:$tag.sif" + singularity build --fakeroot container.sif Singularity + if [ "$?" == "0" ]; then + echo "Successfully built container $container." + mv container.sif "$container" + else + echo "There was an issue building $container." + fi + done + - name: Upload Release Assets + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + upload_url: ${{ steps.create_release.outputs.upload_url }} + tag_name: ${{ env.release_tag }} + run: | + hub release edit $(find . -type f -name "*.sif" -printf "-a %p ") -m "" "$tag_name" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..784a3b1 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,42 @@ +name: singularity-deploy-test + +# run on creation of a release tag +on: + pull_request: [] + +jobs: + builder: + name: Test Container Builds + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - uses: eWaterCycle/setup-singularity@v6 + with: + singularity-version: 3.6.4 + - name: Build the singularity containers + run: | + repo=$(echo "${GITHUB_REPOSITORY/\//-}") + + # For each Singularity* container, build based on the prefix (tag) + for recipe in $(ls Singularity*); do + echo "Building $recipe" + tag=$(echo "${recipe/Singularity\./}") + # If we find empty, use latest + if [ "$tag" == "Singularity" ]; then + tag=latest + fi + # Build the container and name by tag + echo "Tag is $tag." + container="$repo:$tag.sif" + singularity build --fakeroot container.sif Singularity + if [ "$?" == "0" ]; then + echo "Successfully built container $container." + mv container.sif "$container" + else + echo "There was an issue building $container." + fi + done + - name: Run Additional Tests + run: | + echo "Run any additional tests here." diff --git a/README.md b/README.md new file mode 100644 index 0000000..639abd0 --- /dev/null +++ b/README.md @@ -0,0 +1,108 @@ +# Singularity Deploy + +[![GitHub actions status](https://github.com/singularityhub/singularity-deploy/workflows/singularity-deploy/badge.svg?branch=main)](https://github.com/singularityhub/singularity-deploy/actions?query=branch%3Amain+workflow%3Asingularity-deploy) +[![GitHub actions status](https://github.com/singularityhub/singularity-deploy/workflows/singularity-deploy-test/badge.svg?branch=main)](https://github.com/singularityhub/singularity-deploy/actions?query=branch%3Amain+workflow%3Asingularity-deploy-test) + +Wouldn't it be nice to build Singularity images without a registry proper, +and just keep them alongside the GitHub codebase? This is now possible! +This small repository provides an example to get you started. It will +build one or more images (whatever Singularity.* files that are present at +the root) and then release them as assets to your GitHub repository so +that they can be programatically obtained. Note that assets are limited to +2 GB in size, which is still fairly good. You can use +it as a template for your own recipes as is, or modify it for your custom +use case. Instructions are below! + +## Getting Started + +### 1. Template or Fork + +If you haven't already, template or fork this repository. You can then clone +your fork: + +```bash +$ git clone git@github.com:/singularity-deploy +``` + +You likely want to name the repository by the container. For example, if I would +have created a container on Docker Hub or similar with the name `vsoch/salad`, +here I'd call the repository `salad`. You obviously are limited to your username +or an organizational namespace. + +### 1. Write your Singularity Recipe(s) + +First, you should write your container recipe(s) in the present working directory. +For good practice, when you are updating recipes you should checkout a new branch +and open a pull request, as the repository comes with a workflow to trigger on a PR +to [test your container build](.github/workflows/test.yml). You can add any additional +tests that that you might need. By default, any Singularity.* file will be automatically detected. +If there is no extension (the name Singularity), the name used will be "latest." +You can use these tags across multiple releases of your containers. For example, +these files would generate packages with sifs named as follows: + + - [Singularity](Singularity) maps to [https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.latest.sif](https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.latest.sif) + - [Singularity.pokemon](Singularity.pokemon) maps to [https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.pokemon.sif](https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.pokemon.sif) + - [Singularity.salad](Singularity.salad) maps to [https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.latest.sif](https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.latest.sif) + +For each name, you can see the direct download URL contains the repository (singularityhub/singularity-deploy), +You should not use any `:` characters in either your container tag (the GitHub extension) or +the GitHub tags (the release tags) as this might interfere with parsing. +The GitHub release tag (0.0.1 in the example above) is discussed next. + +### 2. Update the VERSION file + +Any time that you prepare new container recipes, you should update the [VERSION](VERSION) +file. The way that this repository works is to generate a release based on the +string in `VERSION`. A version is just a tag, so it could be something like +`0.0.1` or `0.0.1-slim`. Keep in mind that GitHub releases cannot have duplicated +names, so you should not repeat the same tag. Do not use `:` in your tag names. +If you do need to re-release a tag (not recommended if a user might be using it and then it's changed) you can manually delete +the release and the tag in the GitHub interface. This is a nice structure because it +means you can have containers with different names under the same tag. In the example +above, we have each of "deploy," "latest," and "salad" released under tag 0.0.1. +This is how it looks on GitHub: + +![img/releases.png](img/releases.png) + +### 3. How to Develop + +As we mentioned previously, the container builds will be tested on a pull request, +and the release will trigger on merge into your main branch (main). See the [.github/workflows/builder.yml](.github/workflows/builder.yml)) +to edit this. The idea is that you can: + +1. Develop your container via a development branch +2. Open a pull request to test the container (the [.github/workflows/test.yml](.github/workflows/test.yml)) +3. On merge, your container will be released! + +### 4. How to pull + +Technically, Singularity can pull just knowing the URL. For example: + +```bash +$ singularity pull https://github.com/singularityhub/singularity-deploy/releases/download/0.0.1/singularityhub-singularity-deploy.latest.sif +``` + +However, the [singularity-hpc](singularity-hpc) tool (will be) designed to be able to parse and handle +these container uris automatically. For the containers here, you could do: + +```bash +$ shpc pull gh://singularityhub/singularity-deploy/0.0.1:latest +$ shpc pull gh://singularityhub/singularity-deploy/0.0.1:salad +$ shpc pull gh://singularityhub/singularity-deploy/0.0.1:pokemon +``` + +or write the container URI into a registry entry: + +``` +gh: singularityhub/singularity-deploy +latest: + latest: "0.0.1" +tags: + "latest": "0.0.1" + "salad": "0.0.1" + "pokemon": "0.0.1" +maintainer: "@vsoch" +url: https://github.com/singularityhub/singularity-deploy +``` + +(This part is still under development!) diff --git a/Singularity b/Singularity new file mode 100644 index 0000000..59192fe --- /dev/null +++ b/Singularity @@ -0,0 +1,6 @@ +Bootstrap: docker +From: busybox + +%post + +echo "this is a tiny example container." diff --git a/Singularity.pokemon b/Singularity.pokemon new file mode 100644 index 0000000..49dedc4 --- /dev/null +++ b/Singularity.pokemon @@ -0,0 +1,6 @@ +Bootstrap: docker +From: vanessa/pokemon + +%post + +echo "hello this is pokemon build." diff --git a/Singularity.salad b/Singularity.salad new file mode 100644 index 0000000..ef0a291 --- /dev/null +++ b/Singularity.salad @@ -0,0 +1,6 @@ +Bootstrap: docker +From: vanessa/salad + +%post + +echo "hello salad, I am spoon." diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..8acdd82 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.1 diff --git a/img/releases.png b/img/releases.png new file mode 100644 index 0000000..06b7ffb Binary files /dev/null and b/img/releases.png differ