Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update to viash 0.9 #911

Merged
merged 10 commits into from
Oct 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 10 additions & 115 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,121 +6,16 @@ on:
workflow_dispatch:
inputs:
version:
description: Version tag
required: true
description: |
The version of the project to build. Example: `1.0.3`.

If not provided, a development build with a version name
based on the branch name will be built. Otherwise, a release
build with the provided version will be built.
required: false

jobs:
# phase 1
list:
runs-on: ubuntu-latest

outputs:
target_branch: ${{ steps.variables.outputs.target_branch }}
version: ${{ steps.variables.outputs.version }}
docker_matrix: ${{ steps.ns_list_docker.outputs.output_matrix }}

steps:
- name: Check out repository
uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0

- uses: viash-io/viash-actions/setup@v6

- name: Determine variables
id: variables
run: |
VERSION="${{ inputs.version }}"
SOURCE_BRANCH=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///')

if [[ -z $VERSION ]]; then
TARGET_BRANCH="build/$SOURCE_BRANCH"
VERSION=$(echo "$TARGET_BRANCH" | sed 's/[^a-zA-Z0-9_]/_/g')
else
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+.*$ ]]; then
echo "Version '$VERSION' does not match PEP440"
exit 1
fi
TARGET_BRANCH="release/${VERSION%.*}.x"
fi

echo "Set version of Viash package to '$VERSION'"
echo "version=$VERSION" >> $GITHUB_OUTPUT

echo "Set target branch to '$TARGET_BRANCH'"
echo "target_branch=$TARGET_BRANCH" >> $GITHUB_OUTPUT

- name: Remove target folder from .gitignore
run: |
# allow publishing the target folder
sed -i 's#^/target/$##g' .gitignore

- uses: viash-io/viash-actions/ns-build@v6
with:
config_mod: .functionality.version := '${{ steps.variables.outputs.version }}'
parallel: true

- name: Deploy to target branch
uses: peaceiris/actions-gh-pages@v4
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
publish_dir: .
publish_branch: ${{ steps.variables.outputs.target_branch }}

- id: ns_list_docker
uses: viash-io/viash-actions/ns-list@v6
with:
platform: docker
src: src
format: json

# phase 2
build:
needs: list

runs-on: ubuntu-latest

permissions:
contents: read
packages: write

strategy:
fail-fast: false
matrix:
component: ${{ fromJson(needs.list.outputs.docker_matrix) }}

steps:
# Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
- uses: data-intuitive/reclaim-the-bytes@v2

- uses: actions/checkout@v4
with:
submodules: 'recursive'
fetch-depth: 0
ref: ${{ needs.list.outputs.target_branch }}

- uses: viash-io/viash-actions/setup@v6

- name: Build container
uses: viash-io/viash-actions/ns-build@v6
with:
config_mod: .functionality.version := '${{ needs.list.outputs.version }}'
platform: docker
src: ${{ matrix.component.dir }}
setup: build

- name: Login to container registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Push container
uses: viash-io/viash-actions/ns-build@v6
with:
config_mod: .functionality.version := '${{ needs.list.outputs.version }}'
platform: docker
src: ${{ matrix.component.dir }}
setup: push
uses: viash-io/viash-actions/.github/workflows/build.yaml@v6
with:
version: ${{ github.event.inputs.version }}
87 changes: 5 additions & 82 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -1,88 +1,11 @@
name: Test

on:
pull_request:
push:
branches: [ main ]
branches:
- main
pull_request:

jobs:

# phase 1
list:
env:
s3_bucket: s3://openproblems-data/resources_test/
runs-on: ubuntu-latest

outputs:
matrix: ${{ steps.set_matrix.outputs.matrix }}
cache_key: ${{ steps.cache.outputs.cache_key }}

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- uses: viash-io/viash-actions/setup@v6

- uses: viash-io/viash-actions/project/sync-and-cache-s3@v6
id: cache
with:
s3_bucket: $s3_bucket
dest_path: resources_test
cache_key_prefix: resources_test__

- id: ns_list
uses: viash-io/viash-actions/ns-list@v6
with:
platform: docker
format: json

- id: ns_list_filtered
uses: viash-io/viash-actions/project/detect-changed-components@v6
with:
input_file: "${{ steps.ns_list.outputs.output_file }}"

- id: set_matrix
run: |
echo "matrix=$(jq -c '[ .[] |
{
"name": (.functionality.namespace + "/" + .functionality.name),
"config": .info.config
}
]' ${{ steps.ns_list_filtered.outputs.output_file }} )" >> $GITHUB_OUTPUT

# phase 2
viash_test:
needs: list
if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
runs-on: ubuntu-latest

strategy:
fail-fast: false
matrix:
component: ${{ fromJson(needs.list.outputs.matrix) }}

steps:
# Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
- uses: data-intuitive/reclaim-the-bytes@v2

- uses: actions/checkout@v4

- uses: viash-io/viash-actions/setup@v6

# use cache
- name: Cache resources data
uses: actions/cache@v4
timeout-minutes: 10
with:
path: resources_test
key: ${{ needs.list.outputs.cache_key }}

- name: Run test
timeout-minutes: 30
run: |
VIASH_TEMP=$RUNNER_TEMP/viash viash test \
"${{ matrix.component.config }}" \
--cpus 2 \
--memory "5gb"

test:
uses: viash-io/viash-actions/.github/workflows/test.yaml@v6
3 changes: 3 additions & 0 deletions .gitmodules
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[submodule "common"]
path = common
url = https://github.com/openproblems-bio/common_resources.git
6 changes: 0 additions & 6 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,4 @@
{
"yaml.schemas": {
"src/common/schemas/api_component.yaml": "src/**/api/comp_*.yaml",
"src/common/schemas/api_file.yaml": "src/**/api/file_*.yaml",
"src/common/schemas/task_info.yaml": "src/**/api/task_info.yaml",
"src/common/schemas/task_method.yaml": "src/tasks/**/methods/**/config.vsh.yaml",
"src/common/schemas/task_control_method.yaml": "src/tasks/**/control_methods/**/config.vsh.yaml",
"src/common/schemas/task_metric.yaml": "src/tasks/**/metrics/**/config.vsh.yaml"
}
}
20 changes: 12 additions & 8 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,25 @@

## Breaking changes

- Moved `src/tasks/batch_integration` to [`task_batch_integration`](https://github.com/openproblems-bio/task_batch_integration).
- Moved `src/tasks/batch_integration` to [`task_batch_integration`](https://github.com/openproblems-bio/task_batch_integration) (PR #910).

- Moved `src/tasks/denoising` to [`task_denoising`](https://github.com/openproblems-bio/task_denoising).
- Moved `src/tasks/denoising` to [`task_denoising`](https://github.com/openproblems-bio/task_denoising) (PR #910).

- Moved `src/tasks/dimensionality_reduction` to [`task_dimensionality_reduction`](https://github.com/openproblems-bio/task_dimensionality_reduction).
- Moved `src/tasks/dimensionality_reduction` to [`task_dimensionality_reduction`](https://github.com/openproblems-bio/task_dimensionality_reduction) (PR #910).

- Moved `src/tasks/label_projection` to [`task_label_projection`](https://github.com/openproblems-bio/task_label_projection).
- Moved `src/tasks/label_projection` to [`task_label_projection`](https://github.com/openproblems-bio/task_label_projection) (PR #910).

- Moved `src/tasks/match_modalities` to [`task_match_modalities`](https://github.com/openproblems-bio/task_match_modalities).
- Moved `src/tasks/match_modalities` to [`task_match_modalities`](https://github.com/openproblems-bio/task_match_modalities) (PR #910).

- Moved `src/tasks/predict_modality` to [`task_predict_modality`](https://github.com/openproblems-bio/task_predict_modality).
- Moved `src/tasks/predict_modality` to [`task_predict_modality`](https://github.com/openproblems-bio/task_predict_modality) (PR #910).

- Moved `src/tasks/spatial_decomposition` to [`task_spatial_decomposition`](https://github.com/openproblems-bio/task_spatial_decomposition).
- Moved `src/tasks/spatial_decomposition` to [`task_spatial_decomposition`](https://github.com/openproblems-bio/task_spatial_decomposition) (PR #910).

- Moved `src/tasks/spatially_variable_genes` to [`task_spatially_variable_genes`](https://github.com/openproblems-bio/task_spatially_variable_genes).
- Moved `src/tasks/spatially_variable_genes` to [`task_spatially_variable_genes`](https://github.com/openproblems-bio/task_spatially_variable_genes) (PR #910).

## Major changes

- Update Viash to 0.9.0 (PR #911).

## Minor changes

Expand Down
43 changes: 32 additions & 11 deletions _viash.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,35 @@
viash_version: 0.8.6
name: openproblems
version: dev
organization: openproblems-bio
viash_version: 0.9.0

source: src
target: target
description: |
Open Problems is a living, extensible, community-guided benchmarking platform.
license: MIT
keywords: [openproblems, benchmarking, single-cell omics]

references:
doi:
# Malte Luecken, Scott Gigante, Daniel Burkhardt, Robrecht Cannoodt, et al.
# Defining and benchmarking open problems in single-cell analysis,
# 03 April 2024, PREPRINT (Version 1) available at Research Square [https://doi.org/10.21203/rs.3.rs-4181617/v1]
- 10.21203/rs.3.rs-4181617/v1

links:
issue_tracker: https://github.com/openproblems-bio/openproblems/issues
repository: https://github.com/openproblems-bio/openproblems
docker_registry: ghcr.io

config_mods: |
.functionality.version := 'dev'
.platforms[.type == 'docker'].target_registry := 'ghcr.io'
.platforms[.type == 'docker'].target_organization := 'openproblems-bio/openproblems'
.platforms[.type == 'docker'].target_image_source := 'https://github.com/openproblems-bio/openproblems'
.platforms[.type == "nextflow"].directives.tag := "$id"
.platforms[.type == "nextflow"].auto.simplifyOutput := false
.platforms[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
.platforms[.type == "nextflow"].config.script := "process.errorStrategy = 'ignore'"
.runners[.type == "nextflow"].config.labels := { lowmem : "memory = 20.Gb", midmem : "memory = 50.Gb", highmem : "memory = 100.Gb", lowcpu : "cpus = 5", midcpu : "cpus = 15", highcpu : "cpus = 30", lowtime : "time = 1.h", midtime : "time = 4.h", hightime : "time = 8.h", veryhightime : "time = 24.h" }
.runners[.type == "nextflow"].config.script := "process.errorStrategy = 'ignore'"


info:
test_resources:
- type: s3
path: s3://openproblems-data/resources_test/common
dest: resources_test/common
- type: s3
path: s3://openproblems-data/resources_test/openproblems
dest: resources_test/openproblems
1 change: 1 addition & 0 deletions common
Submodule common added at e64f47
30 changes: 30 additions & 0 deletions scripts/create_resources/task_results_v1.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/bash

# get the root of the directory
REPO_ROOT=$(git rev-parse --show-toplevel)

# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"

OUT_DIR="resources_test/openproblems/task_results_v1"

set -e

echo ">> Removing previously generated results"
if [ -d "$OUT_DIR" ]; then
rm -r "$OUT_DIR"
fi

echo ">> Fetch results in v1 format"
mkdir -p "$OUT_DIR/data/"
TMPDIR=$(mktemp -d)

wget https://github.com/openproblems-bio/website/archive/refs/tags/v2.3.6.zip -O "$TMPDIR/website-v2.3.6.zip"
unzip "$TMPDIR/website-v2.3.6.zip" -d "$TMPDIR"
cp -r "$TMPDIR/website-2.3.6/results/batch_integration_embed/data/" "$OUT_DIR/processed"

echo ">> Uploading results to S3"
aws s3 sync --profile op \
"resources_test/openproblems/task_results_v1/" \
"s3://openproblems-data/resources_test/openproblems/task_results_v1/" \
--delete --dryrun
Loading
Loading