From 24de9aef04c950346ed87b086faf032c01141392 Mon Sep 17 00:00:00 2001 From: Javier Garcia Ordonez Date: Wed, 28 Aug 2024 18:34:05 +0200 Subject: [PATCH] initial commit --- .cookiecutterrc | 41 +++++++++++ .dockerignore | 47 ++++++++++++ .gitignore | 27 +++++++ .osparc/docker-compose.overwrite.yml | 6 ++ .osparc/metadata.yml | 31 ++++++++ .osparc/runtime.yml | 9 +++ Makefile | 105 +++++++++++++++++++++++++++ README.md | 30 ++++++++ docker-compose.yml | 32 ++++++++ docker/ubuntu/Dockerfile | 105 +++++++++++++++++++++++++++ docker/ubuntu/entrypoint.sh | 73 +++++++++++++++++++ service.cli/execute.sh | 33 +++++++++ src/dummy/.gitkeep | 1 + validation/input/inputs.json | 3 + validation/output/outputs.json | 3 + 15 files changed, 546 insertions(+) create mode 100644 .cookiecutterrc create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .osparc/docker-compose.overwrite.yml create mode 100644 .osparc/metadata.yml create mode 100644 .osparc/runtime.yml create mode 100644 Makefile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 docker/ubuntu/Dockerfile create mode 100755 docker/ubuntu/entrypoint.sh create mode 100755 service.cli/execute.sh create mode 100644 src/dummy/.gitkeep create mode 100644 validation/input/inputs.json create mode 100644 validation/output/outputs.json diff --git a/.cookiecutterrc b/.cookiecutterrc new file mode 100644 index 0000000..63098cb --- /dev/null +++ b/.cookiecutterrc @@ -0,0 +1,41 @@ +# This file exists so you can easily regenerate your project. +# +# `cookiepatcher` is a convenient shim around `cookiecutter` +# for regenerating projects (it will generate a .cookiecutterrc +# automatically for any template). To use it: +# +# pip install cookiepatcher +# cookiepatcher gh:itisfoundation/cookiecutter-osparc-service project-path +# +# See: +# https://pypi.python.org/pypi/cookiepatcher +# +# Alternatively, you can run: +# +# cookiecutter --overwrite-if-exists --config-file=project-path/.cookiecutterrc gh:itisfoundation/cookiecutter-osparc-service +# + +default_context: + + _checkout: None + _output_dir: '/home/ordonez/osparc_projects/cookiecutter-osparc-service' + _repo_dir: '.' + _template: '.' + author_affiliation: 'sdfas' + author_email: 'asdf@email.com' + author_name: 'asdf' + contact_email: 'asdf@email.com' + default_docker_registry: 'itisfoundation' + docker_base: 'ubuntu:18.04' + git_repo: 'local' + git_username: 'Yourusername' + number_of_inputs: '1' + number_of_outputs: '1' + project_name: 'dummy' + project_package_name: 'dummy' + project_short_description: 'dummy' + project_slug: 'dummy' + project_type: 'computational' + release_date: '2024' + version: '0.1.0' + version_display: '0.1.0' diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..fbeeae1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,47 @@ +## Common.dockerignore + +* +!src/ +!service.cli/ +!docker/ +!.osparc/ + +# Common +README.md +CHANGELOG.md +docker-compose.yml +Dockerfile + +# git +.git +.gitattributes +.gitignore +.git* + +## Common.gitignore + +# output folders +build/ +output/ +out/ + +# temporary folders +tmp/ + +# explicit mark +*ignore* +.tmp* + +# vscode configuration +.vscode + +# make outputs +pytest_*.xml +.compose* + +# validation folder +!validation/**/* +# docker ignore +!.dockerignore +# git ignore +!.gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e692c51 --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +## Common.gitignore + +# output folders +build/ +output/ +out/ + +# temporary folders +tmp/ + +# explicit mark +*ignore* +.tmp* + +# vscode configuration +.vscode + +# make outputs +pytest_*.xml +.compose* + +# validation folder +!validation/**/* +# docker ignore +!.dockerignore +# git ignore +!.gitignore diff --git a/.osparc/docker-compose.overwrite.yml b/.osparc/docker-compose.overwrite.yml new file mode 100644 index 0000000..df0cfe9 --- /dev/null +++ b/.osparc/docker-compose.overwrite.yml @@ -0,0 +1,6 @@ +version: "3.7" +services: + dummy: + build: + dockerfile: docker/ubuntu/Dockerfile + target: production diff --git a/.osparc/metadata.yml b/.osparc/metadata.yml new file mode 100644 index 0000000..bf11236 --- /dev/null +++ b/.osparc/metadata.yml @@ -0,0 +1,31 @@ +name: dummy +key: simcore/services/comp/dummy +type: computational +integration-version: 2.0.0 +version: 0.1.0 +description: dummy +contact: asdf@email.com +thumbnail: https://github.com/ITISFoundation/osparc-assets/blob/cb43207b6be2f4311c93cd963538d5718b41a023/assets/default-thumbnail-cookiecutter-osparc-service.png?raw=true +authors: + - name: asdf + email: asdf@email.com + affiliation: sdfas +inputs: + input_1: + displayOrder: 1 + label: input_1_label + description: The input 1 description + type: string + defaultValue: some_value(optional) + fileToKeyMap: + somefilename.ext: input_1 + +outputs: + output_1: + displayOrder: 1 + label: output_1_label + description: The input 1 description + type: string + fileToKeyMap: + somefilename.ext: output_1 + diff --git a/.osparc/runtime.yml b/.osparc/runtime.yml new file mode 100644 index 0000000..c0d3d9f --- /dev/null +++ b/.osparc/runtime.yml @@ -0,0 +1,9 @@ +restart-policy: no-restart +settings: + - name: Resources + type: Resources + value: + Limits: + NanoCPUs: 1000000000 # 100% of CPU cycles on 1 CPU + MemoryBytes: 2147483648 # 2 Gigabytes + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..6525bde --- /dev/null +++ b/Makefile @@ -0,0 +1,105 @@ +# +# Author: asdf + +SHELL = /bin/sh +.DEFAULT_GOAL := help + +export VCS_URL := $(shell git config --get remote.origin.url 2> /dev/null || echo unversioned repo) +export VCS_REF := $(shell git rev-parse --short HEAD 2> /dev/null || echo unversioned repo) +export VCS_STATUS := $(if $(shell git status -s 2> /dev/null || echo unversioned repo),'modified/untracked','clean') +export BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ") + +export DOCKER_IMAGE_NAME ?= dummy +export DOCKER_IMAGE_TAG ?= 0.1.0 + +OSPARC_DIR:=$(CURDIR)/.osparc + +APP_NAME := dummy + +# Builds new service version ---------------------------------------------------------------------------- + +define _bumpversion + # upgrades as $(subst $(1),,$@) version, commits and tags + @docker run -it --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ + -u $(shell id -u):$(shell id -g) \ + itisfoundation/ci-service-integration-library:v1.0.4 \ + sh -c "cd /${DOCKER_IMAGE_NAME} && bump2version --verbose --list --config-file $(1) $(subst $(2),,$@)" +endef + +.PHONY: version-patch version-minor version-major +version-patch version-minor version-major: .bumpversion.cfg ## increases service's version + @make compose-spec + @$(call _bumpversion,$<,version-) + @make compose-spec + +.PHONY: compose-spec +compose-spec: ## runs ooil to assemble the docker-compose.yml file + @docker run --rm -v $(PWD):/${DOCKER_IMAGE_NAME} \ + -u $(shell id -u):$(shell id -g) \ + itisfoundation/ci-service-integration-library:v1.0.4 \ + sh -c "cd /${DOCKER_IMAGE_NAME} && ooil compose" + +build: | compose-spec ## build docker image + docker compose build + +# To test built service locally ------------------------------------------------------------------------- +.PHONY: run-local +run-local: ## runs image with local configuration + docker compose --file docker-compose-local.yml up + +.PHONY: publish-local +publish-local: ## push to local oSPARC to test integration. It requires the oSPARC platform running on your computer, you can find more information here: https://github.com/ITISFoundation/osparc-simcore/blob/master/README.md + docker tag simcore/services/dynamic/${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG} registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) + docker push registry:5000/simcore/services/dynamic/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG) + @curl registry:5000/v2/_catalog | jq + +.PHONY: help +help: ## this colorful help + @echo "Recipes for '$(notdir $(CURDIR))':" + @echo "" + @awk 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:space:]_-]+:.*?## / {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) + @echo "" + + +# COOKIECUTTER ----------------------------------------------------------------- + +.PHONY: replay +replay: .cookiecutterrc ## re-applies cookiecutter + # Replaying . ... + @cookiecutter --no-input --overwrite-if-exists \ + --config-file=$< \ + --output-dir="$(abspath $(CURDIR)/..)" \ + "." + + +.PHONY: info +info: ## general info + # env vars: version control + @echo " VCS_URL : $(VCS_URL)" + @echo " VCS_REF : $(VCS_REF)" + @echo " VCS_STATUS : $(VCS_STATUS)" + # env vars: docker + @echo " DOCKER_IMAGE_TAG : $(DOCKER_IMAGE_TAG)" + @echo " BUILD_DATE : $(BUILD_DATE)" + # exe: recommended dev tools + @echo ' git : $(shell git --version 2>/dev/null || echo not found)' + @echo ' make : $(shell make --version 2>&1 | head -n 1)' + @echo ' jq : $(shell jq --version 2>/dev/null || echo not found z)' + @echo ' awk : $(shell awk -W version 2>&1 | head -n 1 2>/dev/null || echo not found)' + @echo ' python : $(shell python3 --version 2>/dev/null || echo not found )' + @echo ' docker : $(shell docker --version)' + @echo ' docker buildx : $(shell docker buildx version)' + @echo ' docker compose : $(shell docker compose --version)' + +# MISC ----------------------------------------------------------------- + + +.PHONY: clean +git_clean_args = -dxf --exclude=.vscode/ + +clean: ## cleans all unversioned files in project and temp files create by this makefile + # Cleaning unversioned + @git clean -n $(git_clean_args) + @echo -n "Are you sure? [y/N] " && read ans && [ $${ans:-N} = y ] + @echo -n "$(shell whoami), are you REALLY sure? [y/N] " && read ans && [ $${ans:-N} = y ] + @git clean $(git_clean_args) diff --git a/README.md b/README.md new file mode 100644 index 0000000..3d38132 --- /dev/null +++ b/README.md @@ -0,0 +1,30 @@ +# dummy + +dummy + +## Usage + +```console +$ make help + +$ make build +$ make info-build +$ make tests +``` + +## Workflow + +1. The source code shall be copied to the [src](dummy/src/dummy) folder. +2. The [Dockerfile](dummy/src/Dockerfile) shall be modified to compile the source code. +3. The [.osparc](.osparc) is the configuration folder and source of truth for metadata: describes service info and expected inputs/outputs of the service. +4. The [execute](dummy/service.cli/execute) shell script shall be modified to run the service using the expected inputs and retrieve the expected outputs. +5. The test input/output shall be copied to [validation](dummy/validation). +6. The service docker image may be built and tested as ``make build tests`` (see usage above) +7. Optional: if your code requires specific CPU/RAM resources, edit [runtime.yml](.osparc/runtime.yml). In doubt, leave it as default. + +## Have an issue or question? +Please open an issue [in this repository](https://github.com/ITISFoundation/cookiecutter-osparc-service/issues/). +--- +

+Made with love at www.z43.swiss +

diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d970096 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3.7' +services: + dummy: + build: + context: ./ + dockerfile: docker/ubuntu/Dockerfile + labels: + io.simcore.name: '{"name": "dummy"}' + io.simcore.thumbnail: '{"thumbnail": "https://github.com/ITISFoundation/osparc-assets/blob/cb43207b6be2f4311c93cd963538d5718b41a023/assets/default-thumbnail-cookiecutter-osparc-service.png?raw=true"}' + io.simcore.description: '{"description": "dummy"}' + io.simcore.key: '{"key": "simcore/services/comp/dummy"}' + io.simcore.version: '{"version": "0.1.0"}' + io.simcore.integration-version: '{"integration-version": "2.0.0"}' + io.simcore.type: '{"type": "computational"}' + io.simcore.authors: '{"authors": [{"name": "asdf", "email": "asdf@email.com", + "affiliation": "sdfas"}]}' + io.simcore.contact: '{"contact": "asdf@email.com"}' + io.simcore.inputs: '{"inputs": {"input_1": {"displayOrder": 1.0, "label": + "input_1_label", "description": "The input 1 description", "type": "string", + "fileToKeyMap": {"somefilename.ext": "input_1"}, "defaultValue": "some_value(optional)"}}}' + io.simcore.outputs: '{"outputs": {"output_1": {"displayOrder": 1.0, "label": + "output_1_label", "description": "The input 1 description", "type": "string", + "fileToKeyMap": {"somefilename.ext": "output_1"}}}}' + org.label-schema.build-date: '2024-08-28T16:31:41Z' + org.label-schema.schema-version: '1.0' + org.label-schema.vcs-ref: '' + org.label-schema.vcs-url: '' + simcore.service.restart-policy: no-restart + simcore.service.settings: '[{"name": "Resources", "type": "Resources", "value": + {"Limits": {"NanoCPUs": 1000000000, "MemoryBytes": 2147483648}}}]' + target: production + image: simcore/services/comp/dummy:0.1.0 diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile new file mode 100644 index 0000000..76a5517 --- /dev/null +++ b/docker/ubuntu/Dockerfile @@ -0,0 +1,105 @@ +FROM ubuntu:18.04 as base +# +# USAGE: +# cd services/dummy +# docker build -f Dockerfile -t dummy:prod --target production ../../ +# docker run dummy:prod +# + +LABEL maintainer=Yourusername + +# simcore-user uid=8004(${SC_USER_NAME}) gid=8004(${SC_USER_NAME}) groups=8004(${SC_USER_NAME}) +ENV SC_USER_ID 8004 +ENV SC_USER_NAME scu +RUN adduser --uid ${SC_USER_ID} --disabled-password --gecos "" --shell /bin/sh --home /home/${SC_USER_NAME} ${SC_USER_NAME} + +RUN apt-get update \ + && apt-get -y install --no-install-recommends \ + jq \ + && rm --recursive --force /var/lib/apt/lists/* + +# -------------------------- Build stage ------------------- +# Installs build/package management tools and third party dependencies +# +# + /build WORKDIR +# + +FROM base as build + +ENV SC_BUILD_TARGET build + +# ------------------------------------------------------------------------------------ +#TODO: +# uncomment and adapt if build dependencies shall be installed +#RUN apt-get update \ +# && apt-get -y install --no-install-recommends \ +# gcc \ +# git \ +# && rm -rf /var/lib/apt/lists/* + +# uncomment and adapt if python necessary +#RUN $SC_PIP install --upgrade pip wheel setuptools +# ------------------------------------------------------------------------------------ + +WORKDIR /build +# defines the output of the build +RUN mkdir --parents /build/bin +# copy src code +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} src/dummy src/dummy +# ------------------------------------------------------------------------------------ +#TODO: +# uncomment and adapt if build steps are necessary +# RUN cp -R src/dummy/* /build/bin +# ------------------------------------------------------------------------------------ + +# --------------------------Production stage ------------------- +# Final cleanup up to reduce image size and startup setup +# Runs as ${SC_USER_NAME} (non-root user) +# +# + /home/${SC_USER_NAME} $HOME = WORKDIR +# + dummy [${SC_USER_NAME}:${SC_USER_NAME}] +# + docker [${SC_USER_NAME}:${SC_USER_NAME}] +# + service.cli [${SC_USER_NAME}:${SC_USER_NAME}] +# +FROM base as production + +ENV SC_BUILD_TARGET production +ENV SC_BOOT_MODE production + + +ENV INPUT_FOLDER="/input" \ + OUTPUT_FOLDER="/output" + + +WORKDIR /home/${SC_USER_NAME} + +# ------------------------------------------------------------------------------------ +#TODO: +# uncomment and adapt to install runtime dependencies +#RUN apt-get update \ +# && apt-get -y install --no-install-recommends \ +# && rm -rf /var/lib/apt/lists/* +# ------------------------------------------------------------------------------------ + +# copy docker bootup scripts +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} docker/ubuntu/*.sh docker/ +# copy simcore service cli +COPY --chown=${SC_USER_NAME}:${SC_USER_NAME} service.cli/ service.cli/ +# necessary to be able to call run directly without sh in front +ENV PATH="/home/${SC_USER_NAME}/service.cli:${PATH}" + +# copy binaries from build +COPY --from=build --chown=${SC_USER_NAME}:${SC_USER_NAME} /build/bin dummy + +# ------------------------------------------------------------------------------------ +#TODO: +# uncomment and provide a healtchecker if possible +# HEALTHCHECK --interval=30s \ +# --timeout=120s \ +# --start-period=30s \ +# --retries=3 \ +# CMD ["healthchecker app"] +# ------------------------------------------------------------------------------------ + +ENTRYPOINT [ "/bin/sh", "docker/entrypoint.sh", "/bin/sh", "-c" ] +CMD ["run"] diff --git a/docker/ubuntu/entrypoint.sh b/docker/ubuntu/entrypoint.sh new file mode 100755 index 0000000..b29fd56 --- /dev/null +++ b/docker/ubuntu/entrypoint.sh @@ -0,0 +1,73 @@ +#!/bin/sh +set -o errexit +set -o nounset + +IFS=$(printf '\n\t') +# This entrypoint script: +# +# - Executes *inside* of the container upon start as --user [default root] +# - Notice that the container *starts* as --user [default root] but +# *runs* as non-root user [$SC_USER_NAME] +# +echo Entrypoint for stage "${SC_BUILD_TARGET}" ... +echo User : "$(id "$(whoami)")" +echo Workdir : "$(pwd)" + + +# expect input/output folders to be mounted +stat "${INPUT_FOLDER}" > /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${INPUT_FOLDER}' to deduce user and group ids" && exit 1) +stat "${OUTPUT_FOLDER}" > /dev/null 2>&1 || \ + (echo "ERROR: You must mount '${OUTPUT_FOLDER}' to deduce user and group ids" && exit 1) + +# NOTE: expects docker run ... -v /path/to/input/folder:${INPUT_FOLDER} +# check input/output folders are owned by the same user +if [ "$(stat -c %u "${INPUT_FOLDER}")" -ne "$(stat -c %u "${OUTPUT_FOLDER}")" ] +then + echo "ERROR: '${INPUT_FOLDER}' and '${OUTPUT_FOLDER}' have different user id's. not allowed" && exit 1 +fi +# check input/outputfolders are owned by the same group +if [ "$(stat -c %g "${INPUT_FOLDER}")" -ne "$(stat -c %g "${OUTPUT_FOLDER}")" ] +then + echo "ERROR: '${INPUT_FOLDER}' and '${OUTPUT_FOLDER}' have different group id's. not allowed" && exit 1 +fi + +echo "setting correct user id/group id..." +HOST_USERID=$(stat --format=%u "${INPUT_FOLDER}") +HOST_GROUPID=$(stat --format=%g "${INPUT_FOLDER}") +CONT_GROUPNAME=$(getent group "${HOST_GROUPID}" | cut --delimiter=: --fields=1) +if [ "$HOST_USERID" -eq 0 ] +then + echo "Warning: Folder mounted owned by root user... adding $SC_USER_NAME to root..." + adduser "$SC_USER_NAME" root +else + echo "Folder mounted owned by user $HOST_USERID:$HOST_GROUPID-'$CONT_GROUPNAME'..." + # take host's credentials in $SC_USER_NAME + if [ -z "$CONT_GROUPNAME" ] + then + echo "Creating new group my$SC_USER_NAME" + CONT_GROUPNAME=my$SC_USER_NAME + addgroup --gid "$HOST_GROUPID" "$CONT_GROUPNAME" + else + echo "group already exists" + fi + echo "adding $SC_USER_NAME to group $CONT_GROUPNAME..." + adduser "$SC_USER_NAME" "$CONT_GROUPNAME" + + echo "changing $SC_USER_NAME:$SC_USER_NAME ($SC_USER_ID:$SC_USER_ID) to $SC_USER_NAME:$CONT_GROUPNAME ($HOST_USERID:$HOST_GROUPID)" + usermod --uid "$HOST_USERID" --gid "$HOST_GROUPID" "$SC_USER_NAME" + + echo "Changing group properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / \( -path /proc -o -path /sys \) -prune -o -group "$SC_USER_ID" -exec chgrp --no-dereference "$CONT_GROUPNAME" {} \; + # change user property of files already around + echo "Changing ownership properties of files around from $SC_USER_ID to group $CONT_GROUPNAME" + find / \( -path /proc -o -path /sys \) -prune -o -user "$SC_USER_ID" -exec chown --no-dereference "$SC_USER_NAME" {} \; +fi + +echo "Starting $* ..." +echo " $SC_USER_NAME rights : $(id "$SC_USER_NAME")" +echo " local dir : $(ls -al)" +echo " input dir : $(ls -al "${INPUT_FOLDER}")" +echo " output dir : $(ls -al "${OUTPUT_FOLDER}")" + +su --command "export PATH=${PATH}:/home/$SC_USER_NAME/service.cli; $*" "$SC_USER_NAME" \ No newline at end of file diff --git a/service.cli/execute.sh b/service.cli/execute.sh new file mode 100755 index 0000000..14bfe32 --- /dev/null +++ b/service.cli/execute.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# set sh strict mode +set -o errexit +set -o nounset +IFS=$(printf '\n\t') + +cd /home/scu/dummy + +echo "starting service as" +echo User : "$(id "$(whoami)")" +echo Workdir : "$(pwd)" +echo "..." +echo +# ---------------------------------------------------------------- +# This script shall be modified according to the needs in order to run the service +# The inputs defined in ${INPUT_FOLDER}/inputs.json are available as env variables by their key in capital letters +# For example: input_1 -> $INPUT_1 + +# put the code to execute the service here +# For example: +env +ls -al "${INPUT_FOLDER}" + +# then retrieve the output and move it to the $OUTPUT_FOLDER +# as defined in the output labels +# For example: cp output.csv $OUTPUT_FOLDER or to $OUTPUT_FOLDER/outputs.json using jq +#TODO: Replace following +cat > "${OUTPUT_FOLDER}"/outputs.json << EOF +{ + "output_1":"some_stuff" +} +EOF + diff --git a/src/dummy/.gitkeep b/src/dummy/.gitkeep new file mode 100644 index 0000000..788d55e --- /dev/null +++ b/src/dummy/.gitkeep @@ -0,0 +1 @@ +add source code here diff --git a/validation/input/inputs.json b/validation/input/inputs.json new file mode 100644 index 0000000..c8c64f7 --- /dev/null +++ b/validation/input/inputs.json @@ -0,0 +1,3 @@ +{ + "input_1": "some_stuff" +} diff --git a/validation/output/outputs.json b/validation/output/outputs.json new file mode 100644 index 0000000..a280725 --- /dev/null +++ b/validation/output/outputs.json @@ -0,0 +1,3 @@ +{ + "output_1": "some_output_stuff" +}