Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updating ODK to 1.5 #244

Merged
merged 1 commit into from
Apr 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/qc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ jobs:
ontology_qc:
# The type of runner that the job will run on
runs-on: ubuntu-latest
container: obolibrary/odkfull:v1.4
container: obolibrary/odkfull:v1.5

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
Expand All @@ -29,5 +29,5 @@ jobs:
- name: Run ontology QC checks
env:
DEFAULT_BRANCH: main
run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false
run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false MIR=false

1 change: 1 addition & 0 deletions docs/odk-workflows/ManageDocumentation.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,4 @@ The documentation is _not_ automatically updated from the Markdown, and needs to
Just make sure you give GitHub 2-5 minutes to build the pages!



2 changes: 1 addition & 1 deletion docs/odk-workflows/RepoManagement.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

## Updating your ODK repository

Your ODK repositories configuration is managed in `src/ontology/apollo_sv-odk.yaml`. Once you have made your changes, you can run the following to apply your changes to the repository:
Your ODK repositories configuration is managed in `src/ontology/apollo_sv-odk.yaml`. The [ODK Project Configuration Schema](https://github.com/INCATools/ontology-development-kit/blob/master/docs/project-schema.md) defines all possible parameters that can be used in this config YAML. Once you have made your changes, you can run the following to apply your changes to the repository:


```
Expand Down
131 changes: 97 additions & 34 deletions src/ontology/Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# ----------------------------------------
# Makefile for apollo_sv
# Generated using ontology-development-kit
# ODK Version: v1.4.3
# ODK Version: v1.5
# ----------------------------------------
# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use apollo_sv.Makefile instead

Expand All @@ -15,6 +15,9 @@
# 2. [Refresh imports](all_imports)
# 3. [Update repo to latest ODK](update_repo)

# Fingerprint of the configuration file when this Makefile was last generated
CONFIG_HASH= 8c83508f90aec6eebe48cba22814c2547bdf197ecf902ac8f55664f6ae708683


# ----------------------------------------
# Standard Constants
Expand Down Expand Up @@ -49,7 +52,7 @@ REPORT_PROFILE_OPTS =
OBO_FORMAT_OPTIONS =
SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range
SPARQL_EXPORTS = basic-report
ODK_VERSION_MAKEFILE = v1.4.3
ODK_VERSION_MAKEFILE = v1.5

TODAY ?= $(shell date +%Y-%m-%d)
OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
Expand All @@ -64,6 +67,11 @@ FORMATS = $(sort owl obo json owl)
FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full )

ifeq ($(ODK_DEBUG),yes)
ODK_DEBUG_FILE = debug.log
SHELL = $(SCRIPTSDIR)/run-command.sh
endif

# ----------------------------------------
# Top-level targets
# ----------------------------------------
Expand All @@ -74,12 +82,16 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full )
all: all_odk

.PHONY: all_odk
all_odk: odkversion test all_assets
all_odk: odkversion config_check test custom_reports all_assets

.PHONY: test
test: odkversion reason_test sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
echo "Finished running all tests successfully."

.PHONY: test
test_fast:
$(MAKE_FAST) test

.PHONY: release_diff
release_diff: $(REPORTDIR)/release-diff.md

Expand All @@ -90,13 +102,52 @@ reason_test: $(EDIT_PREPROCESSED)

.PHONY: odkversion
odkversion:
echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
not the version of the ODK you are running)" &&\
echo "ROBOT version (ODK): " && $(ROBOT) --version
@echo "ODK Makefile $(ODK_VERSION_MAKEFILE)"
@odk-info --tools
.PHONY: config_check
config_check:
@if [ "$$(sha256sum $(ONT)-odk.yaml | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \
echo "Repository is up-to-date." ; else \
echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'make update_repo'." ; fi


$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
mkdir -p $@

# ----------------------------------------
# ODK-managed ROBOT plugins
# ----------------------------------------

# Make sure ROBOT knows where to find plugins
export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins

# Override this rule in apollo_sv.Makefile to install custom plugins
.PHONY: custom_robot_plugins
custom_robot_plugins:


.PHONY: extra_robot_plugins
extra_robot_plugins:


# Install all ROBOT plugins to the runtime plugins directory
.PHONY: all_robot_plugins
all_robot_plugins: $(foreach plugin,$(notdir $(wildcard /tools/robot-plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
$(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
custom_robot_plugins extra_robot_plugins \

# Default rule to install plugins
$(ROBOT_PLUGINS_DIRECTORY)/%.jar:
@mkdir -p $(ROBOT_PLUGINS_DIRECTORY)
@if [ -f ../../plugins/$*.jar ]; then \
ln ../../plugins/$*.jar $@ ; \
elif [ -f /tools/robot-plugins/$*.jar ]; then \
cp /tools/robot-plugins/$*.jar $@ ; \
fi

# Specific rules for supplementary plugins defined in configuration


# ----------------------------------------
# Release assets
# ----------------------------------------
Expand Down Expand Up @@ -186,12 +237,12 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt

SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql)

sparql_test: $(EDIT_PREPROCESSED) apollo_sv-base.owl catalog-v001.xml | $(REPORTDIR)
sparql_test: $(EDIT_PREPROCESSED) apollo_sv-base.owl | $(REPORTDIR)
ifneq ($(SPARQL_VALIDATION_QUERIES),)

$(ROBOT) verify --catalog catalog-v001.xml -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
$(ROBOT) verify -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)

$(ROBOT) verify --catalog catalog-v001.xml -i apollo_sv-base.owl --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
$(ROBOT) verify -i apollo_sv-base.owl --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
endif

# ----------------------------------------
Expand All @@ -204,6 +255,10 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri $(URIBASE)/APOLLO_SV_ --base-iri $(URIBASE)/apollo_sv --print 5 -o $@

check_for_robot_updates:
echo "You are not using a custom profile, so you are getting the joy of the latest ROBOT report!"


# ----------------------------------------
# Release assets
# ----------------------------------------
Expand All @@ -220,15 +275,15 @@ RELEASE_ASSETS = \
$(SUBSET_FILES)

.PHONY: all_assets
all_assets: $(ASSETS)
all_assets: $(ASSETS) check_rdfxml_assets

.PHONY: show_assets
show_assets:
echo $(ASSETS)
du -sh $(ASSETS)

check_rdfxml_%: %
@check-rdfxml $<
@check-rdfxml $<

.PHONY: check_rdfxml_assets
check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl)
Expand Down Expand Up @@ -397,55 +452,55 @@ IMP=true # Global parameter to bypass import generation
MIR=true # Global parameter to bypass mirror generation
IMP_LARGE=true # Global parameter to bypass handling of large imports

ifeq ($(strip $(MIR)),true)


## ONTOLOGY: ro
.PHONY: mirror-ro
.PRECIOUS: $(MIRRORDIR)/ro.owl
mirror-ro: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o [email protected] &&\
mv [email protected] $(TMPDIR)/[email protected]; fi
curl -L $(OBOBASE)/ro.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o $(TMPDIR)/[email protected]


## ONTOLOGY: pno
.PHONY: mirror-pno
.PRECIOUS: $(MIRRORDIR)/pno.owl
mirror-pno: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/iao/pno/dev/pno.owl -o [email protected] &&\
mv [email protected] $(TMPDIR)/[email protected]; fi
$(ROBOT) convert -I http://purl.obolibrary.org/obo/iao/pno/dev/pno.owl -o $(TMPDIR)/[email protected]


## ONTOLOGY: geo
.PHONY: mirror-geo
.PRECIOUS: $(MIRRORDIR)/geo.owl
mirror-geo: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then $(ROBOT) convert -I http://purl.obolibrary.org/obo/geo/dev/geo.owl -o [email protected] &&\
mv [email protected] $(TMPDIR)/[email protected]; fi
$(ROBOT) convert -I http://purl.obolibrary.org/obo/geo/dev/geo.owl -o $(TMPDIR)/[email protected]


## ONTOLOGY: ido
.PHONY: mirror-ido
.PRECIOUS: $(MIRRORDIR)/ido.owl
mirror-ido: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/ido.owl --create-dirs -o $(MIRRORDIR)/ido.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/ido.owl -o [email protected] &&\
mv [email protected] $(TMPDIR)/[email protected]; fi
curl -L $(OBOBASE)/ido.owl --create-dirs -o $(TMPDIR)/ido-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/ido-download.owl -o $(TMPDIR)/[email protected]


## ONTOLOGY: mf
.PHONY: mirror-mf
.PRECIOUS: $(MIRRORDIR)/mf.owl
mirror-mf: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/mf.owl --create-dirs -o $(MIRRORDIR)/mf.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/mf.owl -o [email protected] &&\
mv [email protected] $(TMPDIR)/[email protected]; fi
curl -L $(OBOBASE)/mf.owl --create-dirs -o $(TMPDIR)/mf-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/mf-download.owl -o $(TMPDIR)/[email protected]


$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
cp $(TMPDIR)/mirror-$*.owl $@; fi; fi

else # MIR=false
$(MIRRORDIR)/%.owl:
@echo "Not refreshing $@ because the mirrorring pipeline is disabled (MIR=$(MIR))."
endif



Expand Down Expand Up @@ -528,9 +583,13 @@ ROBOT_RELEASE_IMPORT_MODE=$(ROBOT) merge --input $<
# ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base"
ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false merge $(patsubst %, -i %, $(OTHER_SRC))

# base: All the axioms as they are editted by the editors, excluding reasoning
$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC)
$(ROBOT_RELEASE_IMPORT_MODE_BASE) \
# base: A version of the ontology that does not include any externally imported axioms.
$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES)
$(ROBOT_RELEASE_IMPORT_MODE) \
reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural --annotate-inferred-axioms False \
relax \
reduce -r ELK \
remove --base-iri $(URIBASE)/APOLLO_SV --axioms external --preserve-structure false --trim false \
$(SHARED_ROBOT_COMMANDS) \
annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
Expand Down Expand Up @@ -597,13 +656,16 @@ update_repo:
update_docs:
mkdocs gh-deploy --config-file ../../mkdocs.yaml

# Note to future generations: prepending ./ is a safety measure to ensure that
# the environment does not malicously set `CLEANFILES` to `\`.
# Note to future generations: computing the real path relative to the
# current directory is a way to ensure we only clean up directories that
# are located below the current directory, regardless of the contents of
# the *DIR variables.
.PHONY: clean
clean:
[ -n "$(MIRRORDIR)" ] && [ $(MIRRORDIR) != "." ] && [ $(MIRRORDIR) != "/" ] && [ $(MIRRORDIR) != ".." ] && [ -d ./$(MIRRORDIR) ] && rm -rf ./$(MIRRORDIR)/*
[ -n "$(TMPDIR)" ] && [ $(TMPDIR) != "." ] && [ $(TMPDIR) != "/" ] && [ $(TMPDIR) != ".." ] && [ -d ./$(TMPDIR) ] && rm -rf ./$(TMPDIR)/*
[ -n "$(UPDATEREPODIR)" ] && [ $(UPDATEREPODIR) != "." ] && [ $(UPDATEREPODIR) != "/" ] && [ $(UPDATEREPODIR) != ".." ] && [ -d ./$(UPDATEREPODIR) ] && rm -rf ./$(UPDATEREPODIR)/*
for dir in $(MIRRORDIR) $(TMPDIR) $(UPDATEREPODIR) ; do \
reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \
case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \
done
rm -f $(CLEANFILES)

.PHONY: help
Expand All @@ -622,6 +684,7 @@ Core commands:
* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns.
* update_repo: Update the ODK repository setup using the config file apollo_sv-odk.yaml
* test: Running all validation tests
* test_fast: Runs the test suite, but without updating imports or components
* odkversion: Show the current version of the ODK Makefile and ROBOT.
* clean: Delete all temporary files
* help: Print ODK Usage information
Expand All @@ -639,7 +702,7 @@ Imports management:

Editor utilities:
* validate_idranges: Make sure your ID ranges file is formatted correctly
* normalize_src: Load and safe your apollo_sv-edit file after you to make sure its serialised correctly
* normalize_src: Load and save your apollo_sv-edit file after you to make sure its serialised correctly
* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes
* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files
* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv.
Expand Down
38 changes: 34 additions & 4 deletions src/ontology/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
#
# See README-editors.md for more details.

set -e

if [ -f run.sh.conf ]; then
. ./run.sh.conf
fi
Expand All @@ -29,6 +31,22 @@ elif [ -f "$HOME/Library/Application Support/ontology-development-kit/github/tok
GH_TOKEN=$(cat "$HOME/Library/Application Support/ontology-development-kit/github/token")
fi

# SSH agent socket
# On macOS, we cannot use $SSH_AUTH_SOCK directly,
# we need to use a "magic" socket instead.
case "$(uname)" in
Darwin)
ODK_SSH_AUTH_SOCKET=/run/host-services/ssh-auth.sock
;;
*)
ODK_SSH_AUTH_SOCKET=$SSH_AUTH_SOCK
;;
esac
ODK_SSH_BIND=
if [ -n "$ODK_SSH_AUTH_SOCKET" ]; then
ODK_SSH_BIND=",$ODK_SSH_AUTH_SOCKET:/run/host-services/ssh-auth.sock"
fi

ODK_IMAGE=${ODK_IMAGE:-odkfull}
TAG_IN_IMAGE=$(echo $ODK_IMAGE | awk -F':' '{ print $2 }')
if [ -n "$TAG_IN_IMAGE" ]; then
Expand All @@ -40,16 +58,28 @@ ODK_TAG=${ODK_TAG:-latest}
ODK_JAVA_OPTS=${ODK_JAVA_OPTS:--Xmx8G}
ODK_DEBUG=${ODK_DEBUG:-no}

ODK_USER_ID=${ODK_USER_ID:-$(id -u)}
ODK_GROUP_ID=${ODK_GROUP_ID:-$(id -g)}

# Convert OWLAPI_* environment variables to the OWLAPI as Java options
# See http://owlcs.github.io/owlapi/apidocs_4/org/semanticweb/owlapi/model/parameters/ConfigurationOptions.html
# for a list of allowed options
OWLAPI_OPTIONS_NAMESPACE=org.semanticweb.owlapi.model.parameters.ConfigurationOptions
for owlapi_var in $(env | sed -n s/^OWLAPI_//p) ; do
ODK_JAVA_OPTS="$ODK_JAVA_OPTS -D$OWLAPI_OPTIONS_NAMESPACE.${owlapi_var%=*}=${owlapi_var#*=}"
done

TIMECMD=
if [ x$ODK_DEBUG = xyes ]; then
# If you wish to change the format string, take care of using
# non-breaking spaces (U+00A0) instead of normal spaces, to
# prevent the shell from tokenizing the format string.
echo "Running ${IMAGE} with ${ODK_JAVA_OPTS} of memory for ROBOT and Java-based pipeline steps."
echo "Running obolibrary/${ODK_IMAGE}:${ODK_TAG} with '${ODK_JAVA_OPTS}' as options for ROBOT and other Java-based pipeline steps."
TIMECMD="/usr/bin/time -f ### DEBUG STATS ###\nElapsed time: %E\nPeak memory: %M kb"
fi
rm -f tmp/debug.log

VOLUME_BIND=$PWD/../../:/work
VOLUME_BIND=$PWD/../../:/work$ODK_SSH_BIND
WORK_DIR=/work/src/ontology

if [ -n "$ODK_BINDS" ]; then
Expand All @@ -59,14 +89,14 @@ fi
if [ -n "$USE_SINGULARITY" ]; then

singularity exec --cleanenv $ODK_SINGULARITY_OPTIONS \
--env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS" \
--env "ROBOT_JAVA_ARGS=$ODK_JAVA_OPTS,JAVA_OPTS=$ODK_JAVA_OPTS,SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock,ODK_USER_ID=$ODK_USER_ID,ODK_GROUP_ID=$ODK_GROUP_ID,ODK_DEBUG=$ODK_DEBUG" \
--bind $VOLUME_BIND \
-W $WORK_DIR \
docker://obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
else
BIND_OPTIONS="-v $(echo $VOLUME_BIND | sed 's/,/ -v /')"
docker run $ODK_DOCKER_OPTIONS $BIND_OPTIONS -w $WORK_DIR \
-e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" \
-e ROBOT_JAVA_ARGS="$ODK_JAVA_OPTS" -e JAVA_OPTS="$ODK_JAVA_OPTS" -e SSH_AUTH_SOCK=/run/host-services/ssh-auth.sock -e ODK_USER_ID=$ODK_USER_ID -e ODK_GROUP_ID=$ODK_GROUP_ID -e ODK_DEBUG=$ODK_DEBUG \
--rm -ti obolibrary/$ODK_IMAGE:$ODK_TAG $TIMECMD "$@"
fi

Expand Down
Loading
Loading