Skip to content

Commit

Permalink
ODK release 2024-04-09
Browse files Browse the repository at this point in the history
  • Loading branch information
@chris-grove
chris-grove committed Apr 9, 2024
1 parent fb7600c commit 10fedcd
Show file tree
Hide file tree
Showing 24 changed files with 14,327 additions and 2,127 deletions.
17 changes: 9 additions & 8 deletions .github/workflows/qc.yml
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
# This is a basic workflow to help you get started with Actions
# Basic ODK workflow

name: CI

# Controls when the action will run.
on:
# Triggers the workflow on push or pull request events but only for the master branch
# Triggers the workflow on push or pull request events but only for the main branch
push:
branches: [ master ]
branches: [ main ]
pull_request:
branches: [ master ]
branches: [ main ]

# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
Expand All @@ -19,14 +19,15 @@ jobs:
ontology_qc:
# The type of runner that the job will run on
runs-on: ubuntu-latest
container: obolibrary/odkfull:v1.3.1
container: obolibrary/odkfull:v1.5

# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/checkout@v3

- name: Run ontology QC checks
env:
DEFAULT_BRANCH: master
run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false
DEFAULT_BRANCH: main
run: cd src/ontology && make ROBOT_ENV='ROBOT_JAVA_ARGS=-Xmx6G' test IMP=false PAT=false MIR=false

119 changes: 93 additions & 26 deletions src/ontology/Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
# ----------------------------------------
# Makefile for wbls
# Generated using ontology-development-kit
# ODK Version: v1.4.3
# ODK Version: v1.5
# ----------------------------------------
# IMPORTANT: DO NOT EDIT THIS FILE. To override default make goals, use wbls.Makefile instead


# ----------------------------------------
# More information: https://github.com/INCATools/ontology-development-kit/

# Fingerprint of the configuration file when this Makefile was last generated
CONFIG_HASH= c7a3686200f67d225e72eee2ff274fd117815e4ef712c2c40f3bf9b81db8197b


# ----------------------------------------
# Standard Constants
Expand Down Expand Up @@ -41,9 +44,9 @@ REPORT_FAIL_ON = None
REPORT_LABEL = -l true
REPORT_PROFILE_OPTS =
OBO_FORMAT_OPTIONS =
SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri multiple-replaced_by
SPARQL_EXPORTS = basic-report class-count-by-prefix edges xrefs obsoletes synonyms
ODK_VERSION_MAKEFILE = v1.4.3
SPARQL_VALIDATION_CHECKS = owldef-self-reference iri-range label-with-iri multiple-replaced_by dc-properties
SPARQL_EXPORTS = basic-report edges xrefs obsoletes synonyms
ODK_VERSION_MAKEFILE = v1.5

TODAY ?= $(shell date +%Y-%m-%d)
OBODATE ?= $(shell date +'%d:%m:%Y %H:%M')
Expand All @@ -58,6 +61,11 @@ FORMATS = $(sort owl obo json owl)
FORMATS_INCL_TSV = $(sort $(FORMATS) tsv)
RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full $(ONT)-simple )

ifeq ($(ODK_DEBUG),yes)
ODK_DEBUG_FILE = debug.log
SHELL = $(SCRIPTSDIR)/run-command.sh
endif

# ----------------------------------------
# Top-level targets
# ----------------------------------------
Expand All @@ -68,12 +76,16 @@ RELEASE_ARTEFACTS = $(sort $(ONT)-base $(ONT)-full $(ONT)-simple )
all: all_odk

.PHONY: all_odk
all_odk: odkversion test all_assets
all_odk: odkversion config_check test custom_reports all_assets

.PHONY: test
test: odkversion reason_test sparql_test robot_reports $(REPORTDIR)/validate_profile_owl2dl_$(ONT).owl.txt
echo "Finished running all tests successfully."

.PHONY: test
test_fast:
$(MAKE_FAST) test

.PHONY: release_diff
release_diff: $(REPORTDIR)/release-diff.md

Expand All @@ -84,13 +96,52 @@ reason_test: $(EDIT_PREPROCESSED)

.PHONY: odkversion
odkversion:
echo "ODK Makefile version: $(ODK_VERSION_MAKEFILE) (this is the version of the ODK with which this Makefile was generated, \
not the version of the ODK you are running)" &&\
echo "ROBOT version (ODK): " && $(ROBOT) --version
@echo "ODK Makefile $(ODK_VERSION_MAKEFILE)"
@odk-info --tools
.PHONY: config_check
config_check:
@if [ "$$(sha256sum $(ONT)-odk.yaml | cut -c1-64)" = "$(CONFIG_HASH)" ]; then \
echo "Repository is up-to-date." ; else \
echo "Your ODK configuration has changed since this Makefile was generated. You may need to run 'make update_repo'." ; fi


$(TMPDIR) $(REPORTDIR) $(MIRRORDIR) $(IMPORTDIR) $(COMPONENTSDIR) $(SUBSETDIR):
mkdir -p $@

# ----------------------------------------
# ODK-managed ROBOT plugins
# ----------------------------------------

# Make sure ROBOT knows where to find plugins
export ROBOT_PLUGINS_DIRECTORY=$(TMPDIR)/plugins

# Override this rule in wbls.Makefile to install custom plugins
.PHONY: custom_robot_plugins
custom_robot_plugins:


.PHONY: extra_robot_plugins
extra_robot_plugins:


# Install all ROBOT plugins to the runtime plugins directory
.PHONY: all_robot_plugins
all_robot_plugins: $(foreach plugin,$(notdir $(wildcard /tools/robot-plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
$(foreach plugin,$(notdir $(wildcard ../../plugins/*.jar)),$(ROBOT_PLUGINS_DIRECTORY)/$(plugin)) \
custom_robot_plugins extra_robot_plugins \

# Default rule to install plugins
$(ROBOT_PLUGINS_DIRECTORY)/%.jar:
@mkdir -p $(ROBOT_PLUGINS_DIRECTORY)
@if [ -f ../../plugins/$*.jar ]; then \
ln ../../plugins/$*.jar $@ ; \
elif [ -f /tools/robot-plugins/$*.jar ]; then \
cp /tools/robot-plugins/$*.jar $@ ; \
fi

# Specific rules for supplementary plugins defined in configuration


# ----------------------------------------
# Release assets
# ----------------------------------------
Expand Down Expand Up @@ -180,10 +231,10 @@ validate_profile_%: $(REPORTDIR)/validate_profile_owl2dl_%.txt

SPARQL_VALIDATION_QUERIES = $(foreach V,$(SPARQL_VALIDATION_CHECKS),$(SPARQLDIR)/$(V)-violation.sparql)

sparql_test: $(EDIT_PREPROCESSED) catalog-v001.xml | $(REPORTDIR)
sparql_test: $(EDIT_PREPROCESSED) | $(REPORTDIR)
ifneq ($(SPARQL_VALIDATION_QUERIES),)

$(ROBOT) verify --catalog catalog-v001.xml -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
$(ROBOT) verify -i $(EDIT_PREPROCESSED) --queries $(SPARQL_VALIDATION_QUERIES) -O $(REPORTDIR)
endif

# ----------------------------------------
Expand All @@ -196,6 +247,10 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR)
$(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR)
$(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) --base-iri http://purl.obolibrary.org/obo/WBls_ --print 5 -o $@

check_for_robot_updates:
echo "You are not using a custom profile, so you are getting the joy of the latest ROBOT report!"


# ----------------------------------------
# Release assets
# ----------------------------------------
Expand All @@ -212,15 +267,15 @@ RELEASE_ASSETS = \
$(SUBSET_FILES)

.PHONY: all_assets
all_assets: $(ASSETS)
all_assets: $(ASSETS) check_rdfxml_assets

.PHONY: show_assets
show_assets:
echo $(ASSETS)
du -sh $(ASSETS)

check_rdfxml_%: %
@check-rdfxml $<
@check-rdfxml $<

.PHONY: check_rdfxml_assets
check_rdfxml_assets: $(foreach product,$(MAIN_PRODUCTS),check_rdfxml_$(product).owl)
Expand Down Expand Up @@ -345,21 +400,25 @@ IMP=true # Global parameter to bypass import generation
MIR=true # Global parameter to bypass mirror generation
IMP_LARGE=true # Global parameter to bypass handling of large imports

ifeq ($(strip $(MIR)),true)


## ONTOLOGY: ro
.PHONY: mirror-ro
.PRECIOUS: $(MIRRORDIR)/ro.owl
mirror-ro: | $(TMPDIR)
if [ $(MIR) = true ] && [ $(IMP) = true ]; then curl -L $(OBOBASE)/ro.owl --create-dirs -o $(MIRRORDIR)/ro.owl --retry 4 --max-time 200 &&\
$(ROBOT) convert -i $(MIRRORDIR)/ro.owl -o $@.tmp.owl &&\
mv $@.tmp.owl $(TMPDIR)/$@.owl; fi
curl -L $(OBOBASE)/ro.owl --create-dirs -o $(TMPDIR)/ro-download.owl --retry 4 --max-time 200 && \
$(ROBOT) convert -i $(TMPDIR)/ro-download.owl -o $(TMPDIR)/$@.owl


$(MIRRORDIR)/%.owl: mirror-% | $(MIRRORDIR)
if [ $(IMP) = true ] && [ $(MIR) = true ] && [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
if [ -f $(TMPDIR)/mirror-$*.owl ]; then if cmp -s $(TMPDIR)/mirror-$*.owl $@ ; then echo "Mirror identical, ignoring."; else echo "Mirrors different, updating." &&\
cp $(TMPDIR)/mirror-$*.owl $@; fi; fi

else # MIR=false
$(MIRRORDIR)/%.owl:
@echo "Not refreshing $@ because the mirrorring pipeline is disabled (MIR=$(MIR))."
endif



Expand Down Expand Up @@ -448,9 +507,13 @@ ROBOT_RELEASE_IMPORT_MODE=$(ROBOT) merge --input $<
# ROBOT pipeline that removes imports, then merges components. This is for release artefacts that start from "base"
ROBOT_RELEASE_IMPORT_MODE_BASE=$(ROBOT) remove --input $< --select imports --trim false merge $(patsubst %, -i %, $(OTHER_SRC))

# base: All the axioms as they are editted by the editors, excluding reasoning
$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC)
$(ROBOT_RELEASE_IMPORT_MODE_BASE) \
# base: A version of the ontology that does not include any externally imported axioms.
$(ONT)-base.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES)
$(ROBOT_RELEASE_IMPORT_MODE) \
reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural --annotate-inferred-axioms False \
relax \
reduce -r ELK \
remove --base-iri http://purl.obolibrary.org/obo/WBls_ --axioms external --preserve-structure false --trim false \
$(SHARED_ROBOT_COMMANDS) \
annotate --link-annotation http://purl.org/dc/elements/1.1/type http://purl.obolibrary.org/obo/IAO_8000001 \
--ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) \
Expand All @@ -467,7 +530,7 @@ $(ONT)-full.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES)
# remove --select imports --trim false
$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES)
$(ROBOT_RELEASE_IMPORT_MODE) \
reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural \
reason --reasoner ELK --equivalent-classes-allowed asserted-only --exclude-tautologies structural --annotate-inferred-axioms False \
relax \
remove --axioms equivalent \
relax \
Expand Down Expand Up @@ -528,13 +591,16 @@ update_repo:



# Note to future generations: prepending ./ is a safety measure to ensure that
# the environment does not malicously set `CLEANFILES` to `\`.
# Note to future generations: computing the real path relative to the
# current directory is a way to ensure we only clean up directories that
# are located below the current directory, regardless of the contents of
# the *DIR variables.
.PHONY: clean
clean:
[ -n "$(MIRRORDIR)" ] && [ $(MIRRORDIR) != "." ] && [ $(MIRRORDIR) != "/" ] && [ $(MIRRORDIR) != ".." ] && [ -d ./$(MIRRORDIR) ] && rm -rf ./$(MIRRORDIR)/*
[ -n "$(TMPDIR)" ] && [ $(TMPDIR) != "." ] && [ $(TMPDIR) != "/" ] && [ $(TMPDIR) != ".." ] && [ -d ./$(TMPDIR) ] && rm -rf ./$(TMPDIR)/*
[ -n "$(UPDATEREPODIR)" ] && [ $(UPDATEREPODIR) != "." ] && [ $(UPDATEREPODIR) != "/" ] && [ $(UPDATEREPODIR) != ".." ] && [ -d ./$(UPDATEREPODIR) ] && rm -rf ./$(UPDATEREPODIR)/*
for dir in $(MIRRORDIR) $(TMPDIR) $(UPDATEREPODIR) ; do \
reldir=$$(realpath --relative-to=$$(pwd) $$dir) ; \
case $$reldir in .*|"") ;; *) rm -rf $$reldir/* ;; esac \
done
rm -f $(CLEANFILES)

.PHONY: help
Expand All @@ -553,6 +619,7 @@ Core commands:
* prepare_release_fast: Run the entire release pipeline without refreshing imports, recreating components or recompiling patterns.
* update_repo: Update the ODK repository setup using the config file wbls-odk.yaml
* test: Running all validation tests
* test_fast: Runs the test suite, but without updating imports or components
* odkversion: Show the current version of the ODK Makefile and ROBOT.
* clean: Delete all temporary files
* help: Print ODK Usage information
Expand All @@ -570,7 +637,7 @@ Imports management:

Editor utilities:
* validate_idranges: Make sure your ID ranges file is formatted correctly
* normalize_src: Load and safe your wbls-edit file after you to make sure its serialised correctly
* normalize_src: Load and save your wbls-edit file after you to make sure its serialised correctly
* explain_unsat: If you have unsatisfiable classes, this command will create a markdown file (tmp/explain_unsat.md) which will explain all your unsatisfiable classes
* validate-all-tsv: Check all your tsv files for possible problems in syntax. Use ALL_TSV_FILES variable to list files
* validate-tsv: Check a tsv file for syntactic problems with tsvalid. Use TSV variable to pass filepath, e.g. make TSV=../my.tsv validate-tsv.
Expand Down
Loading

0 comments on commit 10fedcd

Please sign in to comment.