Use LinkML to specify metadata model and generate supporting material

.github/workflows/deploy-docs.yaml

@@ -0,0 +1,33 @@
+---
+name: Auto-deployment of uk_cross_government_metadata_exchange_model Documentation
+on:
+  push:
+    branches: 
+      - main
+
+jobs:
+  build-docs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # otherwise, you will failed to push refs to dest repo
+
+      - name: Set up Python 3.
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install Poetry.
+        uses: snok/[email protected]
+
+      - name: Install dependencies.
+        run: poetry install -E docs
+
+      - name: Build documentation.
+        run: |
+          mkdir -p docs
+          touch docs/.nojekyll
+          make gendoc
+          make mkd-gh-deploy

.github/workflows/main.yaml

@@ -0,0 +1,37 @@
+# Built from:
+# https://docs.github.com/en/actions/guides/building-and-testing-python
+# https://github.com/snok/install-poetry#workflows-and-tips
+---
+name: Build and test uk_cross_government_metadata_exchange_model
+
+on: [pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+
+    steps:
+
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/[email protected]
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root
+
+      - name: Install project
+        run: poetry install --no-interaction
+
+      - name: Run test suite
+        run: make test

.gitignore

@@ -0,0 +1,4 @@
+/docs/
+/project/
+/venv/
+.install.stamp

Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.10
+# Copy configuration files
+COPY ./pyproject.toml ./mkdocs.yml /
+# Copy license file
+COPY ./LICENSE.md ./src/docs/ /docs/
+# Copy source files for model
+COPY ./src/ /src/
+RUN pip install poetry
+RUN poetry install
+RUN poetry run gen-doc -d /docs src/model/uk_cross_government_metadata_exchange_model.yaml
+EXPOSE 8080
+CMD poetry run mkdocs serve

LICENSE.md

@@ -0,0 +1,76 @@
+*Source: [http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/](http://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/)*
+
+# Open Government Licence v3.0 (OGL-UK-3.0)
+You are encouraged to use and re-use the Information that is available under this licence freely and flexibly, with only a few conditions.
+
+## Using Information under this licence
+
+Use of copyright and database right material expressly made available under this licence (the 'Information') indicates your acceptance of the terms and conditions below.
+
+The Licensor grants you a worldwide, royalty-free, perpetual, non-exclusive licence to use the Information subject to the conditions below.
+
+This licence does not affect your freedom under fair dealing or fair use or any other copyright or database right exceptions and limitations.
+
+**You are free to:**
+
+* copy, publish, distribute and transmit the Information;
+* adapt the Information;
+* exploit the Information commercially and non-commercially for example, by combining it with other Information, or by including it in your own product or application.
+
+**You must (where you do any of the above):**
+
+* acknowledge the source of the Information in your product or application by including or linking to any attribution statement specified by the Information Provider(s) and, where possible, provide a link to this licence;
+* If the Information Provider does not provide a specific attribution statement, you must use the following:
+* Contains public sector information licensed under the Open Government Licence v3.0.
+
+If you are using Information from several Information Providers and listing multiple attributions is not practical in your product or application, you may include a URI or hyperlink to a resource that contains the required attribution statements.
+
+These are important conditions of this licence and if you fail to comply with them the rights granted to you under this licence, or any similar licence granted by the Licensor, will end automatically.
+
+## Exemptions
+
+**This licence does not cover:**
+
+* personal data in the Information;
+* Information that has not been accessed by way of publication or disclosure under information access legislation (including the Freedom of Information Acts for the UK and Scotland) by or with the consent of the Information Provider;
+* departmental or public sector organisation logos, crests and the Royal Arms except where they form an integral part of a document or dataset;
+* military insignia;
+* third party rights the Information Provider is not authorised to license;
+* other intellectual property rights, including patents, trade marks, and design rights; and
+* identity documents such as the British Passport
+
+## Non-endorsement
+
+This licence does not grant you any right to use the Information in a way that suggests any official status or that the Information Provider and/or Licensor endorse you or your use of the Information.
+
+## No warranty
+
+The Information is licensed 'as is' and the Information Provider and/or Licensor excludes all representations, warranties, obligations and liabilities in relation to the Information to the maximum extent permitted by law.
+
+The Information Provider and/or Licensor are not liable for any errors or omissions in the Information and shall not be liable for any loss, injury or damage of any kind caused by its use. The Information Provider does not guarantee the continued supply of the Information.
+
+## Governing Law
+
+This licence is governed by the laws of the jurisdiction in which the Information Provider has its principal place of business, unless otherwise specified by the Information Provider.
+
+## Definitions
+
+In this licence, the terms below have the following meanings:
+
+* 'Information' means information protected by copyright or by database right (for example, literary and artistic works, content, data and source code) offered for use under the terms of this licence.
+* 'Information Provider' means the person or organisation providing the Information under this licence.
+* 'Licensor' means any Information Provider which has the authority to offer Information under the terms of this licence or the Keeper of Public Records, who has the authority to offer Information subject to Crown copyright and Crown database rights and Information subject to copyright and database right that has been assigned to or acquired by the Crown, under the terms of this licence.
+* 'Use' means doing any act which is restricted by copyright or database right, whether in the original medium or in any other medium, and includes without limitation distributing, copying, adapting, modifying as may be technically necessary to use it in a different mode or format.
+* 'You', 'you' and 'your' means the natural or legal person, or body of persons corporate or incorporate, acquiring rights in the Information (whether the Information is obtained directly from the Licensor or otherwise) under this licence.
+
+## About the Open Government Licence
+
+The National Archives has developed this licence as a tool to enable Information Providers in the public sector to license the use and re-use of their Information under a common open licence. The National Archives invites public sector bodies owning their own copyright and database rights to permit the use of their Information under this licence.
+
+The Keeper of the Public Records has authority to license Information subject to copyright and database right owned by the Crown. The extent of the offer to license this Information under the terms of this licence is set out in the UK Government Licensing Framework.
+
+This is version 3.0 of the Open Government Licence. The National Archives may, from time to time, issue new versions of the Open Government Licence. If you are already using Information under a previous version of the Open Government Licence, the terms of that licence will continue to apply.
+
+These terms are compatible with the Creative Commons Attribution License 4.0 and the Open Data Commons Attribution License, both of which license copyright and database rights. This means that when the Information is adapted and licensed under either of those licences, you automatically satisfy the conditions of the OGL when you comply with the other licence. The OGLv3.0 is Open Definition compliant.
+
+Further context, best practice and guidance can be found in the UK Government Licensing Framework section on The National Archives website.

Makefile

@@ -0,0 +1,142 @@
+NAME := uk-cross-government-metadata exchange-model
+INSTALL_STAMP := .install.stamp
+POETRY = $(shell command -v poetry 2> /dev/null)
+
+.DEFAULT_GOAL := help
+
+.PHONY: help clean
+
+help:
+	@echo "Please use 'make <target>' where <target> is one of"
+	@echo ""
+	@echo "  install      install packages and prepare environment"
+	@echo "  update       update packages"
+	@echo "  clean        remove all temporary files"
+	@echo "  gen-project  generate model constraints in different representations"
+	@echo "  test         run all the tests"
+	@echo "  serve        run the documentation locally; need to use ctrl-c to close the documentation server down"
+	@echo "  docker-serve build and run the documentation in a Docker container"
+	@echo "  all          run clean, gen-project, test, and serve"
+	@echo ""
+	@echo "Check the Makefile to know exactly what each target is doing."
+
+RUN = $(POETRY) run
+DEST = project
+DOCDIR = docs
+EXAMPLEDIR = examples
+
+all: clean gen-project test serve
+
+###########################################################
+# Environment configuration
+###########################################################
+
+install: $(INSTALL_STAMP)
+$(INSTALL_STAMP): pyproject.toml poetry.lock
+	@if [ -z $(POETRY) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	$(POETRY) install
+	touch $(INSTALL_STAMP)
+
+update: pyproject.toml poetry.lock
+	@if [ -z $(POETRY) ]; then echo "Poetry could not be found. See https://python-poetry.org/docs/"; exit 2; fi
+	$(POETRY) update
+	touch $(INSTALL_STAMP)
+
+###########################################################
+# Schema generation
+###########################################################
+
+gen-project: 
+	$(RUN) gen-project -d $(DEST) src/model/uk_cross_government_metadata_exchange_model.yaml
+
+###########################################################
+# Examples and Tests
+###########################################################
+
+gen-examples: $(DOCDIR)
+	cp src/data/README.md $(DOCDIR)/$(EXAMPLEDIR)
+	# TODO: Convert to JSON for the example directory
+	cp src/data/*/valid/* $(DOCDIR)/$(EXAMPLEDIR)
+
+# TODO: Extend tests to lint schema to ensure elements are correctly described, see https://linkml.io/linkml/schemas/linter.html
+
+test: gen-project gen-examples test-valid test-invalid
+
+test-valid: src/model/uk_cross_government_metadata_exchange_model.yaml
+	@for dir in src/data/*; do\
+		if [ -d $${dir} ]; then\
+			echo `basename $${dir}`;\
+  			for file in $${dir}/valid/*.yaml; do\
+    			echo $${file};\
+    			$(RUN) linkml-validate \
+      				$${file} \
+      				-s $< \
+      				--target-class `basename $${dir}`&&\
+				echo "Test passed" || { echo "Test failed!"; exit 1; };\
+  			done;\
+		fi;\
+	done;
+	@echo "Successfully passed all positive tests!\n"
+
+test-invalid: src/model/uk_cross_government_metadata_exchange_model.yaml
+	@for dir in src/data/*; do\
+		if [ -d $${dir} ]; then\
+  			for file in $${dir}/invalid/*.yaml; do\
+    			echo $${file};\
+    			$(RUN) linkml-validate \
+      				$${file} \
+      				-s $< \
+      				--target-class `basename $${dir}` &> $${dir}/invalid/$${file}-test-output.txt &&\
+				{ echo "Test failed"; exit 1; } || echo "Test passed";\
+  			done;\
+		fi;\
+	done;
+	@echo "Successfully passed all negative tests!\n"
+
+#TODO: When linkml fixes their linkml-run-examples command we should be able to use the following to run the tests
+# linkml-run-examples \
+# 	--output-formats json \
+# 	--output-formats yaml \
+# 	--input-directory src/data/examples-valid \
+# 	--counter-example-input-directory src/data/examples-invalid \
+# 	--output-directory $@ \
+# 	--schema $< \
+# 	> $@/README.md
+
+###########################################################
+# DOCUMENTATION
+###########################################################
+
+# Run documentation locally
+# serve will be passed as an argument to mkdocs
+serve: mkd-serve
+
+$(DOCDIR):
+	mkdir -p $(DOCDIR)/$(EXAMPLEDIR)
+
+gendoc: $(DOCDIR)
+	cp src/docs/*.md $(DOCDIR)
+	cp LICENSE.md $(DOCDIR)
+	$(RUN) gen-doc -d $(DOCDIR) src/model/uk_cross_government_metadata_exchange_model.yaml
+
+MKDOCS = $(RUN) mkdocs
+
+# Run mkdocs with whatever argument was used in the make target
+mkd-%: gendoc
+	$(MKDOCS) $*
+
+# Allows the testing of documentation locally on macOS machines
+# Needed as macOS does not support multiple entities with the same name but different cases
+docker-serve:
+	docker build -t model-docs .   
+	docker run -p 8080:8080 model-docs
+###########################################################
+# CLEANUP
+###########################################################
+
+clean:
+	-rm -r $(DEST)
+	-rm -r $(DOCDIR)
+	-rm -r $(VENV)
+	find . -type d -name "__pycache__" | xargs rm -rf {};
+	rm -rf $(INSTALL_STAMP) .coverage .mypy_cache