Merge pull request #78 from ExposuresProvider/add-kubernetes

This PR adds the Kubernetes CronJob file to the repository. It moves one of the steps from the Kubernetes file into the Makefile. It seems to be mostly working -- I've tested one of the files generated by this job and it produces a small number of errors on CAM-KP-API (#80, ExposuresProvider/cam-kp-api#613) -- and I'd rather not delay this any further so that I can move on to trying to upgrade cam-pipeline to Biolink 3.

Makefile

@@ -15,10 +15,12 @@ SCALA_RUN=$(JAVA_ENV) scala-cli run --home /tools
 NOCTUA_MODELS_REPO=gene-data/noctua-models
 BIOLINK=2.1.0
 
-.PHONY: clean validate
+.PHONY: clean validate all
 
+all: cam-db-reasoned.jnl
 clean:
 	rm -rf gene-data
+	rm -rf noctua-models.jnl
 
 owlrl-datalog:
 	git clone https://github.com/balhoff/owlrl-datalog.git
@@ -63,15 +65,17 @@ missing-biolink-terms.ttl: sparql/reports/owl-missing-biolink-term.rq cam-db-rea
 missing-biolink-relation.ttl: sparql/reports/owl-missing-biolink-relation.rq cam-db-reasoned.jnl
 	$(BLAZEGRAPH-RUNNER) select --journal=cam-db-reasoned.jnl --properties=blazegraph.properties --outformat=TSV $< $@
 
-all: cam-db-reasoned.jnl
+$(NOCTUA_MODELS_REPO):
+	mkdir -p gene-data
+	git clone --depth 1 https://github.com/geneontology/noctua-models gene-data/noctua-models
 
-noctua-models.jnl: $(NOCTUA_MODELS_REPO)/models/*.ttl signor-models
+noctua-models.jnl: $(NOCTUA_MODELS_REPO) signor-models
 	$(BLAZEGRAPH-RUNNER) load --journal=$@ --properties=blazegraph.properties --informat=turtle --use-ontology-graph=true signor-models &&\
 	$(BLAZEGRAPH-RUNNER) update --journal=$@ --properties=blazegraph.properties sparql/set-provenance-to-signor.ru &&\
 	$(BLAZEGRAPH-RUNNER) load --journal=$@ --properties=blazegraph.properties --informat=turtle --use-ontology-graph=true $(NOCTUA_MODELS_REPO)/models &&\
 	$(BLAZEGRAPH-RUNNER) update --journal=$@ --properties=blazegraph.properties sparql/delete-non-production-models.ru
 
-noctua-models-inferences.nq: $(NOCTUA_MODELS_REPO)/models/*.ttl sparql/is-production.rq ontologies-merged.ttl
+noctua-models-inferences.nq: $(NOCTUA_MODELS_REPO) sparql/is-production.rq ontologies-merged.ttl
 	$(MAT) --ontology-file ontologies-merged.ttl --input $(NOCTUA_MODELS_REPO)/models --output $@ --output-graph-name '#inferred' --suffix-graph true --mark-direct-types true --output-indirect-types true --parallelism 20 --filter-graph-query sparql/is-production.rq --reasoner arachne
 
 signor-models-inferences.nq: signor-models

kubernetes/build-cam-database.yaml

@@ -0,0 +1,46 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: build-cam-database
+  labels:
+    project: CAM-KP
+    env: dev
+spec:
+  schedule: "@weekly"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: pipeline-tools
+            image: "renciorg/cam-pipeline-tools:v1.3"
+            resources:
+              limits:
+                cpu: '8'
+                memory: 150G
+            command: ["sh", "-c", "cd /workspace; rm -rf current; mkdir current; cd current; git clone https://github.com/NCATS-Tangerine/cam-pipeline.git; cd cam-pipeline; git checkout master; make all; rm -rf /workspace/success; mv /workspace/current /workspace/success"]
+            volumeMounts:
+            - mountPath: "/workspace"
+              name: storage
+          restartPolicy: Never
+          volumes:
+          - name: storage
+            persistentVolumeClaim:
+              claimName: build-cam-database-storage
+      backoffLimit: 0
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: build-cam-database-storage
+  labels:
+    project: CAM-KP
+    env: dev
+spec:
+  accessModes:
+  - ReadWriteMany
+  resources:
+    requests:
+      storage: 1000G
+  storageClassName: basic