diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml new file mode 100644 index 00000000..ca26694c --- /dev/null +++ b/.github/workflows/PR-validation.yml @@ -0,0 +1,53 @@ +name: PR validation +on: + pull_request: + types: [synchronize, opened, reopened, edited] + branches: + - master +jobs: + test-api: + permissions: + id-token: write # Required for authentication through OIDC to AWS + runs-on: ubuntu-22.04 + steps: + - name: Report workflow details + run: | + echo "Repository ${{ github.repository }}." + echo "Trigger ref ${{ github.ref }}, base-ref ${{ github.base_ref }}, head_ref ${{ github.head_ref }}." + - name: Check out repository code + uses: actions/checkout@v3 + - name: Report files updated in PR + run: | + git fetch -q origin ${{ github.base_ref }} ${{ github.head_ref }} + git diff --name-only origin/${{ github.base_ref }} origin/${{ github.head_ref }} + - name: Set up JDK 8 + uses: actions/setup-java@v3 + with: + java-version: '8' + distribution: 'temurin' + - name: Install clojure and clojure cli (clj) + uses: DeLaGuardo/setup-clojure@12.3 + with: + cli: 1.10.1.536 + - name: Report runtime details + run: | + echo "Github runner OS: ${{ runner.os }}" + - name: AWS credentials configuration + uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} + role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-test-api + aws-region: us-east-1 + - name: Download and install Datomic Pro + run: | + aws s3 cp s3://wormbase/datomic-pro/distro/datomic-pro-1.0.6165.zip ./ + unzip datomic-pro-1.0.6165.zip + cd datomic-pro-1.0.6165/ + bin/maven-install + - name: Generate pom file + run: | + clojure -Spom + - name: Run Integration tests + run: | + make run-tests GOOGLE_APP_PROFILE=dev + #TODO: add UI and API build and container packaging test diff --git a/Makefile b/Makefile index c9f7f4ee..27c3ae62 100644 --- a/Makefile +++ b/Makefile @@ -235,7 +235,7 @@ run-tests: google-oauth2-secrets \ @ export API_GOOGLE_OAUTH_CLIENT_ID=${GOOGLE_OAUTH_CLIENT_ID} && \ export API_GOOGLE_OAUTH_CLIENT_SECRET=${GOOGLE_OAUTH_CLIENT_SECRET} && \ export GOOGLE_REDIRECT_URI=${LOCAL_GOOGLE_REDIRECT_URI} && \ - clj -A:datomic-pro:logging:webassets:dev:test:run-tests + clojure -A:datomic-pro:logging:webassets:dev:test:run-tests .PHONY: run-dev-server run-dev-webserver: PORT := 4010 diff --git a/deps.edn b/deps.edn index dbc060bc..441a567d 100644 --- a/deps.edn +++ b/deps.edn @@ -21,9 +21,11 @@ metosin/compojure-api {:mvn/version "2.0.0-alpha30" :exclusions [frankiesardo/linked]} metosin/muuntaja {:mvn/version "0.6.8"} + metosin/ring-http-response {:mvn/version "0.9.3"} ikitommi/linked {:mvn/version "1.3.1-alpha1"} mount/mount {:mvn/version "0.1.17"} phrase/phrase {:mvn/version "0.3-alpha4"} + ring/ring-core {:mvn/version "1.9.4"} ring/ring-codec {:mvn/version "1.2.0"} ring/ring-defaults {:mvn/version "0.4.0"} ring/ring-jetty-adapter {:mvn/version "1.10.0"} diff --git a/export/src/wormbase/names/export.clj b/export/src/wormbase/names/export.clj index 96a80d06..33d72899 100644 --- a/export/src/wormbase/names/export.clj +++ b/export/src/wormbase/names/export.clj @@ -2,7 +2,6 @@ (:require [clojure.java.io :as io] [clojure.data.csv :as cd-csv] - [clojure.set :as set] [clojure.string :as str] [clojure.tools.cli :as cli] [datomic.api :as d] diff --git a/ids/src/wormbase/ids/batch.clj b/ids/src/wormbase/ids/batch.clj index 7c1ee7e3..36500787 100644 --- a/ids/src/wormbase/ids/batch.clj +++ b/ids/src/wormbase/ids/batch.clj @@ -19,7 +19,7 @@ [clojure.core :as cc] [clojure.set :as set] [datomic.api :as d] - [wormbase.ids.core :refer [attr-schema-unique? identifier-format]])) + [wormbase.ids.core :refer [identifier-format]])) (defn- assoc-prov "Attach an identifier to `prov` making this a mapping suitable for tracking provenance for a batch. @@ -32,12 +32,6 @@ (when tx-data (cons prov tx-data))) -(defn db-error? [exc] - (some->> (ex-data exc) - (keys) - (filter (fn has-db-ns? [k] - (#{"db" "db.error"} (namespace k)))))) - (defrecord BatchResult [tx-result errors]) (defrecord BatchError [error-type exc]) diff --git a/ids/src/wormbase/ids/core.clj b/ids/src/wormbase/ids/core.clj index 0331e075..fce32bea 100644 --- a/ids/src/wormbase/ids/core.clj +++ b/ids/src/wormbase/ids/core.clj @@ -9,7 +9,7 @@ (try (let [s-attr (:db/unique (d/entity db attr))] (#{:db.unique/identity :db.unique/value} s-attr)) - (catch IllegalArgumentException iae))) + (catch IllegalArgumentException _))) (defn entids->data [db m k v] (assoc m diff --git a/src/wormbase/names/importers/processing.clj b/src/wormbase/names/importers/processing.clj index aa841bc5..cfc5f1a3 100644 --- a/src/wormbase/names/importers/processing.clj +++ b/src/wormbase/names/importers/processing.clj @@ -130,7 +130,7 @@ (jt/zoned-date-time tz) (jt/with-zone-same-instant tz) (jt/instant) - (jt/to-java-date))) + (jt/java-date))) (defn check-environ! [] (when-not (environ/env :token) diff --git a/src/wormbase/names/provenance.clj b/src/wormbase/names/provenance.clj index d9623930..1d4b7642 100644 --- a/src/wormbase/names/provenance.clj +++ b/src/wormbase/names/provenance.clj @@ -47,7 +47,7 @@ whence (-> (get prov :provenance/when (jt/instant)) (jt/zoned-date-time tz) (jt/with-zone-same-instant tz) - (jt/to-java-date)) + (jt/java-date)) how (-> request :headers wna/identify) why (:provenance/why prov) prov {:db/id "datomic.tx" diff --git a/src/wormbase/names/recent.clj b/src/wormbase/names/recent.clj index 472d4103..cefcd785 100644 --- a/src/wormbase/names/recent.clj +++ b/src/wormbase/names/recent.clj @@ -10,8 +10,6 @@ [wormbase.util :as wu]) (:import (java.util Date))) -(def conf (:recent (wu/read-app-config))) - (defn- find-max-imported-date [db] (let [max-tx-inst (d/q '[:find (max ?inst) . :where @@ -23,7 +21,7 @@ (jt/instant))] (-> max-date (jt/plus (jt/seconds 1)) - (jt/to-java-date)))) + (jt/java-date)))) (def imported-date (memoize find-max-imported-date)) @@ -45,7 +43,7 @@ from-t (if (and from (>= (compare from import-date) 0)) from import-date) - until-t (or until (jt/to-java-date (jt/instant)))] + until-t (or until (jt/java-date (jt/instant)))] ;; Timings for the `tx-ids` query below with default configured time window (60 days) ;; (excluding pull expressions) ;; jvm (cold): 107.266427 msecs @@ -108,8 +106,6 @@ (def batch-rules '[[(filter-events ?tx ?needle) [?tx :batch/id _ _ ]]]) -(def response-schema (wnu/response-map ok {:schema {:activities ::wsr/activities}})) - (defn handle ([request rules puller needle from until] (handle request rules puller needle from until #{:agent/console :agent/web})) @@ -117,7 +113,7 @@ (let [{conn :conn db :db} request log (d/log conn) from* (or from (wu/days-ago wsr/*default-days-ago*)) - until* (or until (jt/to-java-date (jt/instant))) + until* (or until (jt/java-date (jt/instant))) items (activities db log rules puller (or needle "") how from* until*) etag (some-> items first :t wnu/encode-etag)] (some-> {:from from* :until until*} @@ -179,7 +175,7 @@ (binding [db (d/db conn) log (d/log conn) pull-prov-only (prov-only-puller db log) pull-changes-and-prov (changes-and-prov-puller db log) - from (jt/to-java-date (jt/instant)) + from (jt/java-date (jt/instant)) until (wu/days-ago 2)] (activities db log entity-rules pull-changes-and-prov "gene") (activities db log entity-rules-rules pull-changes-and-prov "gene" from until) diff --git a/src/wormbase/names/service.clj b/src/wormbase/names/service.clj index c949a641..79bd9c15 100644 --- a/src/wormbase/names/service.clj +++ b/src/wormbase/names/service.clj @@ -23,7 +23,8 @@ [ring.middleware.gzip :as ring-gzip] [ring.middleware.not-modified :as rmnm] [ring.middleware.resource :as ring-resource] - [ring.util.http-response :as http-response])) + [ring.util.http-response :as http-response] + [ring.util.response :as ring-response])) (defn- wrap-not-found "Fallback 404 handler." @@ -36,9 +37,9 @@ (http-response/not-found {:message "Resource not found (fallback)"}) :else - (-> (http-response/resource-response "client_build/index.html") - (http-response/content-type "text/html") - (http-response/status 200)))))) + (-> (ring-response/resource-response "client_build/index.html") + (ring-response/content-type "text/html") + (ring-response/status 200)))))) (def ^:private swagger-validator-url "The URL used to validate the swagger JSON produced by the application." diff --git a/src/wormbase/names/stats.clj b/src/wormbase/names/stats.clj index f47fea8d..81ac2f42 100644 --- a/src/wormbase/names/stats.clj +++ b/src/wormbase/names/stats.clj @@ -3,7 +3,6 @@ [clojure.walk :as w] [compojure.api.sweet :as sweet] [datomic.api :as d] - [ring.middleware.not-modified :as rmnm] [ring.util.http-response :refer [ok]] [wormbase.names.util :as wnu] [wormbase.specs.stats :as wsst])) @@ -29,14 +28,11 @@ (w/stringify-keys)))) (defn handle-summary [request] - (let [etag (some-> request :db d/basis-t wnu/encode-etag)] - (-> (summary request) - (ok) - (wnu/add-etag-header-maybe etag)))) + (-> (summary request) + (ok))) (def routes (sweet/routes (sweet/context "/stats" [] - :middleware [rmnm/wrap-not-modified] :tags ["stats"] (sweet/resource {:get diff --git a/src/wormbase/names/util.clj b/src/wormbase/names/util.clj index d7ce2d91..2e3d7d75 100644 --- a/src/wormbase/names/util.clj +++ b/src/wormbase/names/util.clj @@ -6,20 +6,15 @@ [clojure.string :as str] [clojure.walk :as w] [buddy.core.codecs :as codecs] - [buddy.core.codecs.base64 :as b64] [datomic.api :as d] [expound.alpha :refer [expound-str]] [phrase.alpha :as ph] - [ring.util.http-response :refer [bad-request conflict header not-found not-modified ok]] + [ring.util.http-response :refer [bad-request conflict not-found not-modified ok]] + [ring.util.response :refer [header]] [wormbase.db :as wdb] [wormbase.specs.common :as wsc] [wormbase.specs.validation :as wsv])) -(defn- nsify [domain kw] - (if (namespace kw) - kw - (keyword domain (name kw)))) - ;; trunc and datom-table taken from day-of-datomic repo (congnitect). (defn trunc @@ -143,7 +138,7 @@ bid))) (defn encode-etag [latest-t] - (some-> latest-t str b64/encode codecs/bytes->str)) + (some-> latest-t codecs/long->bytes codecs/bytes->b64 codecs/bytes->str)) (defn add-etag-header-maybe [response etag] (if (seq etag) diff --git a/src/wormbase/util.clj b/src/wormbase/util.clj index 0278b8c2..f0008636 100644 --- a/src/wormbase/util.clj +++ b/src/wormbase/util.clj @@ -49,7 +49,7 @@ (defn days-ago [n] (-> (jt/instant) (jt/minus (jt/days n)) - (jt/to-java-date))) + (jt/java-date))) (defn format-java-date [dt & {:keys [tz fmt] :or {tz (jt/zone-id) @@ -73,7 +73,7 @@ ([tz] (-> (jt/instant) (jt/zoned-date-time (jt/zone-id tz)) - (jt/to-java-date))) + (jt/java-date))) ([] (now "UTC"))) diff --git a/test/integration/service_test.clj b/test/integration/service_test.clj index 52634912..dcba78d8 100644 --- a/test/integration/service_test.clj +++ b/test/integration/service_test.clj @@ -2,7 +2,7 @@ (:require [clojure.test :as t] [ring.util.http-predicates :as ru-hp] - [ring.util.http-response :as http-response] + [ring.util.response :as ring-response] [wormbase.db-testing :as db-testing] [wormbase.fake-auth] [wormbase.names.service :as service])) @@ -19,5 +19,5 @@ :request-method :get})] (t/is (not (nil? response))) (t/is (ru-hp/ok? response)) - (t/is (http-response/get-header response "content-type") "text/html")))) + (t/is (ring-response/get-header response "content-type") "text/html")))) diff --git a/test/integration/stats_test.clj b/test/integration/stats_test.clj index d98e891b..15e2a06e 100644 --- a/test/integration/stats_test.clj +++ b/test/integration/stats_test.clj @@ -16,11 +16,6 @@ (t/deftest test-summary (t/testing "Stats summary renders with ok response on first access." - (let [response (stats-summary) - etag (get-in response [:headers "etag"])] + (let [response (stats-summary)] (t/is (ru-hp/ok? response)) - (t/is (set/subset? #{:gene :variation :sequence-feature} (:body response))) - (t/testing "Status summary renders a not-modified response when nothing has changed." - (let [extra-headers {"if-none-match" etag} - response2 (stats-summary extra-headers)] - (t/is (ru-hp/not-modified? response2))))))) + (t/is (set/subset? #{:gene :variation :sequence-feature} (:body response)))))) diff --git a/test/wormbase/gen_specs/person.clj b/test/wormbase/gen_specs/person.clj index 4f3fc667..7f25c2f7 100644 --- a/test/wormbase/gen_specs/person.clj +++ b/test/wormbase/gen_specs/person.clj @@ -2,7 +2,6 @@ (:require [clojure.spec.alpha :as s] [miner.strgen :as sg] - [wormbase.gen-specs.util :as util] [wormbase.specs.person :as wsp])) (def email (sg/string-generator wsp/email-regexp)) diff --git a/test/wormbase/test_utils.clj b/test/wormbase/test_utils.clj index 29faf982..5d9852ef 100644 --- a/test/wormbase/test_utils.clj +++ b/test/wormbase/test_utils.clj @@ -238,7 +238,7 @@ (defn provenance [_ & {:keys [how what whence why person _ batch-id] :or {how :agent/console - whence (jt/to-java-date (jt/instant)) + whence (jt/java-date (jt/instant)) what :event/test-fixture-assertion person [:person/email "tester@wormbase.org"]}}] (merge {:db/id "datomic.tx"