Skip to content

Commit

Permalink
More regression tests (#272)
Browse files Browse the repository at this point in the history
* Fix rio.loader/getter expecting an ooapi/id instead of rio/opleidingscode

* e2e: assert hoOpleiding data in RIO correctly upserted

* e2e: add gh actions workflow

* e2e: add "try to create edspecs with invalid data" scenario

* e2e: rename eduspec entity for clarity

* e2e: support remote entity programs with offerings

* e2e: add "create a program (for the edSpec child)" scenario

* e2e: add "try to create a program with invalid data" scenario

* e2e: add "create a course with its own edspec" scenario

* Fix rio.loader/getter expecting an ooapi/id instead of rio/opleidingscode [FEEDBACK]

* e2e: assert hoOpleiding data in RIO correctly upserted [FEEDBACK]

* e2e: support remote entity programs with offerings [FEEDBACK]

* gh: upgrade gh actions

* e2e: github action on "workflow_dispatch" [WIP]
  • Loading branch information
remvee authored Jan 29, 2024
1 parent 3564594 commit cffdc2a
Show file tree
Hide file tree
Showing 24 changed files with 1,780 additions and 302 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/deps.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
steps:

- name: Checkout code
uses: actions/checkout@v2
uses: actions/checkout@v4

- name: NVD clojure
uses: jomco/nvd-clojure-action@v2
uses: jomco/nvd-clojure-action@v3
49 changes: 49 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
name: Run e2e tests

on:
workflow_dispatch:
push:

jobs:
e2e:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Start Redis
uses: supercharge/[email protected]
with:
redis-version: 6.2

- name: Run tests
env:
CLIENT_ID: ${{ secrets.CLIENT_ID }}
CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
CLIENTS_INFO_PATH: ${{ secrets.CLIENTS_INFO_PATH }}
GATEWAY_PASSWORD: ${{ secrets.GATEWAY_PASSWORD }}
GATEWAY_ROOT_URL: ${{ secrets.GATEWAY_ROOT_URL }}
GATEWAY_USER: ${{ secrets.GATEWAY_USER }}
KEYSTORE_ALIAS: ${{ secrets.KEYSTORE_ALIAS }}
KEYSTORE_JKS_B64: ${{ secrets.KEYSTORE_JKS_B64 }}
KEYSTORE_PASSWORD: ${{ secrets.KEYSTORE_PASSWORD }}
KEYSTORE: test/keystore.jks
OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }}
OS_CONTAINER_NAME: ${{ secrets.OS_CONTAINER_NAME }}
OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
OS_USERNAME: ${{ secrets.OS_USERNAME }}
RIO_READ_URL: ${{ secrets.RIO_READ_URL }}
RIO_RECIPIENT_OIN: ${{ secrets.RIO_RECIPIENT_OIN }}
RIO_SENDER_OIN: ${{ secrets.RIO_SENDER_OIN }}
RIO_UPDATE_URL: ${{ secrets.RIO_UPDATE_URL }}
SURF_CONEXT_CLIENT_ID: ${{ secrets.SURF_CONEXT_CLIENT_ID }}
SURF_CONEXT_CLIENT_SECRET: ${{ secrets.SURF_CONEXT_CLIENT_SECRET }}
SURF_CONEXT_INTROSPECTION_ENDPOINT: ${{ secrets.SURF_CONEXT_INTROSPECTION_ENDPOINT }}
TOKEN_ENDPOINT: ${{ secrets.TOKEN_ENDPOINT }}
TRUSTSTORE_JKS_B64: ${{ secrets.TRUSTSTORE_JKS_B64 }}
TRUSTSTORE_PASSWORD: ${{ secrets.TRUSTSTORE_PASSWORD }}
TRUSTSTORE: truststore.jks
run: lein test :e2e
26 changes: 13 additions & 13 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ jobs:
NVD_API_TOKEN: ${{ secrets.NVD_API_TOKEN }}

steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Check dependency freshness
Expand All @@ -23,8 +23,8 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Run linters
Expand All @@ -37,8 +37,8 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Run tests
Expand Down Expand Up @@ -81,8 +81,8 @@ jobs:
# runs-on: ubuntu-latest
#
# steps:
# - uses: actions/checkout@v3
# - uses: actions/cache@v3
# - uses: actions/checkout@v4
# - uses: actions/cache@v4
# with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
#
# - name: Start Redis
Expand Down Expand Up @@ -122,8 +122,8 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Proof data specs
Expand All @@ -133,8 +133,8 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
- uses: actions/checkout@v4
- uses: actions/cache@v4
with: { path: "~/.m2", key: "${{ runner.os }}-m2" }

- name: Run compile
Expand All @@ -144,7 +144,7 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Run docker build
run: docker build -t eduhub-rio-mapper .
4 changes: 3 additions & 1 deletion src/nl/surf/eduhub_rio_mapper/cli.clj
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,9 @@
response-type (and response-type (keyword response-type))]
(assert (rio.loader/valid-get-types type))
(-> (when pagina {:pagina pagina})
(assoc (if (= type "opleidingsrelatiesBijOpleidingseenheid") ::rio/opleidingscode ::ooapi/id) id
(assoc (if (rio.loader/aangeboden-opleiding-types type)
::ooapi/id
::rio/opleidingscode) id
:response-type response-type
::rio/type type))))

Expand Down
11 changes: 9 additions & 2 deletions src/nl/surf/eduhub_rio_mapper/job.clj
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,15 @@
:institution-schac-home institution-schac-home
:institution-oin institution-oin
:institution-name institution-name)
job (select-keys request [:action :args :institution-oin :institution-name :institution-schac-home
::rio/aangeboden-opleiding-code ::rio/opleidingscode ::ooapi/type ::ooapi/id])]
job (select-keys request [:action
:args
:institution-oin
:institution-name
:institution-schac-home
::rio/aangeboden-opleiding-code
::rio/opleidingscode
::ooapi/type
::ooapi/id])]
(logging/with-mdc log-context
(log/infof "Started job %s, action %s, type %s, id %s" token action type id)
(binding [*http-messages* (if http-logging-enabled (atom []) nil)]
Expand Down
3 changes: 1 addition & 2 deletions src/nl/surf/eduhub_rio_mapper/processing.clj
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,7 @@
(defn- make-updater-mutate-rio-phase [{:keys [rio-config]}]
(fn mutate-rio-phase [{:keys [job result eduspec]}]
{:pre [(s/valid? ::Mutation/mutation-response result)]}
(logging/with-mdc
{:soap-action (:action result) :ooapi-id (::ooapi/id job)}
(logging/with-mdc {:soap-action (:action result) :ooapi-id (::ooapi/id job)}
{:job job :eduspec eduspec :mutate-result (mutator/mutate! result rio-config)})))

(defn- make-updater-confirm-rio-phase [{:keys [resolver]} rio-config]
Expand Down
47 changes: 24 additions & 23 deletions src/nl/surf/eduhub_rio_mapper/rio/loader.clj
Original file line number Diff line number Diff line change
Expand Up @@ -45,12 +45,14 @@
(def aangeboden-opleiding-namen
#{:aangebodenHOOpleidingsonderdeel :aangebodenHOOpleiding :aangebodenParticuliereOpleiding})

;; NOTE: aangeboden opleidingen are referenced by OOAPI UID
(def aangeboden-opleiding-types #{aangeboden-opleiding
aangeboden-opleidingen-van-organisatie})

(def valid-get-types #{aangeboden-opleiding
aangeboden-opleidingen-van-organisatie
opleidingseenheid
opleidingseenheden-van-organisatie
opleidingsrelaties-bij-opleidingseenheid})
(def valid-get-types (into aangeboden-opleiding-types
#{opleidingseenheid
opleidingseenheden-van-organisatie
opleidingsrelaties-bij-opleidingseenheid}))

(def schema "http://duo.nl/schema/DUO_RIO_Raadplegen_OnderwijsOrganisatie_V4")
(def contract "http://duo.nl/contract/DUO_RIO_Raadplegen_OnderwijsOrganisatie_V4")
Expand Down Expand Up @@ -184,17 +186,20 @@

(defn- response-handler-for-type [response-type type]
(case response-type
:xml rio-xml-getter-response
:json rio-json-getter-response
:literal identity
:xml rio-xml-getter-response
:json rio-json-getter-response
;; If unspecified, use edn for relations and json for everything else
(if (= type opleidingsrelaties-bij-opleidingseenheid)
rio-relation-getter-response
rio-json-getter-response)))

(defn find-opleidingseenheid [rio-code getter institution-oin]
{:pre [rio-code]}
(-> (getter {::rio/type opleidingseenheid ::ooapi/id rio-code
:institution-oin institution-oin :response-type :xml})
(-> (getter {::rio/type opleidingseenheid
::rio/opleidingscode rio-code
:institution-oin institution-oin
:response-type :xml})
clj-xml/parse-str
xml-seq
(xml-utils/find-in-xmlseq #(when (opleidingseenheid-namen (:tag %)) %))))
Expand Down Expand Up @@ -244,28 +249,25 @@
::rio/keys [type opleidingscode]
:keys [institution-oin pagina response-type]
:or {pagina 0}}]
{:pre [(or (not= type opleidingseenheid)
id)]}
{:pre [(or (and (aangeboden-opleiding-types type) id)
opleidingscode)]}
(when-not (valid-get-types type)
(throw (ex-info (str "Unexpected type: " type)
{:id id,
:opleidingscode opleidingscode,
{:id id
:opleidingscode opleidingscode
:retryable? false})))

(when (and (= type opleidingseenheden-van-organisatie)
(not (valid-onderwijsbestuurcode? id)))
;; WHOAA!! This is not a real OOAPI ID but a hack to allow
;; command line to get opleidingseenheden.
(throw (ex-info (str "Type 'onderwijsbestuurcode' has ID invalid format: " id)
{:type type,
:opleidingscode opleidingscode
(not (valid-onderwijsbestuurcode? opleidingscode)))
(throw (ex-info (str "Type 'onderwijsbestuurcode' has ID invalid format: " opleidingscode)
{:type type
:retryable? false})))

(let [soap-action (str "opvragen_" type)
rio-sexp (condp = type
;; Command line only.
opleidingseenheden-van-organisatie
[[:duo:onderwijsbestuurcode id]
[[:duo:onderwijsbestuurcode opleidingscode] ;; FIXME: this is not an opleidingscode!
[:duo:pagina pagina]]

;; Command line only.
Expand All @@ -280,9 +282,8 @@
[[:duo:aangebodenOpleidingCode id]]

opleidingseenheid
[[:duo:opleidingseenheidcode id]])]
(logging/with-mdc
{:soap-action soap-action}
[[:duo:opleidingseenheidcode opleidingscode]])]
(logging/with-mdc {:soap-action soap-action}
(let [xml (soap/prepare-soap-call soap-action
rio-sexp
(make-datamap institution-oin recipient-oin)
Expand Down
1 change: 1 addition & 0 deletions src/nl/surf/eduhub_rio_mapper/rio/mutator.clj
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
"A01160" ;; Externe identificatie niet uniek is niet recoverable dus zou niet moeten retryen
"K01010" ;; 'propedeutischeFase' komt niet vaak genoeg voor als kenmerk
"P01081" ;; Er bestaat al een opleidingseenheid met dezelfde eigenOpleidingseenheidSleutel
"N11030" ;; De einddatum van een instroomperiode moet na de begindatum liggen
})

;; Note: `P01810: Er bestaan nog verwijzingen naar de te verwijderen
Expand Down
Loading

0 comments on commit cffdc2a

Please sign in to comment.