More regression tests

.github/workflows/deps.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
 
     - name: Checkout code
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: NVD clojure
-      uses: jomco/nvd-clojure-action@v2
+      uses: jomco/nvd-clojure-action@v3

.github/workflows/e2e.yml

@@ -0,0 +1,49 @@
+name: Run e2e tests
+
+on:
+  workflow_dispatch:
+  push:
+
+jobs:
+  e2e:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
+      with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
+
+    - name: Start Redis
+      uses: supercharge/[email protected]
+      with:
+        redis-version: 6.2
+
+    - name: Run tests
+      env:
+        CLIENT_ID: ${{ secrets.CLIENT_ID }}
+        CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+        CLIENTS_INFO_PATH: ${{ secrets.CLIENTS_INFO_PATH }}
+        GATEWAY_PASSWORD: ${{ secrets.GATEWAY_PASSWORD }}
+        GATEWAY_ROOT_URL: ${{ secrets.GATEWAY_ROOT_URL }}
+        GATEWAY_USER: ${{ secrets.GATEWAY_USER }}
+        KEYSTORE_ALIAS: ${{ secrets.KEYSTORE_ALIAS }}
+        KEYSTORE_JKS_B64: ${{ secrets.KEYSTORE_JKS_B64 }}
+        KEYSTORE_PASSWORD: ${{ secrets.KEYSTORE_PASSWORD }}
+        KEYSTORE: test/keystore.jks
+        OS_AUTH_URL: ${{ secrets.OS_AUTH_URL }}
+        OS_CONTAINER_NAME: ${{ secrets.OS_CONTAINER_NAME }}
+        OS_PASSWORD: ${{ secrets.OS_PASSWORD }}
+        OS_PROJECT_NAME: ${{ secrets.OS_PROJECT_NAME }}
+        OS_USERNAME: ${{ secrets.OS_USERNAME }}
+        RIO_READ_URL: ${{ secrets.RIO_READ_URL }}
+        RIO_RECIPIENT_OIN: ${{ secrets.RIO_RECIPIENT_OIN }}
+        RIO_SENDER_OIN: ${{ secrets.RIO_SENDER_OIN }}
+        RIO_UPDATE_URL: ${{ secrets.RIO_UPDATE_URL }}
+        SURF_CONEXT_CLIENT_ID: ${{ secrets.SURF_CONEXT_CLIENT_ID }}
+        SURF_CONEXT_CLIENT_SECRET: ${{ secrets.SURF_CONEXT_CLIENT_SECRET }}
+        SURF_CONEXT_INTROSPECTION_ENDPOINT: ${{ secrets.SURF_CONEXT_INTROSPECTION_ENDPOINT }}
+        TOKEN_ENDPOINT: ${{ secrets.TOKEN_ENDPOINT }}
+        TRUSTSTORE_JKS_B64: ${{ secrets.TRUSTSTORE_JKS_B64 }}
+        TRUSTSTORE_PASSWORD: ${{ secrets.TRUSTSTORE_PASSWORD }}
+        TRUSTSTORE: truststore.jks
+      run: lein test :e2e

.github/workflows/test.yml

@@ -9,8 +9,8 @@ jobs:
       NVD_API_TOKEN: ${{ secrets.NVD_API_TOKEN }}
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
       with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 
     - name: Check dependency freshness
@@ -23,8 +23,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
       with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 
     - name: Run linters
@@ -37,8 +37,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
       with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 
     - name: Run tests
@@ -81,8 +81,8 @@ jobs:
 #    runs-on: ubuntu-latest
 #
 #    steps:
-#    - uses: actions/checkout@v3
-#    - uses: actions/cache@v3
+#    - uses: actions/checkout@v4
+#    - uses: actions/cache@v4
 #      with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 #
 #    - name: Start Redis
@@ -122,8 +122,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
       with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 
     - name: Proof data specs
@@ -133,8 +133,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/cache@v3
+    - uses: actions/checkout@v4
+    - uses: actions/cache@v4
       with: { path: "~/.m2", key: "${{ runner.os }}-m2" }
 
     - name: Run compile
@@ -144,7 +144,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Run docker build
       run: docker build -t eduhub-rio-mapper .

src/nl/surf/eduhub_rio_mapper/cli.clj

@@ -198,7 +198,9 @@
         response-type (and response-type (keyword response-type))]
     (assert (rio.loader/valid-get-types type))
     (-> (when pagina {:pagina pagina})
-        (assoc (if (= type "opleidingsrelatiesBijOpleidingseenheid") ::rio/opleidingscode ::ooapi/id) id
+        (assoc (if (rio.loader/aangeboden-opleiding-types type)
+                 ::ooapi/id
+                 ::rio/opleidingscode) id
                :response-type response-type
                ::rio/type type))))
 

src/nl/surf/eduhub_rio_mapper/job.clj

@@ -39,8 +39,15 @@
                       :institution-schac-home institution-schac-home
                       :institution-oin institution-oin
                       :institution-name institution-name)
-        job         (select-keys request [:action :args :institution-oin :institution-name :institution-schac-home
-                                          ::rio/aangeboden-opleiding-code ::rio/opleidingscode ::ooapi/type ::ooapi/id])]
+        job         (select-keys request [:action
+                                          :args
+                                          :institution-oin
+                                          :institution-name
+                                          :institution-schac-home
+                                          ::rio/aangeboden-opleiding-code
+                                          ::rio/opleidingscode
+                                          ::ooapi/type
+                                          ::ooapi/id])]
     (logging/with-mdc log-context
       (log/infof "Started job %s, action %s, type %s, id %s" token action type id)
       (binding [*http-messages* (if http-logging-enabled (atom []) nil)]

src/nl/surf/eduhub_rio_mapper/processing.clj

@@ -108,8 +108,7 @@
 (defn- make-updater-mutate-rio-phase [{:keys [rio-config]}]
   (fn mutate-rio-phase [{:keys [job result eduspec]}]
     {:pre [(s/valid? ::Mutation/mutation-response result)]}
-    (logging/with-mdc
-      {:soap-action (:action result) :ooapi-id (::ooapi/id job)}
+    (logging/with-mdc {:soap-action (:action result) :ooapi-id (::ooapi/id job)}
       {:job job :eduspec eduspec :mutate-result (mutator/mutate! result rio-config)})))
 
 (defn- make-updater-confirm-rio-phase [{:keys [resolver]} rio-config]

src/nl/surf/eduhub_rio_mapper/rio/loader.clj

@@ -45,12 +45,14 @@
 (def aangeboden-opleiding-namen
   #{:aangebodenHOOpleidingsonderdeel :aangebodenHOOpleiding :aangebodenParticuliereOpleiding})
 
+;; NOTE: aangeboden opleidingen are referenced by OOAPI UID
+(def aangeboden-opleiding-types #{aangeboden-opleiding
+                                  aangeboden-opleidingen-van-organisatie})
 
-(def valid-get-types #{aangeboden-opleiding
-                       aangeboden-opleidingen-van-organisatie
-                       opleidingseenheid
-                       opleidingseenheden-van-organisatie
-                       opleidingsrelaties-bij-opleidingseenheid})
+(def valid-get-types (into aangeboden-opleiding-types
+                           #{opleidingseenheid
+                             opleidingseenheden-van-organisatie
+                             opleidingsrelaties-bij-opleidingseenheid}))
 
 (def schema "http://duo.nl/schema/DUO_RIO_Raadplegen_OnderwijsOrganisatie_V4")
 (def contract "http://duo.nl/contract/DUO_RIO_Raadplegen_OnderwijsOrganisatie_V4")
@@ -184,17 +186,20 @@
 
 (defn- response-handler-for-type [response-type type]
   (case response-type
-    :xml rio-xml-getter-response
-    :json rio-json-getter-response
+    :literal identity
+    :xml     rio-xml-getter-response
+    :json    rio-json-getter-response
     ;; If unspecified, use edn for relations and json for everything else
     (if (= type opleidingsrelaties-bij-opleidingseenheid)
       rio-relation-getter-response
       rio-json-getter-response)))
 
 (defn find-opleidingseenheid [rio-code getter institution-oin]
   {:pre [rio-code]}
-  (-> (getter {::rio/type       opleidingseenheid ::ooapi/id rio-code
-               :institution-oin institution-oin :response-type :xml})
+  (-> (getter {::rio/type           opleidingseenheid
+               ::rio/opleidingscode rio-code
+               :institution-oin     institution-oin
+               :response-type       :xml})
       clj-xml/parse-str
       xml-seq
       (xml-utils/find-in-xmlseq #(when (opleidingseenheid-namen (:tag %)) %))))
@@ -244,28 +249,25 @@
                ::rio/keys   [type opleidingscode]
                :keys        [institution-oin pagina response-type]
                :or          {pagina 0}}]
-    {:pre [(or (not= type opleidingseenheid)
-               id)]}
+    {:pre [(or (and (aangeboden-opleiding-types type) id)
+               opleidingscode)]}
     (when-not (valid-get-types type)
       (throw (ex-info (str "Unexpected type: " type)
-                      {:id             id,
-                       :opleidingscode opleidingscode,
+                      {:id             id
+                       :opleidingscode opleidingscode
                        :retryable?     false})))
 
     (when (and (= type opleidingseenheden-van-organisatie)
-               (not (valid-onderwijsbestuurcode? id)))
-      ;; WHOAA!! This is not a real OOAPI ID but a hack to allow
-      ;; command line to get opleidingseenheden.
-      (throw (ex-info (str "Type 'onderwijsbestuurcode' has ID invalid format: " id)
-                      {:type           type,
-                       :opleidingscode opleidingscode
+               (not (valid-onderwijsbestuurcode? opleidingscode)))
+      (throw (ex-info (str "Type 'onderwijsbestuurcode' has ID invalid format: " opleidingscode)
+                      {:type           type
                        :retryable?     false})))
 
     (let [soap-action (str "opvragen_" type)
           rio-sexp    (condp = type
                         ;; Command line only.
                         opleidingseenheden-van-organisatie
-                        [[:duo:onderwijsbestuurcode id]
+                        [[:duo:onderwijsbestuurcode opleidingscode] ;; FIXME: this is not an opleidingscode!
                          [:duo:pagina pagina]]
 
                         ;; Command line only.
@@ -280,9 +282,8 @@
                         [[:duo:aangebodenOpleidingCode id]]
 
                         opleidingseenheid
-                        [[:duo:opleidingseenheidcode id]])]
-      (logging/with-mdc
-        {:soap-action soap-action}
+                        [[:duo:opleidingseenheidcode opleidingscode]])]
+      (logging/with-mdc {:soap-action soap-action}
         (let [xml (soap/prepare-soap-call soap-action
                                           rio-sexp
                                           (make-datamap institution-oin recipient-oin)

src/nl/surf/eduhub_rio_mapper/rio/mutator.clj

@@ -60,6 +60,7 @@
     "A01160" ;; Externe identificatie niet uniek is niet recoverable dus zou niet moeten retryen
     "K01010" ;; 'propedeutischeFase' komt niet vaak genoeg voor als kenmerk
     "P01081" ;; Er bestaat al een opleidingseenheid met dezelfde eigenOpleidingseenheidSleutel
+    "N11030" ;; De einddatum van een instroomperiode moet na de begindatum liggen
     })
 
 ;; Note: `P01810: Er bestaan nog verwijzingen naar de te verwijderen