From c58d6c079b19253a11201082dbab9bb433be2b95 Mon Sep 17 00:00:00 2001 From: Michiel de Mare Date: Wed, 15 May 2024 16:50:39 +0200 Subject: [PATCH] Support `lein mapper show programs` (#302) * Support `lein mapper show programs` * Never return format string * Use recur, and switch to loop --- CLI.md | 24 +++++++ .../surf/eduhub_rio_mapper/ooapi/loader.clj | 68 ++++++++++--------- .../eduhub_rio_mapper/ooapi/loader_test.clj | 2 +- 3 files changed, 62 insertions(+), 32 deletions(-) diff --git a/CLI.md b/CLI.md index 9b6f4618..0dfd6b79 100644 --- a/CLI.md +++ b/CLI.md @@ -76,6 +76,14 @@ lein mapper delete-by-code uni-id education-specification 1234O1234 The `show` command retrieves data from OOAPI. The following entities are supported: +#### courses + +Example: + +```sh +lein mapper show uni-id courses +``` + #### course Example: @@ -84,6 +92,14 @@ Example: lein mapper show uni-id course 123e4567-e89b-12d3-a456-426655440000 ``` +#### programs + +Example: + +```sh +lein mapper show uni-id programs +``` + #### program Example: @@ -92,6 +108,14 @@ Example: lein mapper show uni-id program 123e4567-e89b-12d3-a456-426655440000 ``` +#### education-specifications + +Example: + +```sh +lein mapper show uni-id education-specifications +``` + #### education-specification Example: diff --git a/src/nl/surf/eduhub_rio_mapper/ooapi/loader.clj b/src/nl/surf/eduhub_rio_mapper/ooapi/loader.clj index d9f0fcc6..9e9944ac 100644 --- a/src/nl/surf/eduhub_rio_mapper/ooapi/loader.clj +++ b/src/nl/surf/eduhub_rio_mapper/ooapi/loader.clj @@ -39,17 +39,21 @@ 250) (defn- ooapi-type->path [ooapi-type id page] - (let [page-suffix (if page (str "&pageNumber=" page) "")] - (-> ooapi-type - (case - "education-specification" "education-specifications/%s?returnTimelineOverrides=true" - "program" "programs/%s?returnTimelineOverrides=true" - "course" "courses/%s?returnTimelineOverrides=true" - "course-offerings" (str "courses/%s/offerings?pageSize=" max-offerings "&consumer=rio" page-suffix) - "program-offerings" (str "programs/%s/offerings?pageSize=" max-offerings "&consumer=rio" page-suffix)) - (format id)))) - -(defn ooapi-http-loader + (if id + (let [page-suffix (if page (str "&pageNumber=" page) "") + path (case ooapi-type + "education-specification" "education-specifications/%s?returnTimelineOverrides=true" + "program" "programs/%s?returnTimelineOverrides=true" + "course" "courses/%s?returnTimelineOverrides=true" + "course-offerings" (str "courses/%s/offerings?pageSize=" max-offerings "&consumer=rio" page-suffix) + "program-offerings" (str "programs/%s/offerings?pageSize=" max-offerings "&consumer=rio" page-suffix))] + (format path id)) + (case ooapi-type + "education-specifications" "education-specifications" + "programs" "programs" + "courses" "courses"))) + +(defn- ooapi-http-loader [{::ooapi/keys [root-url type id] :keys [institution-schac-home gateway-credentials connection-timeout page] :as ooapi-request}] @@ -72,33 +76,35 @@ (get-in response-body [:responses (keyword institution-schac-home)])))) ;; For type "offerings", loads all pages and merges them into "items" -(defn ooapi-http-recursive-loader - [{::ooapi/keys [type] - :keys [page page-size] - :as ooapi-request}] +(defn- ooapi-http-recursive-loader + [{:keys [page-size] :as ooapi-request} items] {:pre [(s/valid? ::request/request ooapi-request)]} - (let [responses (ooapi-http-loader ooapi-request)] - (if (not (#{"course-offerings" "program-offerings"} type)) - responses - ;; handle offerings, which may be recursive - (let [items (:items responses)] - (if (< (count items) (or page-size max-offerings)) - {:items items} - ;; We need to recurse, not all offerings seen yet. - (let [next-page (inc (or page 1)) - remaining-items (ooapi-http-recursive-loader (assoc ooapi-request :page next-page)) - all-items (into items remaining-items)] - (if (= 1 page) {:items all-items} all-items))))))) + (loop [next-page 2 + current-page-size (count items) + all-items items] + (if (< current-page-size (or page-size max-offerings)) + ;; Fewer items than maximum allowed means that this is the last page + {:items all-items} + ;; We need to iterate, not all offerings seen yet. + (let [next-items (-> ooapi-request + (assoc :page next-page) + ooapi-http-loader + :items)] + (recur (inc next-page) (count next-items) (into all-items next-items)))))) ;; Returns function that takes context with the following keys: ;; ::ooapi/root-url, ::ooapi/id, ::ooapi/type, :gateway-credentials, institution-schac-home (defn make-ooapi-http-loader [root-url credentials rio-config] (fn wrapped-ooapi-http-loader [context] - (ooapi-http-recursive-loader (assoc context - ::ooapi/root-url root-url - :gateway-credentials credentials - :connection-timeout (:connection-timeout-millis rio-config))))) + (let [request (assoc context + ::ooapi/root-url root-url + :gateway-credentials credentials + :connection-timeout (:connection-timeout-millis rio-config)) + response (ooapi-http-loader request)] + (if (#{"course-offerings" "program-offerings"} (::ooapi/type context)) + (ooapi-http-recursive-loader request (:items response)) + response)))) (defn ooapi-file-loader [{::ooapi/keys [type id]}] diff --git a/test/nl/surf/eduhub_rio_mapper/ooapi/loader_test.clj b/test/nl/surf/eduhub_rio_mapper/ooapi/loader_test.clj index f74170ab..cde0da3a 100644 --- a/test/nl/surf/eduhub_rio_mapper/ooapi/loader_test.clj +++ b/test/nl/surf/eduhub_rio_mapper/ooapi/loader_test.clj @@ -41,5 +41,5 @@ ::ooapi/id "6456b864-c121-bb61-fda2-109251a1c777" :gateway-credentials (:gateway-credentials config)}] (binding [http-utils/*vcr* (vcr "test/fixtures/ooapi-loader" 1 "offering")] - (let [items (ooapi-loader (merge client-info request {:page-size 2}))] + (let [items (:items (ooapi-loader (merge client-info request {:page-size 2})))] (is (= 3 (count items)))))))