From 327ffd062066048c63b77c1907e2e917354a5504 Mon Sep 17 00:00:00 2001 From: Stan Verberkt <2913270+verberktstan@users.noreply.github.com> Date: Mon, 12 Feb 2024 14:43:55 +0100 Subject: [PATCH] EAV diff rows (#9) Implement cedric protocol with mem and csv implementation --- README.md | 36 +++++ deps.edn | 5 +- src/swark/authom.cljc | 73 +++------ src/swark/cedric.cljc | 295 +++++++++++++++++++++++++++++++++++++ src/swark/core.cljc | 28 ++-- src/swark/eav.cljc | 85 ----------- test/swark/authom_test.clj | 10 +- test/swark/cedric_test.clj | 108 ++++++++++++++ test/swark/core_test.clj | 4 +- test/swark/eav_test.clj | 108 -------------- testdata.csv | 5 + 11 files changed, 497 insertions(+), 260 deletions(-) create mode 100644 src/swark/cedric.cljc delete mode 100644 src/swark/eav.cljc create mode 100644 test/swark/cedric_test.clj delete mode 100644 test/swark/eav_test.clj create mode 100644 testdata.csv diff --git a/README.md b/README.md index fa71705..60860c9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,42 @@ Then you can use the Swark utility functions: ```(swark/key-by :id [{:id 1 :name "one"} {:id 2 :name "two"}])``` +## Example + +Let's say you want to store a user record, some credentials and check their credentials. +You can use swark.cedric for the persistence part, and swark.authom for the authentication part. + +1. Let's create/connect to a database via the Csv implementation and store db props related to users. + + ``` +(ns my.ns + (:require [swark.authom :as authom] + [swark.cedric :as cedric])) + +(def DB (cedric/Csv. "db.csv")) +(def PROPS (merge authom/CEDRIC-PROPS {:primary-key :user/id})) + ``` + +2. Create a new user record like so: + +``` +(def USER (-> DB (cedric/upsert-items PROPS [{:user/name "Readme User"}]) first)) +``` + +3. Store credentials by upserting the user + +``` +(let [user (authom/map-with-meta-token USER :user/id "pass" "SECRET")] + (cedric/upsert-items DB PROPS [user])) +``` + +4. Retrieve the user and check their credentials + +``` +(let [user (-> DB (cedric/read-items {::cedric/primary-key #{:user/id}}) first)] + (-> user (authom/map-check-meta-token :user/id "pass" "SECRET") assert)) +``` + ## Tests Run the tests with `clojure -X:test/run` diff --git a/deps.edn b/deps.edn index 66b4402..4e6fcc9 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,8 @@ {:deps ;; Clojure standard library - {org.clojure/clojure {:mvn/version "1.11.0"}} -:aliases + {org.clojure/clojure {:mvn/version "1.11.0"} + org.clojure/data.csv {:mvn/version "1.0.1"}} ;; NOTE: For testing CSV input/output only.. + :aliases {:repl/reloaded {:extra-deps {nrepl/nrepl {:mvn/version "1.0.0"} cider/cider-nrepl {:mvn/version "0.28.7"} diff --git a/src/swark/authom.cljc b/src/swark/authom.cljc index c5064e1..804d732 100644 --- a/src/swark/authom.cljc +++ b/src/swark/authom.cljc @@ -20,20 +20,15 @@ (assert secret) (->hash {::item item ::secret secret} pass))) -(comment - ;; TODO: Turn fiddle code into tests - (hash 1) - (hash-unordered-coll 1) - (->hash {:user/id 123}) - (->hash {:user/id 123} "password") - (->hash {:user/id 123} "password" "SECRET") - ) +(defn- restore-meta-token* + [item token] + (vary-meta item assoc ::token token)) (defn with-meta-token "Returns the item with the hashed token in it's metadata. `item` should implement IMeta, otherwise this simply returns nil." [item & [pass secret :as args]] (try - (vary-meta item assoc ::token (apply ->hash item args)) + (restore-meta-token* item (str (apply ->hash item args))) #?(:cljs (catch :default nil) :clj (catch Throwable _ nil)))) @@ -44,13 +39,7 @@ (-> m (get primary-key) assert) (merge (apply with-meta-token (select-keys m [primary-key]) args) m)) -(comment - ;; TODO: Turn fiddle code into tests - (-> {:user/id 123} with-meta-token meta) - (-> {:user/id 123} (with-meta-token "password") meta) - (-> {:user/id 123} (with-meta-token "password" "SECRET") meta) - ) - +;; Simply return the token from the Authom metadata (def meta-token (comp ::token meta)) (defn check-meta-token @@ -59,7 +48,7 @@ [item & [pass secret :as args]] (let [token (meta-token item)] (assert token) - (when (= token (apply ->hash item args)) + (when (= token (str (apply ->hash item args))) item))) (defn map-check-meta-token @@ -68,37 +57,23 @@ (-> m (get primary-key) assert) (apply check-meta-token (select-keys m [primary-key]) args)) -(comment - ;; TODO: Turn fiddle code into tests - (let [user (with-meta-token {:user/id 123} "password" "SECRET")] - {:valid (check-meta-token user "password" "SECRET") - :invalid (check-meta-token user "wrong-password" "SECRET")}) +(defn enrich-token + "Returns map with Authom's meta-token associated with ::token." + [map] + (-> map map? assert) + (let [token (meta-token map)] + (cond-> map + token (assoc ::token token)))) - ;; Example usage with SQL database via jdbc - (let [primary-key :user/id - user {:user/id 123 :user/name "User Name"} - user' (-> user - (select-keys [primary-key]) ; Generate meta token only with primary map-entry - (with-meta-token "pass") - (merge user)) - get-rows (juxt :user/id meta-token :user/name) ; Retrieve id, token and name from user record - rows (get-rows (merge user' user)) ; NOTE: The metadata is preserved from user' - upsert-query (into ["REPLACE INTO users(id,token,name) values(?,?,?)"] rows)] ; Construct a SQL query to upsert user rows in the database. - (with-open [connection (-> {:dbname "test"} jdbc/get-datasource jdbc/get-connection)] - (jdbc/execute! connection upsert-query))) +(defn restore-enriched-token + "Returns map with the value in ::token restored as meta-token." + [{::keys [token] :as map}] + (-> map map? assert) + (cond-> map + token (restore-meta-token* token) + token (dissoc ::token))) - ;; Example usage - update user in appdb - (let [primary-key :user/id - user {:user/id 123 :user/name "User Name"} - primary-val (get user primary-key) - user' (-> user - (select-keys [primary-key]) ; Generate meta token only with primary map-entry - (with-meta-token "pass") - (merge user)) - db {}] - (-> db - (update-in [:users primary-val] (partial merge user')) ; Update user in db - :users - (get primary-val) - meta-token)) - ) +;; NOTE: Default props to make swark.cedric serialize and parse Authom tokens automatically +(def CEDRIC-PROPS + {:pre-upsert-serializer enrich-token + :post-merge-parser restore-enriched-token}) diff --git a/src/swark/cedric.cljc b/src/swark/cedric.cljc new file mode 100644 index 0000000..3efb57c --- /dev/null +++ b/src/swark/cedric.cljc @@ -0,0 +1,295 @@ +(ns swark.cedric + (:require [swark.core :as swark] + [clojure.edn :as edn] + [clojure.set :as set] + [clojure.data :as data] + #?(:cljs [goog.date :as gd]) + #?(:clj [clojure.string :as str]) + #?(:clj [clojure.java.io :as io]) + [clojure.data.csv :as csv]) + #?(:clj (:import [java.time Instant]))) + +;; TODO: Test in cljs as well +;; TODO: Move back in time by filtering on txd (transaction's utc date) +;; TODO: Write rows to csv, and test with that. +;; TODO: Add some memoization with swark.core/memoire + +(defn- utc-now [] + #?(:cljs (.toUTCIsoString (gd/DateTime.)) + :clj (.toString (Instant/now)))) + +(defmulti value-serializer (juxt ::primary-key ::attribute)) +(defmethod value-serializer :default [_] swark/->str) + +(defmulti attribute-serializer ::primary-key) +(defmethod attribute-serializer :default [_] swark/->str) + +(defmulti primary-value-serializer ::primary-key) +(defmethod primary-value-serializer :default [_] swark/->str) + +(defmulti primary-key-serializer ::primary-key) +(defmethod primary-key-serializer :default [_] swark/->str) + +(defn- unparse + [row-item] + (-> row-item + (update ::primary-key (primary-key-serializer row-item)) + (update ::primary-value (primary-value-serializer row-item)) + (update ::attribute (attribute-serializer row-item)) + (update ::value (value-serializer row-item)) + (update ::flags #(when (seq %) (->> % (map swark/->str) (into []) str))))) + +(def ^:private entry->row (juxt ::tx-utc-at ::primary-key ::primary-value ::attribute ::value ::flags)) + +(defn serialize + [{:keys [primary-key flags] + :or {flags #{}}} item] + (let [entity (find item primary-key) + tx-utc-at (utc-now) + archived? (some-> flags ::archived) + item' (if archived? + (select-keys item [primary-key]) + (dissoc item primary-key))] + (assert entity) + (->> item' + (map (fn [[attribute value]] + {::tx-utc-at tx-utc-at + ::entity entity + ::primary-key primary-key + ::primary-value (get item primary-key) + ::attribute attribute + ::value value + ::flags (map name flags)})) + (map unparse) + (map entry->row)))) + +(defmulti value-parser (juxt ::primary-key ::attribute)) +(defmethod value-parser :default [_] identity) + +(defmulti attribute-parser ::primary-key) +(defmethod attribute-parser :default [_] keyword) + +(defmulti primary-value-parser ::primary-key) +(defmethod primary-value-parser :default [_] identity) + +(defmulti primary-key-parser ::primary-key) +(defmethod primary-key-parser :default [_] keyword) + +(def ^:private row->entry + (partial zipmap [::tx-utc-at ::primary-key ::primary-value ::attribute ::value ::flags])) + +(defn- parse-flags + [s] + (let [coll (edn/read-string s)] + (when (coll? coll) + (->> coll + (mapv (partial keyword (namespace ::this))) + set)))) + +(defn- parse-entry [entry] + (let [find-entity (juxt ::primary-key ::primary-value) + entry (as-> entry e + (update e ::primary-key (primary-key-parser e #_ntry)) + (update e ::primary-value (primary-value-parser e #_ntry)) + (update e ::attribute (attribute-parser e #_ntry)) + (update e ::value (value-parser e #_ntry)) + (update e ::flags parse-flags)) + entity (find-entity entry)] + (assoc entry ::entity entity))) + +(defn- entry->map [{::keys [attribute value entity flags] :as entry}] + {entity (with-meta (into {attribute value} [entity]) {::flags flags}) #_entry}) + +(defn- merge-entries [map1 {::keys [attribute] :as map2}] + (some-> map2 meta ::flags println) + (cond + (some-> map2 meta ::flags ::archived) + nil ; Return nil, this value is to be removed from the result later + + (some-> map2 meta ::flags ::deleted) + (dissoc map1 attribute) + + :else + (merge map1 map2))) + +(defn- filter-entry [props entry] + (if-not (seq props) + true + (let [props (->> props (filter (comp ifn? val)) (into {})) + keyseq (keys props) + keys (set/intersection (-> entry keys set) (-> props keys set)) + values (if (seq keys) + (apply juxt keys) + (fn [_] (vector nil)))] + (some->> (select-keys entry keyseq) + (merge-with #(%1 %2) props) + values + seq + (every? identity))))) + +(defn- filterer [props] + (filter (partial filter-entry props))) + +(defn merge-rows + "Return eagerly parsed and merged rows" + ([rows] + (merge-rows nil rows)) + ;; TODO: Make it possible to return only the items from the last tx. Or a specific tx? + ([{:keys [post-merge-parser] + :or {post-merge-parser identity} + :as props} rows] + (keep + (comp post-merge-parser val) ; Only keep vals of the db maps, and omit archived entries + (transduce + (comp + (map row->entry) + (map parse-entry) + (filterer props) + (map entry->map)) + (partial merge-with merge-entries) + rows)))) + +(defn- diff [primary-key & items] + (let [entity (find (apply merge items) primary-key) + [removed added _] (apply data/diff items) + removed (some->> removed + (remove (comp (or added {}) key)) + seq + (into {}))] + {::added added + ::removed removed})) + +(defn- diff-rows [{:keys [primary-key] :as props} & items] + (let [entity (select-keys (apply merge items) [primary-key]) + {::keys + [added removed]} (apply diff primary-key items)] + (concat + (serialize (assoc props :flags #{::deleted}) (merge removed entity)) + (serialize props (merge added entity))))) + +(defn- upsert-rows + [db-items {:keys [primary-key next-primary-val] + :or {next-primary-val swark/unid} + :as props} item] + (let [update? (contains? item primary-key) + next-pval #(->> db-items + (map (fn [item] (get item primary-key))) ; NOTE: primary-key doesn't have to be a keyword! + set + next-primary-val) + item (cond-> item + (not update?) (assoc primary-key (next-pval))) + entity (find item primary-key)] + (if update? + (diff-rows props (->> db-items (filter (comp #{entity} #(find % primary-key))) first) item) + (serialize props item)))) + +(defn upsert + [rows {:keys [pre-upsert-serializer primary-key] + :or {pre-upsert-serializer identity} + :as props} & items] + (assert (seq items)) + (let [primary-values (fn [items] (->> items (map #(get % primary-key)) set)) + db-items (merge-rows {::primary-key #{primary-key}} rows)] + (reduce + (fn [new-rows item] + ;; Take new rows from the other upserted items into account as well (for next-primary-val fn) + (let [db-items' (concat db-items (merge-rows {} new-rows))] + (concat new-rows (upsert-rows db-items' props item)))) + nil + (map pre-upsert-serializer items)))) + +(defn archive + [rows {:keys [primary-key] :as props} & items] + (assert (and (seq items) (every? #(get % primary-key) items))) + (mapcat (partial serialize (assoc props :flags #{::archived})) items)) + +(comment + ;; Step 1 make this work in memory! + (def DB (atom nil)) + (swap! DB (fn [db] + (concat db (upsert db {:primary-key :user/id + :next-primary-val swark/unid} {:user/name "Antilla"} {:user/name "Ben Hur"})))) + (merge-rows {::entity #{[:user/id "c"] [:user/id "09"]}} @DB) + (swap! DB (fn [db] + (concat db (upsert db {:primary-key :user/id} {:user/id "2" :user/name "VOID"})))) + #_(upsert @DB {:primary-key :user/id + :next-primary-val swark/unid} {:user/name "Antilla"} {:user/name "Ben Hur"}) + + (swap! DB (fn [db] + (concat db (archive db {:primary-key :user/id} {:user/id "bb"} {:user/id "f"})))) + + + ;; Step 2 Make this work with csv + ) + +;; Instead of CRUD, we have URA +;; Upsert = Create and Update +;; Read = Read +;; Archive = Delete + +(defprotocol Cedric + (upsert-items [this props items]) + (read-items [this props]) + (archive-items [this props items])) + +(defrecord Mem [rows-atom] + Cedric + (upsert-items [this {:keys [primary-key] :as props} items] + ;; TODO: Return only the items from this (the last) tx + (let [updated-pvals (seq (keep #(get % primary-key) items))] + (->> (swap! rows-atom (fn [rows] (concat rows (apply upsert rows props items)))) + (merge-rows (cond-> {::primary-key #{primary-key}} + updated-pvals (assoc ::primary-value (set updated-pvals))))))) + (read-items [this props] (merge-rows props @rows-atom)) + (archive-items [this {:keys [primary-key] :as props} items] + (swap! rows-atom (fn [rows] (concat rows (apply archive rows props items)))) + {::archived (count items)})) + +(defn- write-csv! [filename rows] + (with-open [writer (io/writer filename :append true)] + (csv/write-csv writer rows :separator \;))) + +(defn- open-or-create! [filename] + (loop [reader (swark/try? io/reader filename) + retries-left 3] + (if (or reader (zero? retries-left)) + reader + (do + (write-csv! filename []) + (recur (swark/try? io/reader filename) (dec retries-left)))))) + +(defn- read-csv [filename] + (with-open [reader (open-or-create! filename)] + (-> reader (csv/read-csv :separator \;) doall))) + +(defrecord Csv [filename] + Cedric + (upsert-items [this {:keys [primary-key] :as props} items] + (let [rows (read-csv filename) + new-rows (apply upsert rows props items) + updated-pvals (seq (keep #(get % primary-key) items))] + (write-csv! filename new-rows) + (merge-rows + (cond-> {::primary-key #{primary-key}} updated-pvals (assoc ::primary-value (set updated-pvals))) + new-rows))) + (read-items [this props] (merge-rows props (read-csv filename))) + (archive-items [this props items] + (let [rows (read-csv filename) + new-rows (apply archive rows props items)] + (write-csv! filename new-rows) + {::archived (count new-rows)}))) + +(comment + (def CM (Mem. (atom nil))) + (def CC (Csv. "testdata.csv")) + + (upsert-items CC {:primary-key :user/id} [{:user/name "Stan"} {:user/name "Corinne"} {:user/name "David"} {:user/name "Theodor"} {:user/name "Naomi"} {:user/name "Arnold"}]) + ;; TODO: What if this doesn't exist? + (upsert-items CC {:primary-key :user/id} [{:user/id "7890" :user/name "Naomi"}]) + ;; (read-items CM {::entity #{[:user/id "9"]}}) + (read-items #_CM CC {}) + (archive-items #_CM CC {:primary-key :user/id} [{:user/id "1"} {:user/id "9"}]) + @(:rows-atom CM) + + ) + diff --git a/src/swark/core.cljc b/src/swark/core.cljc index 3cb7098..5786aa4 100644 --- a/src/swark/core.cljc +++ b/src/swark/core.cljc @@ -95,24 +95,26 @@ (or (when (keyword? input) (->> ((juxt namespace name) input) - (keep identity) - (map ->str) - (str/join "/"))) - (when input - (some-> input name str/trim non-blank))))) + (keep identity) + (map ->str) + (str/join "/"))) + (let [stringify (if (try? name input) name str)] + (some-> input stringify str/trim non-blank))))) (defn unid "Returns a unique string that does is not yet contained in the existing set." ([] (-> (random-uuid) str)) ([existing] - (-> existing set? assert) - (reduce - (fn [s char] - (if (and s (-> s existing not) (-> s reverse first #{"-"} not)) - (reduced s) - (str s char))) - nil - (seq (unid))))) + (unid nil existing)) + ([{:keys [min-length] :or {min-length 1}} existing] + (-> existing set? assert) + (reduce + (fn [s char] + (if (and s (>= (count s) min-length) (-> s existing not) (-> s reverse first #{"-"} not)) + (reduced s) + (str s char))) + nil + (seq (unid))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Regarding keywords diff --git a/src/swark/eav.cljc b/src/swark/eav.cljc deleted file mode 100644 index d5c5c2f..0000000 --- a/src/swark/eav.cljc +++ /dev/null @@ -1,85 +0,0 @@ -(ns swark.eav - {:added "0.1.3" - :doc "Serialize and parse data as entity-atteibute-value rows."}) - -;; Storing data as Entity / Attribute / Value rows - -(defn- parse [f input] (cond-> input (ifn? f) f)) - -(defn ->rows - "Returns a sequence of vectors with [entity-attribute entity-value attribute value] - for each map-entry in map m." - ([m] (->rows m nil)) - ([m {primary-key :primary/key - parse-entity-attribute :entity/attribute - parse-entity-value :entity/value - parse-attribute :attribute - parse-value :value - :or {primary-key :id - parse-entity-attribute identity parse-entity-value identity - parse-attribute identity parse-value identity}}] - (let [entry (find m primary-key)] - (assert entry "Mapentry can't be found!") - (let [entry (mapv parse [parse-entity-attribute parse-entity-value] entry)] - (->> (dissoc m primary-key) - (map (partial mapv parse [parse-attribute parse-value])) - (map (partial into entry))))))) - -(defn- assert-ifn-vals - [props] - (when (seq props) - (doseq [[k f] props] - (-> f ifn? (assert (str k " does not implement IFn!")))))) - -(defn- parse-row - "Returns row as a map with :entity/attribute, :entity/value, :attribute & :value. Applies supplied parsers on the fly for thise mapentries. You can supply a value parser lookup via :value/parsers, if a parser can be found by [:entity/attribute :attribute], this is used to parse the :value of the row's eav-map." - ([row] - (parse-row nil row)) - ([{:value/keys [parsers] :as props} row] - (let [keyseq [:entity/attribute :entity/value :attribute :value] - props (select-keys props keyseq) - _ (assert-ifn-vals props) - item (->> row - (zipmap keyseq) - (merge-with parse props)) - ea-vector (juxt :entity/attribute :attribute) - v-parser (get parsers (ea-vector item))] - (cond-> item v-parser (update :value v-parser))))) - -(defn parser - [props] - (map (partial parse-row props))) - -(defn- filter-eav - [props eav-map] - (let [props (select-keys props (keys eav-map)) - keyseq (keys props) - filter-vals (when (seq keyseq) (apply juxt keyseq))] - (assert-ifn-vals props) - (if-not (seq props) - true ; Always match when there are no valid props to filter on. - (->> (select-keys eav-map keyseq) - (merge-with parse props) - filter-vals - (every? identity))))) - -(defn filterer - [props] - (filter (partial filter-eav props))) - -(defn- mapify - [{ea :entity/attribute - ev :entity/value - a :attribute - v :value}] - {[ea ev] {ea ev a v}}) - -(defn merge-rows - [parse-props filter-props rows] - (transduce - (comp - (parser parse-props) - (filterer filter-props) - (map mapify)) - (partial merge-with merge) - rows)) diff --git a/test/swark/authom_test.clj b/test/swark/authom_test.clj index 3dec307..4c9651a 100644 --- a/test/swark/authom_test.clj +++ b/test/swark/authom_test.clj @@ -4,11 +4,11 @@ (def ITEM [:some :data]) (def WITH-TOKEN (-> ITEM (sut/with-meta-token "password"))) -(def TOKEN 1446530582) +(def TOKEN "1446530582") (def USER #:user{:id 123 :fullname "User Name"}) (def USER-WITH-TOKEN (-> USER (sut/map-with-meta-token :user/id "password" "SECRET"))) -(def USER-TOKEN -301775488) +(def USER-TOKEN "-301775488") (t/deftest meta-token (t/testing "Returns the token if stored in metadata" @@ -27,3 +27,9 @@ (t/is (-> WITH-TOKEN (sut/check-meta-token "password" "WRONG_SECRET") not)) (t/is (-> USER-WITH-TOKEN (sut/map-check-meta-token :user/id "wrong-password" "SECRET") not)) (t/is (-> USER-WITH-TOKEN (sut/map-check-meta-token :user/id "password" "WRONG_SECRET") not)))) + +(t/deftest enrich-pipeline + (let [enriched (sut/enrich-token USER-WITH-TOKEN) + restored (sut/restore-enriched-token enriched)] + (t/is (= restored USER-WITH-TOKEN)) + (t/is (= (meta USER-WITH-TOKEN) (meta restored))))) diff --git a/test/swark/cedric_test.clj b/test/swark/cedric_test.clj new file mode 100644 index 0000000..a203eca --- /dev/null +++ b/test/swark/cedric_test.clj @@ -0,0 +1,108 @@ +(ns swark.cedric-test + (:require [clojure.edn :as edn] + [clojure.test :refer [are deftest is testing]] + [swark.cedric :as sut] + [swark.core :as swark]) + (:import [swark.cedric Mem Csv])) + +;; Parse :id record's category value as clojure object (int) +(remove-method sut/value-parser [:id :category]) +(defmethod sut/value-parser [:id :category] [_] + edn/read-string) + +;; Parse user record's gender value as a keyword +(remove-method sut/value-parser [:user/id :user/gender]) +(defmethod sut/value-parser [:user/id :user/gender] [_] + keyword) + +;; Parse primary-value as clojure object (int) +(remove-method sut/primary-value-parser :user/id) +(remove-method sut/primary-value-parser :id) +(defmethod sut/primary-value-parser :user/id [_] + edn/read-string) +(defmethod sut/primary-value-parser :id [_] + edn/read-string) + +(deftest pipeline-test + (let [user1 {:id 1 :username "Stan" :category 1} + user2 #:user{:id 2 :name "Nats" :gender :unknown} + rows1 (sut/serialize {:primary-key :id} user1) + rows2 (sut/serialize {:primary-key :user/id} user2)] + (are [result rows] (= result (->> rows + (map (partial drop 1)) + (map (partial take 4)))) + [["id" "1" "username" "Stan"] + ["id" "1" "category" "1"]] rows1 + [["user/id" "2" "user/name" "Nats"] + ["user/id" "2" "user/gender" "unknown"]] rows2) + (is + (= [{:username "Stan" :id 1 :category 1} + #:user{:name "Nats" :id 2 :gender :unknown}] + (sut/merge-rows (concat rows1 rows2)))))) + +(def ^:private NAMES #{"Alfa" "Bravo" "Charlie" "Delta" "Echo" "Foxtrot" "Golf" "Hotel" "India" "Juliett" "Kilo" "Lima" "Mike" "November" "Oscar" "Papa" "Quebec" "Romeo" "Sierra" "Tango" "Uniform" "Victor" "Whiskey" "X-ray" "Yankee" "Zulu"}) + +(defn- some-names + ([] + (some-names 2)) + ([n] + (assert (< n 26)) + (->> NAMES shuffle (take n)))) + +(deftest mem-implementation + (let [db (Mem. (atom nil)) + props {:primary-key :person/id} + the-names (some-names 25) + persons (map (partial assoc nil :person/name) the-names) + result (sut/upsert-items db props persons)] + ;; result + (testing "upsert-items" + (testing "returns the upserted items" + (is (-> result count (= 25))) + (is (->> result (map :person/name) set (= (set the-names)))))) + (let [new-names (some-names 3) + persons (->> result + shuffle + (take 3) + (map #(assoc %2 :person/name %1) new-names)) + updated (sut/upsert-items db props persons)] + (testing "returns the updated items" + (is (-> updated count #{3})) + (is (->> updated (map :person/name) set (= (set new-names)))))) + (let [persons (->> result + shuffle + (take 5)) + archived (sut/archive-items db props persons)] + (testing "returns the number of ::archived items" + (is (= {::sut/archived 5} archived)))) + (testing "returns all the items" + (is (-> db (sut/read-items {}) count #{20}))))) + +(deftest csv-implementation + (let [db (Csv. (str "/tmp/testdb-" (swark/unid) ".csv")) + props {:primary-key :person/id} + the-names (some-names 25) + persons (map (partial assoc nil :person/name) the-names) + result (sut/upsert-items db props persons)] + ;; result + (testing "upsert-items" + (testing "returns the upserted items" + (is (-> result count (= 25))) + (is (->> result (map :person/name) set (= (set the-names)))))) + (let [new-names (some-names 3) + persons (->> result + shuffle + (take 3) + (map #(assoc %2 :person/name %1) new-names)) + updated (sut/upsert-items db props persons)] + (testing "returns the updated items" + (is (-> updated count #{3})) + (is (->> updated (map :person/name) set (= (set new-names)))))) + (let [persons (->> result + shuffle + (take 5)) + archived (sut/archive-items db props persons)] + (testing "returns the number of ::archived items" + (is (= {::sut/archived 5} archived)))) + (testing "returns all the items" + (is (-> db (sut/read-items {}) count #{20}))))) diff --git a/test/swark/core_test.clj b/test/swark/core_test.clj index 966060e..4e44ade 100644 --- a/test/swark/core_test.clj +++ b/test/swark/core_test.clj @@ -68,7 +68,9 @@ (t/deftest unid (t/is (string? (sut/unid))) (t/is (-> #{"x"} sut/unid count #{1})) - (t/is (-> (reduce (fn [x _] (conj x (sut/unid x))) #{} (range 999)) count #{999}))) + (t/is (->> #{"xyzab"} (sut/unid {:min-length 5}) count (>= 5))) + (t/is (-> (reduce (fn [x _] (conj x (sut/unid x))) #{} (range 999)) count #{999})) + (t/is (-> (reduce (fn [x _] (conj x (sut/unid {:min-length 4} x))) #{} (range 999)) count #{999}))) (t/deftest ->keyword (t/are [result args] (= result (apply sut/->keyword args)) diff --git a/test/swark/eav_test.clj b/test/swark/eav_test.clj deleted file mode 100644 index 8300ac3..0000000 --- a/test/swark/eav_test.clj +++ /dev/null @@ -1,108 +0,0 @@ -(ns swark.eav-test - (:require [clojure.edn :as edn] - [clojure.string :as str] - [clojure.test :as t] - [swark.core :as swark] - [swark.eav :as sut])) - -(def USER {:id 1 :username "Arnold"}) -(def USER2 #:user{:id 2 :name "Arnold" :city "Birmingham"}) - -(t/deftest ->rows - (t/is (= [[:id 1 :username "Arnold"]] - (sut/->rows USER))) - (t/is (= [[:user/id 2 :user/name "Arnold"] - [:user/id 2 :user/city "Birmingham"]] - (sut/->rows USER2 {:primary/key :user/id}))) - (t/is (= [[:city/id 3 :city/name "Birmingham"] - [:city/id 4 :city/name "Cork"]] - (mapcat - #(sut/->rows % {:primary/key :city/id}) - [#:city{:id 3 :name "Birmingham"} - #:city{:id 4 :name "Cork"}]))) - (t/is (= [["user/id" "2" "user/name" "Arnold"] - ["user/id" "2" "user/city" "Birmingham"]] - (sut/->rows USER2 {:primary/key :user/id - :entity/attribute swark/->str - :entity/value str - :attribute swark/->str - :value name})))) - -(t/deftest parse-row - (t/is (= [{:entity/attribute :id :entity/value 1 :attribute :username :value "Arnold"}] - (map #'sut/parse-row (sut/->rows USER)))) - (t/is (= [{:entity/attribute :user/id :entity/value 2 :attribute :user/name :value "Arnold"} - {:entity/attribute :user/id :entity/value 2 :attribute :user/city :value "Birmingham"}] - (map #'sut/parse-row (sut/->rows USER2 {:primary/key :user/id})))) - (t/is (= [{:entity/attribute :id :entity/value 1 :attribute "username" :value "Arnold"}] - (map (partial #'sut/parse-row {:attribute name}) (sut/->rows USER)))) - (t/is (= [{:entity/attribute :user/id :entity/value :two :attribute "name" :value "Arnold"} - {:entity/attribute :user/id :entity/value :two :attribute "city" :value "Birmingham"}] - (map (partial #'sut/parse-row {:entity/value {2 :two} :attribute name}) (sut/->rows USER2 {:primary/key :user/id})))) - (t/is (= {:entity/attribute :user/id - :entity/value 2 - :attribute :user/type - :value :member} - (#'sut/parse-row - {:entity/attribute swark/->keyword - :entity/value edn/read-string - :attribute swark/->keyword - :value/parsers {[:user/id :user/type] swark/->keyword}} - ["user/id" "2" "user/type" "member"])))) - -(t/deftest filter-eav - (let [eav1 {:entity/attribute :id - :entity/value 1 - :attribute :username - :value "Arnold"} - eav2 {:entity/attribute :user/id - :entity/value 2 - :attribute :user/name - :value "Bert"}] - (t/are [result props item] (= result (#'sut/filter-eav props item)) - ;; Filtering based on (namespace part of) the :entity/attribute - false {:entity/attribute (comp #{"user"} namespace)} eav1 - true {:entity/attribute (comp #{"user"} namespace)} eav2 - - ;; Filtering on the :entity/attribute and :entity/value - true {:entity/attribute #{:id} :entity/value #{1}} eav1 - false {:entity/attribute #{:id} :entity/value #{1}} eav2 - - ;; Filtering on the :attribute - false {:attribute (comp #{"user"} namespace)} eav1 - true {:attribute (comp #{"user"} namespace)} eav2 - - ;; Filtering on the :value - true {:value (comp #(str/starts-with? % "a") str/lower-case)} eav1 - false {:value (comp #(str/starts-with? % "a") str/lower-case)} eav2 - - ;; Filtering on a combination of these - false {:entity/attribute #{:user/id} :entity/value #{2} :attribute (comp #{"user"} namespace)} eav1 - true {:entity/attribute #{:user/id} :entity/value #{2} :attribute (comp #{"user"} namespace)} eav2 - - ;; Always match, when unsupported props are supplied - true {:something "else"} eav1))) - -(t/deftest merge-rows - (t/testing "Parse, filter and merge rows from a line-seq (e.g. csv-rows)" - (t/is (= {[:id 1] {:id 1 :username "Arnold"} - [:user/id 2] #:user{:id 2 :name "Bert"}} - (sut/merge-rows - {:entity/attribute keyword :entity/value edn/read-string :attribute keyword :value identity} - nil - [["id" "1" "username" "Arnold"] - ["user/id" "2" "user/name" "Bert"]]))) - - (t/is (= {[:id 1] {:id 1 :username "Arnold"}} - (sut/merge-rows - {:entity/attribute keyword :entity/value edn/read-string :attribute keyword :value identity} - {:entity/value #{1}} - [["id" "1" "username" "Arnold"] - ["user/id" "2" "user/name" "Bert"]]))) - - (t/is (= {[:user/id 2] #:user{:id 2 :name "Bert"}} - (sut/merge-rows - {:entity/attribute keyword :entity/value edn/read-string :attribute keyword :value identity} - {:entity/attribute (comp #{"user"} namespace)} - [["id" "1" "username" "Arnold"] - ["user/id" "2" "user/name" "Bert"]]))))) diff --git a/testdata.csv b/testdata.csv new file mode 100644 index 0000000..025c478 --- /dev/null +++ b/testdata.csv @@ -0,0 +1,5 @@ +id,1,name,Arnold, +id,1,city,Birmingham, +id,2,name,Christoff, +id,2,city,Durness, +id,1,name,Arnold,swark.eav/removed \ No newline at end of file