🚑 Add de-lazying within certain I/O dependent code paths

pmonks · Jun 14, 2024 · 33bb098 · 33bb098
1 parent a6d9766
commit 33bb098
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 40 deletions.
diff --git a/src/lice_comb/files.clj b/src/lice_comb/files.clj
@@ -100,19 +100,20 @@
     (let [zip-file (io/file zip)]
       (java.util.zip.ZipFile. zip-file)  ; This no-op forces validation of the zip file - ZipInputStream does not reliably perform validation
       (with-open [zip-is (java.util.zip.ZipInputStream. (io/input-stream zip-file))]
-        (loop [result {}
-               entry  (.getNextEntry zip-is)]
-          (if entry
-            (if (probable-license-file? entry)
-              (if-let [expressions (try
-                                     (file->expressions-info zip-is (lciu/filename entry))
-                                     (catch Exception e
-                                       (log/warn (str "Unexpected exception while processing " (lciu/filename zip) ":" (lciu/filename entry) " - ignoring") e)
-                                       nil))]
-                (recur (merge result expressions) (.getNextEntry zip-is))
+        (doall
+          (loop [result {}
+                 entry  (.getNextEntry zip-is)]
+            (if entry
+              (if (probable-license-file? entry)
+                (if-let [expressions (try
+                                       (file->expressions-info zip-is (lciu/filename entry))
+                                       (catch Exception e
+                                         (log/warn (str "Unexpected exception while processing " (lciu/filename zip) ":" (lciu/filename entry) " - ignoring") e)
+                                         nil))]
+                  (recur (merge result expressions) (.getNextEntry zip-is))
+                  (recur result (.getNextEntry zip-is)))
                 (recur result (.getNextEntry zip-is)))
-              (recur result (.getNextEntry zip-is)))
-            (when-not (empty? result) (lciei/prepend-source (lciu/filepath zip-file) result))))))))
+              (when-not (empty? result) (lciei/prepend-source (lciu/filepath zip-file) result)))))))))
 
 (defn zip->expressions
   "Returns a set of SPDX expressions (`String`s) for `zip`. See

diff --git a/src/lice_comb/impl/matching.clj b/src/lice_comb/impl/matching.clj
@@ -125,7 +125,7 @@
   "Returns an expressions-info map for the given license text, or nil if no
   matches are found."
   {:arglists '([text])}
-  type)
+  class)
 
 (defmethod text->expressions-info java.lang.String
   [s]
@@ -157,7 +157,7 @@
   [src]
   (when src
     (with-open [r (io/reader src)]
-      (text->expressions-info r))))
+      (doall (text->expressions-info r)))))
 
 (defn uri->expressions-info
   "Returns an expressions-info map for the given license uri, or nil if no

diff --git a/src/lice_comb/impl/utils.clj b/src/lice_comb/impl/utils.clj
@@ -153,7 +153,7 @@
 
 (defmulti readable-file?
   "Is f (a String, File, InputStream, or Reader) a readable file?"
-  type)
+  class)
 
 (defmethod readable-file? nil
   [_])
@@ -189,7 +189,7 @@
 (defmulti filepath
   "Returns the full path and name of the given file-like thing (String, File,
   ZipEntry, URI, URL)."
-  type)
+  class)
 
 (defmethod filepath nil
   [_])
@@ -206,10 +206,6 @@
         (filepath (io/as-url s))
         (filepath (io/file   s))))))
 
-(defmethod filepath java.util.zip.ZipEntry
-  [^java.util.zip.ZipEntry ze]
-  (.getName ze))
-
 (defmethod filepath java.net.URI
   [^java.net.URI uri]
   (str uri))
@@ -218,14 +214,18 @@
   [^java.net.URL url]
   (str url))
 
+(defmethod filepath java.util.zip.ZipEntry
+  [^java.util.zip.ZipEntry ze]
+  (.getName ze))
+
 (defmethod filepath java.io.InputStream
   [_]
   (throw (ex-info "Cannot determine filepath of an InputStream - did you forget to provide it separately?" {})))
 
 (defmulti filename
   "Returns just the name component of the given file-like thing (String, File,
   ZipEntry, URI, URL), excluding any parents."
-  type)
+  class)
 
 (defmethod filename nil
   [_])
@@ -242,10 +242,6 @@
         (filename (io/as-url s))
         (filename (io/file s))))))
 
-(defmethod filename java.util.zip.ZipEntry
-  [^java.util.zip.ZipEntry ze]
-  (filename (.getName ze)))
-
 (defmethod filename java.net.URI
   [^java.net.URI uri]
   (filename (.getPath uri)))
@@ -254,6 +250,10 @@
   [^java.net.URL url]
   (filename (.getPath url)))
 
+(defmethod filename java.util.zip.ZipEntry
+  [^java.util.zip.ZipEntry ze]
+  (filename (.getName ze)))
+
 (defmethod filename java.io.InputStream
   [_]
   (throw (ex-info "Cannot determine filename of an InputStream - did you forget to provide it separately?" {})))

diff --git a/src/lice_comb/maven.clj b/src/lice_comb/maven.clj
@@ -207,19 +207,21 @@
   [group-id artifact-id]
   (when-let [metadata-uri (ga->metadata-uri group-id artifact-id)]
     (with-open [metadata-is (io/input-stream metadata-uri)]
-      (let [metadata-xml (xml/parse metadata-is)]
-        (if-let [latest-version (xml-find-first-string metadata-xml [:metadata :versioning :latest])]
-          latest-version
-          (last (xi/find-all metadata-xml [:metadata :versioning :versions :version])))))))
+      (doall
+        (let [metadata-xml (xml/parse metadata-is)]
+          (if-let [latest-version (xml-find-first-string metadata-xml [:metadata :versioning :latest])]
+            latest-version
+            (last (xi/find-all metadata-xml [:metadata :versioning :versions :version]))))))))
 
 (defn ga-release-version
   "Determines the release version (if any) of the given GA as a `String`, or
   `nil` if it cannot be determined or the GA doesn't have a released version."
   [group-id artifact-id]
   (when-let [metadata-uri (ga->metadata-uri group-id artifact-id)]
     (with-open [metadata-is (io/input-stream metadata-uri)]
-      (let [metadata-xml (xml/parse metadata-is)]
-        (xml-find-first-string metadata-xml [:metadata :versioning :release])))))
+      (doall
+        (let [metadata-xml (xml/parse metadata-is)]
+          (xml-find-first-string metadata-xml [:metadata :versioning :release]))))))
 
 (defn- snapshot-version?
   "Is version a SNAPSHOT?"
@@ -234,10 +236,11 @@
   (if (snapshot-version? version)
     (when-let [metadata-uri (gav->metadata-uri group-id artifact-id version)]
       (with-open [metadata-is (io/input-stream metadata-uri)]
-        (let [metadata-xml (xml/parse metadata-is)
-              timestamp    (xml-find-first-string metadata-xml [:metadata :versioning :snapshot :timestamp])
-              build-number (xml-find-first-string metadata-xml [:metadata :versioning :snapshot :buildNumber])]
-          (str (s/replace version #"(?i)SNAPSHOT" (str timestamp "-" build-number))))))
+        (doall
+          (let [metadata-xml (xml/parse metadata-is)
+                timestamp    (xml-find-first-string metadata-xml [:metadata :versioning :snapshot :timestamp])
+                build-number (xml-find-first-string metadata-xml [:metadata :versioning :snapshot :buildNumber])]
+            (str (s/replace version #"(?i)SNAPSHOT" (str timestamp "-" build-number)))))))
     version))
 
 (defn- release-version?
@@ -299,7 +302,7 @@
     (then search that page for 'licenses/license*')
   * throws on XML parsing error"
   {:arglists '([pom] [pom filepath])}
-  (fn [& args] (type (first args))))
+  (fn [& args] (class (first args))))
 
 ; Note: a few rare pom.xml files are missing the xmlns declation (e.g. software.amazon.ion/ion-java) - so we look for both namespaced and non-namespaced versions of all tags
 (defmethod pom->expressions-info java.io.InputStream
@@ -343,9 +346,10 @@
   ([pom filepath]
    (when pom
      (with-open [pom-is (io/input-stream pom)]
-       (if-let [expressions (pom->expressions-info pom-is filepath)]
-         expressions
-         (log/info (str "'" filepath "'") "contains no license information"))))))
+       (doall
+         (if-let [expressions (doall (pom->expressions-info pom-is filepath))]
+           expressions
+           (log/info (str "'" filepath "'") "contains no license information")))))))
 
 (defn pom->expressions
   "Returns a set of SPDX expressions (`String`s) for `pom`. See
@@ -374,7 +378,7 @@
    (when-let [version (or version (ga-latest-version group-id artifact-id))]
      (when-let [pom-uri (gav->pom-uri group-id artifact-id version)]
        (with-open [pom-is (io/input-stream pom-uri)]
-         (lciei/prepend-source (str group-id "/" artifact-id "@" version) (pom->expressions-info pom-is (str pom-uri))))))))
+         (doall (lciei/prepend-source (str group-id "/" artifact-id "@" version) (pom->expressions-info pom-is (str pom-uri)))))))))
 
 (defn gav->expressions
   "Returns a set of SPDX expressions (`String`s) for the given GA and