feat: gzip cache output

jeremylong · Sep 29, 2023 · 9320abe · 9320abe
1 parent 673cd1b
commit 9320abe
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 8 deletions.
diff --git a/...clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java b/...clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java
@@ -311,8 +311,9 @@ public Collection<DefCveItem> next() {
                     this.indexesToRetrieve.remove(call.getStartIndex());
                 } catch (JsonProcessingException e) {
                     return next();
-//                    throw new NvdApiException("Failed to parse JSON starting with: \"" + json.substring(0, 100) + "\"",
-//                            e);
+                    // throw new NvdApiException("Failed to parse JSON starting with: \"" + json.substring(0, 100) +
+                    // "\"",
+                    // e);
                 }
                 this.totalAvailable = current.getTotalResults();
                 lastUpdated = findLastUpdated(lastUpdated, current.getVulnerabilities());

diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java
@@ -39,6 +39,7 @@
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.time.Year;
@@ -50,6 +51,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 import static com.diogonunes.jcolor.Ansi.colorize;
 
@@ -233,12 +236,14 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop
         final String prefix = properties.get("prefix", "nvdcve-");
         // load existing cached files
         for (int year = 2002; year <= Year.now().getValue(); year++) {
-            File file = new File(properties.getDirectory(), prefix + year + ".json");
+            File file = new File(properties.getDirectory(), prefix + year + ".json.gz");
             cves.put(Integer.toString(year), new HashMap<>());
             if (file.isFile()) {
                 CveApiJson20 data;
-                try (InputStream in = new FileInputStream(file)) {
-                    data = objectMapper.readValue(in, CveApiJson20.class);
+                try (FileInputStream fileInputStream = new FileInputStream(file);
+                     GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream);
+                ) {
+                    data = objectMapper.readValue(gzipInputStream, CveApiJson20.class);
                 } catch (IOException exception) {
                     throw new CacheException("Unable to read cached data: " + file, exception);
                 }
@@ -266,7 +271,7 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop
         final String version = "2.0";
 
         for (Map.Entry<String, HashMap<String, DefCveItem>> entry : cves.entrySet()) {
-            File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json");
+            File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json.gz");
             List<DefCveItem> vulnerabilities = new ArrayList(entry.getValue().values());
             vulnerabilities.sort((v1, v2) -> {
                 return v1.getCve().getId().compareTo(v2.getCve().getId());
@@ -284,8 +289,9 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop
             properties.set("lastModifiedDate." + entry.getKey(), timestamp);
             CveApiJson20 data = new CveApiJson20(vulnerabilities.size(), 0, vulnerabilities.size(), format, version,
                     timestamp, vulnerabilities);
-            try {
-                objectMapper.writeValue(file, data);
+            try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+                    GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);) {
+                objectMapper.writeValue(gzipOutputStream, data);
             } catch (IOException ex) {
                 throw new CacheException("Unable to write cached data: " + file, ex);
             }