From 9320abe61d80e2246e54d1da248414f16da99f2f Mon Sep 17 00:00:00 2001 From: Jeremy Long Date: Fri, 29 Sep 2023 06:18:11 -0400 Subject: [PATCH] feat: gzip cache output --- .../client/nvd/NvdCveClient.java | 5 +++-- .../vulnz/cli/commands/CveCommand.java | 18 ++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java index a392f5a5..c754e367 100644 --- a/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java +++ b/open-vulnerability-clients/src/main/java/io/github/jeremylong/openvulnerability/client/nvd/NvdCveClient.java @@ -311,8 +311,9 @@ public Collection next() { this.indexesToRetrieve.remove(call.getStartIndex()); } catch (JsonProcessingException e) { return next(); -// throw new NvdApiException("Failed to parse JSON starting with: \"" + json.substring(0, 100) + "\"", -// e); + // throw new NvdApiException("Failed to parse JSON starting with: \"" + json.substring(0, 100) + + // "\"", + // e); } this.totalAvailable = current.getTotalResults(); lastUpdated = findLastUpdated(lastUpdated, current.getVulnerabilities()); diff --git a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java index 97280c40..38637cb1 100644 --- a/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java +++ b/vulnz/src/main/java/io/github/jeremylong/vulnz/cli/commands/CveCommand.java @@ -39,6 +39,7 @@ import java.io.File; import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.time.Year; @@ -50,6 +51,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import static com.diogonunes.jcolor.Ansi.colorize; @@ -233,12 +236,14 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop final String prefix = properties.get("prefix", "nvdcve-"); // load existing cached files for (int year = 2002; year <= Year.now().getValue(); year++) { - File file = new File(properties.getDirectory(), prefix + year + ".json"); + File file = new File(properties.getDirectory(), prefix + year + ".json.gz"); cves.put(Integer.toString(year), new HashMap<>()); if (file.isFile()) { CveApiJson20 data; - try (InputStream in = new FileInputStream(file)) { - data = objectMapper.readValue(in, CveApiJson20.class); + try (FileInputStream fileInputStream = new FileInputStream(file); + GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream); + ) { + data = objectMapper.readValue(gzipInputStream, CveApiJson20.class); } catch (IOException exception) { throw new CacheException("Unable to read cached data: " + file, exception); } @@ -266,7 +271,7 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop final String version = "2.0"; for (Map.Entry> entry : cves.entrySet()) { - File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json"); + File file = new File(properties.getDirectory(), prefix + entry.getKey() + ".json.gz"); List vulnerabilities = new ArrayList(entry.getValue().values()); vulnerabilities.sort((v1, v2) -> { return v1.getCve().getId().compareTo(v2.getCve().getId()); @@ -284,8 +289,9 @@ private Integer processRequest(NvdCveClientBuilder builder, CacheProperties prop properties.set("lastModifiedDate." + entry.getKey(), timestamp); CveApiJson20 data = new CveApiJson20(vulnerabilities.size(), 0, vulnerabilities.size(), format, version, timestamp, vulnerabilities); - try { - objectMapper.writeValue(file, data); + try (FileOutputStream fileOutputStream = new FileOutputStream(file); + GZIPOutputStream gzipOutputStream = new GZIPOutputStream(fileOutputStream);) { + objectMapper.writeValue(gzipOutputStream, data); } catch (IOException ex) { throw new CacheException("Unable to write cached data: " + file, ex); }