From f303c34e6d6fb3c896a1426f524815b75c16540b Mon Sep 17 00:00:00 2001 From: Hans Aikema Date: Sun, 15 Dec 2024 21:02:42 +0100 Subject: [PATCH] fix: Make the HTTP-Client use pre-emptive authentication for configured server credentials and extend HTTPClient usage to Nexus search Fixes #7108 Fixes #7253 --- .../analyzer/NexusAnalyzer.java | 3 +- .../data/central/CentralSearch.java | 25 +- .../data/nexus/NexusV2Search.java | 166 +++++------- .../data/nexus/NexusV3Search.java | 245 ++++++++++-------- .../data/update/KnownExploitedDataSource.java | 3 +- .../maven/BaseDependencyCheckMojo.java | 82 ++++-- src/site/markdown/data/proxy.md | 22 +- src/test/manual-test-proxy-auth/Readme.md | 2 +- .../debian.conf.override | 2 + .../dependencycheck/utils/Downloader.java | 205 ++++++++++----- ...plicitCharsetToStringResponseHandler.java} | 14 +- .../utils/ToXMLDocumentResponseHandler.java | 41 +++ 12 files changed, 475 insertions(+), 335 deletions(-) rename utils/src/main/java/org/owasp/dependencycheck/utils/{ExplicitEncodingToStringResponseHandler.java => ExplicitCharsetToStringResponseHandler.java} (69%) create mode 100644 utils/src/main/java/org/owasp/dependencycheck/utils/ToXMLDocumentResponseHandler.java diff --git a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java index ce3e5313db3..2b1014a113d 100644 --- a/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java +++ b/core/src/main/java/org/owasp/dependencycheck/analyzer/NexusAnalyzer.java @@ -321,8 +321,7 @@ public void analyzeDependency(Dependency dependency, Engine engine) throws Analy */ public boolean useProxy() { try { - return getSettings().getString(Settings.KEYS.PROXY_SERVER) != null - && getSettings().getBoolean(Settings.KEYS.ANALYZER_NEXUS_USES_PROXY); + return getSettings().getBoolean(Settings.KEYS.ANALYZER_NEXUS_USES_PROXY); } catch (InvalidSettingException ise) { LOGGER.warn("Failed to parse proxy settings.", ise); return false; diff --git a/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java b/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java index 99b7491378d..0b598684e99 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/central/CentralSearch.java @@ -18,7 +18,6 @@ package org.owasp.dependencycheck.data.central; import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; -import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.message.BasicHeader; import org.owasp.dependencycheck.utils.DownloadFailedException; import org.owasp.dependencycheck.utils.Downloader; @@ -26,15 +25,12 @@ import org.owasp.dependencycheck.utils.TooManyRequestsException; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.List; import javax.annotation.concurrent.ThreadSafe; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; @@ -44,12 +40,11 @@ import org.owasp.dependencycheck.data.cache.DataCacheFactory; import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.XmlUtils; +import org.owasp.dependencycheck.utils.ToXMLDocumentResponseHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; import org.w3c.dom.NodeList; -import org.xml.sax.SAXException; /** * Class of methods to search Maven Central via Central. @@ -162,19 +157,7 @@ public List searchSha1(String sha1) throws IOException, TooManyRe // JSON would be more elegant, but there's not currently a dependency // on JSON, so don't want to add one just for this final BasicHeader acceptHeader = new BasicHeader("Accept", "application/xml"); - final AbstractHttpClientResponseHandler handler = new AbstractHttpClientResponseHandler<>() { - @Override - public Document handleEntity(HttpEntity entity) throws IOException { - try (InputStream in = entity.getContent()) { - final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); - return builder.parse(in); - } catch (ParserConfigurationException | SAXException | IOException e) { - // Anything else is jacked up XML stuff that we really can't recover from well - final String errorMessage = "Failed to parse MavenCentral XML Response: " + e.getMessage(); - throw new IOException(errorMessage, e); - } - } - }; + final AbstractHttpClientResponseHandler handler = new ToXMLDocumentResponseHandler(); try { final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(acceptHeader), useProxy); final boolean missing = addMavenArtifacts(doc, result); @@ -194,6 +177,9 @@ public Document handleEntity(HttpEntity entity) throws IOException { } catch (ResourceNotFoundException | DownloadFailedException e) { final String errorMessage = "Could not connect to MavenCentral " + e.getMessage(); throw new IOException(errorMessage, e); + } catch (URISyntaxException e) { + final String errorMessage = "Could not convert central search URL to a URI " + e.getMessage(); + throw new IOException(errorMessage, e); } if (cache != null) { cache.put(sha1, result); @@ -267,4 +253,5 @@ private boolean isInvalidURL(String url) { } return false; } + } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV2Search.java b/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV2Search.java index d1242a77dbb..3c8766fceb9 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV2Search.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV2Search.java @@ -19,25 +19,29 @@ import java.io.FileNotFoundException; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.MalformedURLException; +import java.net.URISyntaxException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.util.Base64; +import java.util.List; import javax.annotation.concurrent.ThreadSafe; -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; + +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.message.BasicHeader; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; -import org.owasp.dependencycheck.utils.XmlUtils; +import org.owasp.dependencycheck.utils.ToXMLDocumentResponseHandler; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; -import org.xml.sax.SAXException; /** * Class of methods to search Nexus repositories. @@ -94,118 +98,76 @@ public MavenArtifact searchSha1(String sha1) throws IOException { LOGGER.debug("Searching Nexus url {}", url); - // Determine if we need to use a proxy. The rules: - // 1) If the proxy is set, AND the setting is set to true, use the proxy - // 2) Otherwise, don't use the proxy (either the proxy isn't configured, - // or proxy is specifically set to false - final HttpURLConnection conn; - final URLConnectionFactory factory = new URLConnectionFactory(settings); - conn = factory.createHttpURLConnection(url, useProxy); - conn.setDoOutput(true); - final String authHeader = buildHttpAuthHeaderValue(); - if (!authHeader.isEmpty()) { - conn.addRequestProperty("Authorization", authHeader); - } - - // JSON would be more elegant, but there's not currently a dependency - // on JSON, so don't want to add one just for this - conn.addRequestProperty("Accept", "application/xml"); - conn.connect(); - - switch (conn.getResponseCode()) { - case 200: - try { - final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); - final Document doc = builder.parse(conn.getInputStream()); - final XPath xpath = XPathFactory.newInstance().newXPath(); - final String groupId = xpath - .evaluate( - "/org.sonatype.nexus.rest.model.NexusArtifact/groupId", - doc); - final String artifactId = xpath.evaluate( - "/org.sonatype.nexus.rest.model.NexusArtifact/artifactId", + try { + // JSON would be more elegant, but there's not currently a dependency + // on JSON, so don't want to add one just for this + final ToXMLDocumentResponseHandler handler = new ToXMLDocumentResponseHandler(); + final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(new BasicHeader(HttpHeaders.ACCEPT, + ContentType.APPLICATION_XML))); + final XPath xpath = XPathFactory.newInstance().newXPath(); + final String groupId = xpath + .evaluate( + "/org.sonatype.nexus.rest.model.NexusArtifact/groupId", doc); - final String version = xpath - .evaluate( - "/org.sonatype.nexus.rest.model.NexusArtifact/version", - doc); - final String link = xpath - .evaluate( - "/org.sonatype.nexus.rest.model.NexusArtifact/artifactLink", - doc); - final String pomLink = xpath - .evaluate( - "/org.sonatype.nexus.rest.model.NexusArtifact/pomLink", - doc); - final MavenArtifact ma = new MavenArtifact(groupId, artifactId, version); - if (link != null && !link.isEmpty()) { - ma.setArtifactUrl(link); - } - if (pomLink != null && !pomLink.isEmpty()) { - ma.setPomUrl(pomLink); + final String artifactId = xpath.evaluate( + "/org.sonatype.nexus.rest.model.NexusArtifact/artifactId", + doc); + final String version = xpath + .evaluate( + "/org.sonatype.nexus.rest.model.NexusArtifact/version", + doc); + final String link = xpath + .evaluate( + "/org.sonatype.nexus.rest.model.NexusArtifact/artifactLink", + doc); + final String pomLink = xpath + .evaluate( + "/org.sonatype.nexus.rest.model.NexusArtifact/pomLink", + doc); + final MavenArtifact ma = new MavenArtifact(groupId, artifactId, version); + if (link != null && !link.isEmpty()) { + ma.setArtifactUrl(link); + } + if (pomLink != null && !pomLink.isEmpty()) { + ma.setPomUrl(pomLink); + } + return ma; + } catch (DownloadFailedException | TooManyRequestsException e) { + if (LOGGER.isDebugEnabled()) { + int responseCode = -1; + String responseMessage = ""; + if (e.getCause() instanceof HttpResponseException) { + final HttpResponseException cause = (HttpResponseException) e.getCause(); + responseCode = cause.getStatusCode(); + responseMessage = cause.getReasonPhrase(); } - return ma; - } catch (ParserConfigurationException | IOException | SAXException | XPathExpressionException e) { - // Anything else is jacked-up XML stuff that we really can't recover - // from well - throw new IOException(e.getMessage(), e); + LOGGER.debug("Could not connect to Nexus received response code: {} {}", + responseCode, responseMessage); } - case 404: - throw new FileNotFoundException("Artifact not found in Nexus"); - default: - LOGGER.debug("Could not connect to Nexus received response code: {} {}", - conn.getResponseCode(), conn.getResponseMessage()); throw new IOException("Could not connect to Nexus"); + } catch (ResourceNotFoundException e) { + throw new FileNotFoundException("Artifact not found in Nexus"); + } catch (XPathExpressionException | URISyntaxException e) { + throw new IOException(e.getMessage(), e); } } @Override public boolean preflightRequest() { - final HttpURLConnection conn; try { final URL url = new URL(rootURL, "status"); - final URLConnectionFactory factory = new URLConnectionFactory(settings); - conn = factory.createHttpURLConnection(url, useProxy); - conn.addRequestProperty("Accept", "application/xml"); - final String authHeader = buildHttpAuthHeaderValue(); - if (!authHeader.isEmpty()) { - conn.addRequestProperty("Authorization", authHeader); - } - conn.connect(); - if (conn.getResponseCode() != 200) { - LOGGER.warn("Expected 200 result from Nexus, got {}", conn.getResponseCode()); - return false; - } - final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); - - final Document doc = builder.parse(conn.getInputStream()); + final ToXMLDocumentResponseHandler handler = new ToXMLDocumentResponseHandler(); + final Document doc = Downloader.getInstance().fetchAndHandle(url, handler, List.of(new BasicHeader(HttpHeaders.ACCEPT, + ContentType.APPLICATION_XML))); if (!"status".equals(doc.getDocumentElement().getNodeName())) { - LOGGER.warn("Expected root node name of status, got {}", doc.getDocumentElement().getNodeName()); + LOGGER.warn("Pre-flight request to Nexus failed; expected root node name of status, got {}", doc.getDocumentElement().getNodeName()); return false; } - } catch (IOException | ParserConfigurationException | SAXException e) { + } catch (IOException | TooManyRequestsException | ResourceNotFoundException | URISyntaxException e) { LOGGER.warn("Pre-flight request to Nexus failed: ", e); return false; } return true; } - /** - * Constructs the base64 encoded basic authentication header value. - * - * @return the base64 encoded basic authentication header value - */ - private String buildHttpAuthHeaderValue() { - final String user = settings.getString(Settings.KEYS.ANALYZER_NEXUS_USER, ""); - final String pass = settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD, ""); - String result = ""; - if (user.isEmpty() || pass.isEmpty()) { - LOGGER.debug("Skip authentication as user and/or password for nexus is empty"); - } else { - final String auth = user + ':' + pass; - final String base64Auth = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); - result = "Basic " + base64Auth; - } - return result; - } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV3Search.java b/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV3Search.java index 058b5329a38..f8de9680976 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV3Search.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/nexus/NexusV3Search.java @@ -17,8 +17,19 @@ */ package org.owasp.dependencycheck.data.nexus; +import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.message.BasicHeader; +import org.jetbrains.annotations.Nullable; +import org.owasp.dependencycheck.utils.DownloadFailedException; +import org.owasp.dependencycheck.utils.Downloader; +import org.owasp.dependencycheck.utils.ResourceNotFoundException; import org.owasp.dependencycheck.utils.Settings; -import org.owasp.dependencycheck.utils.URLConnectionFactory; +import org.owasp.dependencycheck.utils.TooManyRequestsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,19 +38,20 @@ import javax.json.JsonArray; import javax.json.JsonObject; import javax.json.JsonReader; -import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; -import java.net.HttpURLConnection; +import java.io.InputStreamReader; +import java.io.StringReader; import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Base64; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * Class of methods to search Nexus v3 repositories. @@ -100,10 +112,11 @@ public MavenArtifact searchSha1(String sha1) throws IOException { } final List collectedMatchingArtifacts = new ArrayList<>(1); - - String continuationToken = retrievePageAndAddMatchingArtifact(collectedMatchingArtifacts, sha1, null); - while (continuationToken != null && collectedMatchingArtifacts.isEmpty()) { - continuationToken = retrievePageAndAddMatchingArtifact(collectedMatchingArtifacts, sha1, continuationToken); + try (CloseableHttpClient client = Downloader.getInstance().getHttpClient(useProxy)) { + String continuationToken = retrievePageAndAddMatchingArtifact(client, collectedMatchingArtifacts, sha1, null); + while (continuationToken != null && collectedMatchingArtifacts.isEmpty()) { + continuationToken = retrievePageAndAddMatchingArtifact(client, collectedMatchingArtifacts, sha1, continuationToken); + } } if (collectedMatchingArtifacts.isEmpty()) { throw new FileNotFoundException("Artifact not found in Nexus"); @@ -112,8 +125,8 @@ public MavenArtifact searchSha1(String sha1) throws IOException { } } - private String retrievePageAndAddMatchingArtifact(List collectedMatchingArtifacts, String sha1, String continuationToken) - throws IOException { + private String retrievePageAndAddMatchingArtifact(CloseableHttpClient client, List collectedMatchingArtifacts, String sha1, + @Nullable String continuationToken) throws IOException { final URL url; LOGGER.debug("Search with continuation token {}", continuationToken); if (continuationToken == null) { @@ -129,131 +142,145 @@ private String retrievePageAndAddMatchingArtifact(List collectedM // 1) If the proxy is set, AND the setting is set to true, use the proxy // 2) Otherwise, don't use the proxy (either the proxy isn't configured, // or proxy is specifically set to false - final HttpURLConnection conn; - final URLConnectionFactory factory = new URLConnectionFactory(settings); - conn = factory.createHttpURLConnection(url, useProxy); - conn.setDoOutput(true); - final String authHeader = buildHttpAuthHeaderValue(); - if (!authHeader.isEmpty()) { - conn.addRequestProperty("Authorization", authHeader); + final NexusV3SearchResponseHandler handler = new NexusV3SearchResponseHandler(collectedMatchingArtifacts, sha1, acceptedClassifiers); + try { + return Downloader.getInstance().fetchAndHandle(client, url, handler, List.of(new BasicHeader(HttpHeaders.ACCEPT, + ContentType.APPLICATION_JSON))); + } catch (TooManyRequestsException | ResourceNotFoundException | DownloadFailedException e) { + if (LOGGER.isDebugEnabled()) { + int responseCode = -1; + String responseMessage = ""; + if (e.getCause() instanceof HttpResponseException) { + final HttpResponseException cause = (HttpResponseException) e.getCause(); + responseCode = cause.getStatusCode(); + responseMessage = cause.getReasonPhrase(); + } + LOGGER.debug("Could not connect to Nexus received response code: {} {}", + responseCode, responseMessage); + } + throw new IOException("Could not connect to Nexus", e); } + } - conn.addRequestProperty("Accept", "application/json"); - conn.connect(); - final String nextContinuationToken; - if (conn.getResponseCode() == 200) { - nextContinuationToken = parseResponse(conn, sha1, collectedMatchingArtifacts); - } else { - LOGGER.debug("Could not connect to Nexus received response code: {} {}", - conn.getResponseCode(), conn.getResponseMessage()); - throw new IOException(String.format("Could not connect to Nexus, HTTP response code %d", conn.getResponseCode())); + private static final class NexusV3SearchResponseHandler extends AbstractHttpClientResponseHandler { + + /** + * The list to which matching artifacts are to be added + */ + private final List matchingArtifacts; + /** + * The sha1 for which the search results are being handled + */ + private final String sha1; + /** + * The classifiers to be accepted + */ + private final Set acceptedClassifiers; + + private NexusV3SearchResponseHandler(List matchingArtifacts, String sha1, Set acceptedClassifiers) { + this.matchingArtifacts = matchingArtifacts; + this.sha1 = sha1; + this.acceptedClassifiers = acceptedClassifiers; } - return nextContinuationToken; - } - private String parseResponse(HttpURLConnection conn, String sha1, List matchingArtifacts) throws IOException { - try (InputStream in = new BufferedInputStream(conn.getInputStream()); - JsonReader jsonReader = Json.createReader(in)) { - final JsonObject jsonResponse = jsonReader.readObject(); - final String continuationToken = jsonResponse.getString("continuationToken", null); - final JsonArray components = jsonResponse.getJsonArray("items"); - boolean found = false; - for (int i = 0; i < components.size() && !found; i++) { - boolean jarFound = false; - boolean pomFound = false; - String downloadUrl = null; - String groupId = null; - String artifactId = null; - String version = null; - String pomUrl = null; + @Override + public @Nullable String handleEntity(HttpEntity entity) throws IOException { + try (InputStream in = entity.getContent(); + InputStreamReader isReader = new InputStreamReader(in, StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(isReader); + ) { + final String jsonString = reader.lines().collect(Collectors.joining("\n")); + LOGGER.debug("JSON String was >>>{}<<<", jsonString); + final JsonObject jsonResponse; + try ( + StringReader stringReader = new StringReader(jsonString); + JsonReader jsonReader = Json.createReader(stringReader) + ) { + jsonResponse = jsonReader.readObject(); + } + LOGGER.debug("Response: {}", jsonResponse); + final JsonArray components = jsonResponse.getJsonArray("items"); + LOGGER.debug("Items: {}", components); + final String continuationToken = jsonResponse.getString("continuationToken", null); + boolean found = false; + for (int i = 0; i < components.size() && !found; i++) { + boolean jarFound = false; + boolean pomFound = false; + String downloadUrl = null; + String groupId = null; + String artifactId = null; + String version = null; + String pomUrl = null; - final JsonObject component = components.getJsonObject(i); + final JsonObject component = components.getJsonObject(i); - final String format = components.getJsonObject(0).getString("format", "unknown"); - if ("maven2".equals(format)) { - final JsonArray assets = component.getJsonArray("assets"); - for (int j = 0; !found && j < assets.size(); j++) { - final JsonObject asset = assets.getJsonObject(j); - final JsonObject checksums = asset.getJsonObject("checksum"); - final JsonObject maven2 = asset.getJsonObject("maven2"); - if (maven2 != null - && "jar".equals(maven2.getString("extension", null)) - && acceptedClassifiers.contains(maven2.getString("classifier", null)) - && checksums != null && sha1.equals(checksums.getString("sha1", null)) - ) { - downloadUrl = asset.getString("downloadUrl"); - groupId = maven2.getString("groupId"); - artifactId = maven2.getString("artifactId"); - version = maven2.getString("version"); + final String format = component.getString("format", "unknown"); + if ("maven2".equals(format)) { + LOGGER.debug("Checking Maven2 artifact for {}", component); + final JsonArray assets = component.getJsonArray("assets"); + for (int j = 0; !found && j < assets.size(); j++) { + final JsonObject asset = assets.getJsonObject(j); + LOGGER.debug("Checking {}", asset); + final JsonObject checksums = asset.getJsonObject("checksum"); + final JsonObject maven2 = asset.getJsonObject("maven2"); + if (maven2 != null) { + // logical names for the jar acceptance routine + final boolean shaMatch = checksums != null && sha1.equals(checksums.getString("sha1", null)); + final boolean hasAcceptedClassifier = acceptedClassifiers.contains(maven2.getString("classifier", null)); + final boolean isAJar = "jar".equals(maven2.getString("extension", null)); + LOGGER.debug("shaMatch {}", shaMatch); + LOGGER.debug("hasAcceptedClassifier {}", hasAcceptedClassifier); + LOGGER.debug("isAJar {}", isAJar); + if ( + isAJar + && hasAcceptedClassifier + && shaMatch + ) { + downloadUrl = asset.getString("downloadUrl"); + groupId = maven2.getString("groupId"); + artifactId = maven2.getString("artifactId"); + version = maven2.getString("version"); - jarFound = true; - } else if (maven2 != null && "pom".equals(maven2.getString("extension"))) { - pomFound = true; - pomUrl = asset.getString("downloadUrl"); + jarFound = true; + } else if ("pom".equals(maven2.getString("extension"))) { + LOGGER.debug("pom found {}", asset); + pomFound = true; + pomUrl = asset.getString("downloadUrl"); + } + } + if (pomFound && jarFound) { + found = true; + } } - if (pomFound && jarFound) { + if (found) { + matchingArtifacts.add(new MavenArtifact(groupId, artifactId, version, downloadUrl, pomUrl)); + } else if (jarFound) { + final MavenArtifact ma = new MavenArtifact(groupId, artifactId, version, downloadUrl); + ma.setPomUrl(MavenArtifact.derivePomUrl(artifactId, version, downloadUrl)); + matchingArtifacts.add(ma); found = true; } } - if (found) { - matchingArtifacts.add(new MavenArtifact(groupId, artifactId, version, downloadUrl, pomUrl)); - } else if (jarFound) { - final MavenArtifact ma = new MavenArtifact(groupId, artifactId, version, downloadUrl); - ma.setPomUrl(MavenArtifact.derivePomUrl(artifactId, version, downloadUrl)); - matchingArtifacts.add(ma); - found = true; - } } + return continuationToken; } - return continuationToken; } } @Override public boolean preflightRequest() { - final HttpURLConnection conn; try { final URL url = new URL(rootURL, "v1/status"); - final URLConnectionFactory factory = new URLConnectionFactory(settings); - conn = factory.createHttpURLConnection(url, useProxy); - conn.addRequestProperty("Accept", "application/json"); - final String authHeader = buildHttpAuthHeaderValue(); - if (!authHeader.isEmpty()) { - conn.addRequestProperty("Authorization", authHeader); - } - conn.connect(); - if (conn.getResponseCode() != 200) { - LOGGER.warn("Expected 200 result from Nexus, got {}", conn.getResponseCode()); - return false; - } - if (conn.getContentLength() != 0) { - LOGGER.warn("Expected empty OK response (content-length 0), got content-length {}", conn.getContentLength()); + final String response = Downloader.getInstance().fetchContent(url, useProxy, StandardCharsets.UTF_8); + if (response == null || !response.isEmpty()) { + LOGGER.warn("Expected empty OK response (content-length 0), got {}", response == null ? "null" : response.length()); return false; } - } catch (IOException e) { + } catch (IOException | TooManyRequestsException | ResourceNotFoundException e) { LOGGER.warn("Pre-flight request to Nexus failed: ", e); return false; } return true; } - /** - * Constructs the base64 encoded basic authentication header value. - * - * @return the base64 encoded basic authentication header value - */ - private String buildHttpAuthHeaderValue() { - final String user = settings.getString(Settings.KEYS.ANALYZER_NEXUS_USER, ""); - final String pass = settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD, ""); - String result = ""; - if (user.isEmpty() || pass.isEmpty()) { - LOGGER.debug("Skip authentication as user and/or password for nexus is empty"); - } else { - final String auth = user + ':' + pass; - final String base64Auth = Base64.getEncoder().encodeToString(auth.getBytes(StandardCharsets.UTF_8)); - result = "Basic " + base64Auth; - } - return result; - } - } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java index 5d2fee2f1bb..725c6a2df34 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/update/KnownExploitedDataSource.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.URISyntaxException; import java.net.URL; import java.sql.SQLException; @@ -101,7 +102,7 @@ public KnownExploitedVulnerabilitiesSchema handleEntity(HttpEntity entity) throw //all dates in the db are now stored in seconds as opposed to previously milliseconds. dbProperties.save(DatabaseProperties.KEV_LAST_CHECKED, Long.toString(System.currentTimeMillis() / 1000)); return true; - } catch (TooManyRequestsException | ResourceNotFoundException | IOException | DatabaseException | SQLException ex) { + } catch (TooManyRequestsException | ResourceNotFoundException | IOException | DatabaseException | SQLException | URISyntaxException ex) { throw new UpdateException(ex); } } diff --git a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java index 4778b26565b..c723582991d 100644 --- a/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java +++ b/maven/src/main/java/org/owasp/dependencycheck/maven/BaseDependencyCheckMojo.java @@ -131,6 +131,8 @@ public abstract class BaseDependencyCheckMojo extends AbstractMojo implements Ma * Pattern to include all files in a FileSet. */ private static final String INCLUDE_ALL = "**/*"; + public static final String PROTOCOL_HTTPS = "https"; + public static final String PROTOCOL_HTTP = "http"; /** * A flag indicating whether or not the Maven site is being generated. */ @@ -2207,31 +2209,36 @@ protected void populateSettings() { settings.setStringIfNotNull(Settings.KEYS.ANALYZER_YARN_PATH, pathToYarn); settings.setStringIfNotNull(Settings.KEYS.ANALYZER_PNPM_PATH, pathToPnpm); - // use global maven proxy if provided - final Proxy mavenProxy = getMavenProxy(); - if (mavenProxy != null) { - final String existing = System.getProperty("https.proxyHost"); - if (existing == null && mavenProxy.getHost() != null && !mavenProxy.getHost().isEmpty()) { - System.setProperty("https.proxyHost", mavenProxy.getHost()); - if (mavenProxy.getPort() > 0) { - System.setProperty("https.proxyPort", String.valueOf(mavenProxy.getPort())); - } - if (mavenProxy.getUsername() != null && !mavenProxy.getUsername().isEmpty()) { - System.setProperty("https.proxyUser", mavenProxy.getUsername()); - } - String password = mavenProxy.getPassword(); - if (password != null && !password.isEmpty()) { - try { - password = decryptPasswordFromSettings(password); - } catch (SecDispatcherException ex) { - password = handleSecDispatcherException("proxy", mavenProxy.getId(), password, ex); + // use global maven proxy if provided and system properties are not set + final Proxy mavenProxyHttp = getMavenProxy(PROTOCOL_HTTP); + final Proxy mavenProxyHttps = getMavenProxy(PROTOCOL_HTTPS); + String httpsNonProxyHosts = null; + String httpNonProxyHosts = null; + boolean proxySetFromMavenSettings = false; + if (mavenProxyHttps != null || mavenProxyHttp != null) { + final String existingHttps = System.getProperty("https.proxyHost"); + if (existingHttps == null) { + proxySetFromMavenSettings = true; + if (mavenProxyHttps != null) { + setProxyServerSysPropsFromMavenProxy(mavenProxyHttps, PROTOCOL_HTTPS); + if (mavenProxyHttps.getNonProxyHosts() != null && !mavenProxyHttps.getNonProxyHosts().isEmpty()) { + httpsNonProxyHosts = mavenProxyHttps.getNonProxyHosts(); } - System.setProperty("https.proxyPassword", password); - } - if (mavenProxy.getNonProxyHosts() != null && !mavenProxy.getNonProxyHosts().isEmpty()) { - System.setProperty("http.nonProxyHosts", mavenProxy.getNonProxyHosts()); + } else { + setProxyServerSysPropsFromMavenProxy(mavenProxyHttp, PROTOCOL_HTTPS); + httpsNonProxyHosts = mavenProxyHttp.getNonProxyHosts(); } } + final String existingHttp = System.getProperty("http.proxyHost"); + if (mavenProxyHttp != null && existingHttp == null) { + proxySetFromMavenSettings = true; + setProxyServerSysPropsFromMavenProxy(mavenProxyHttp, PROTOCOL_HTTP); + httpNonProxyHosts = mavenProxyHttp.getNonProxyHosts(); + } + if (proxySetFromMavenSettings) { + final String existingNonProxyHosts = System.getProperty("http.nonProxyHosts"); + System.setProperty("http.nonProxyHosts", mergeNonProxyHosts(existingNonProxyHosts, httpNonProxyHosts, httpsNonProxyHosts)); + } } else if (this.proxy != null && this.proxy.getHost() != null) { // or use configured settings.setString(Settings.KEYS.PROXY_SERVER, this.proxy.getHost()); @@ -2377,6 +2384,33 @@ protected void populateSettings() { } //CSON: MethodLength + private String mergeNonProxyHosts(String existingNonProxyHosts, String httpNonProxyHosts, String httpsNonProxyHosts) { + final HashSet mergedNonProxyHosts = new HashSet<>(); + mergedNonProxyHosts.addAll(Arrays.asList(existingNonProxyHosts.split("\\|"))); + mergedNonProxyHosts.addAll(Arrays.asList(httpNonProxyHosts.split("\\|"))); + mergedNonProxyHosts.addAll(Arrays.asList(httpsNonProxyHosts.split("\\|"))); + return String.join("|", mergedNonProxyHosts); + } + + private void setProxyServerSysPropsFromMavenProxy(Proxy mavenProxy, String protocol) { + System.setProperty(protocol + ".proxyHost", mavenProxy.getHost()); + if (mavenProxy.getPort() > 0) { + System.setProperty(protocol + ".proxyPort", String.valueOf(mavenProxy.getPort())); + } + if (mavenProxy.getUsername() != null && !mavenProxy.getUsername().isEmpty()) { + System.setProperty(protocol + ".proxyUser", mavenProxy.getUsername()); + } + String password = mavenProxy.getPassword(); + if (password != null && !password.isEmpty()) { + try { + password = decryptPasswordFromSettings(password); + } catch (SecDispatcherException ex) { + password = handleSecDispatcherException("proxy", mavenProxy.getId(), password, ex); + } + System.setProperty(protocol + ".proxyPassword", password); + } + } + /** * Retrieves the server credentials from the settings.xml, decrypts the * password, and places the values into the settings under the given key @@ -2530,7 +2564,7 @@ private void muteNoisyLoggers() { * * @return the maven proxy */ - private Proxy getMavenProxy() { + private Proxy getMavenProxy(String protocol) { if (mavenSettings != null) { final List proxies = mavenSettings.getProxies(); if (proxies != null && !proxies.isEmpty()) { @@ -2542,7 +2576,7 @@ private Proxy getMavenProxy() { } } else { for (Proxy aProxy : proxies) { - if (aProxy.isActive()) { + if (aProxy.isActive() && aProxy.getProtocol().equals(protocol)) { return aProxy; } } diff --git a/src/site/markdown/data/proxy.md b/src/site/markdown/data/proxy.md index 6e437ae103f..f106c996304 100644 --- a/src/site/markdown/data/proxy.md +++ b/src/site/markdown/data/proxy.md @@ -5,10 +5,10 @@ and, if required, will likely need to be **configured twice**. ## Java Properties -The go-forward proxy configuration is done using standard Java proxy configuration -settings which can be set using an environment variable `JAVA_TOOL_OPTIONS`. -See https://docs.oracle.com/javase/8/docs/technotes/guides/net/proxies.html for -more information. The properties that can be configured are: +The go-forward proxy configuration is done using Apache HTTPClient system property proxy configuration +which can be set using an environment variable `JAVA_TOOL_OPTIONS`. +See https://hc.apache.org/httpcomponents-client-5.4.x/current/httpclient5/apidocs/org/apache/hc/client5/http/impl/classic/HttpClientBuilder.html for +more information. The recommended properties that can be configured are: - https.proxyHost - https.proxyPort @@ -16,12 +16,26 @@ more information. The properties that can be configured are: - https.proxyPassword - http.nonProxyHosts +And in case of legacy URLs that are not (yet) exposing on https: +- http.proxyHost +- http.proxyPort +- http.proxyUser +- http.proxyPassword + + As example configuration would be: ```bash export JAVA_TOOL_OPTIONS="-Dhttps.proxyHost=my-proxy.internal -Dhttps.proxyPort=8083" ``` +If you have some custom internal URLs that are on plain http, but do require use of the proxy you should also add +the `http.*` properties for the proxy. + +```bash +export JAVA_TOOL_OPTIONS="-Dhttps.proxyHost=my-proxy.internal -Dhttps.proxyPort=8083 -Dhttp.proxyHost=my-proxy.internal -Dhttp.proxyPort=8083" +``` + ## Legacy configuration Legacy proxy configuration can be configured in any of the dependency-check integrations diff --git a/src/test/manual-test-proxy-auth/Readme.md b/src/test/manual-test-proxy-auth/Readme.md index df8de14ad0f..71200207687 100644 --- a/src/test/manual-test-proxy-auth/Readme.md +++ b/src/test/manual-test-proxy-auth/Readme.md @@ -29,7 +29,7 @@ shell (or make sure in a new shell that the same `JAVA_TOOL_OPTIONS` environment * Stop the docker container running squid-proxy (due to start with --rm the container will be deleted upon termination) ```shell - ./start-docker-squid-proxy-with-auth + ./stop-docker-squid-proxy-with-auth ``` * Unset JAVA_TOOL_OPTIONS or set it back to your regular value ```shell diff --git a/src/test/manual-test-proxy-auth/debian.conf.override b/src/test/manual-test-proxy-auth/debian.conf.override index 5c145d6de4b..e4a153a56aa 100644 --- a/src/test/manual-test-proxy-auth/debian.conf.override +++ b/src/test/manual-test-proxy-auth/debian.conf.override @@ -4,6 +4,8 @@ # Logs are managed by logrotate on Debian logfile_rotate 0 +# Increase log-level for Hypertext Transfer Protocol (HTTP) to see more detail on auth +debug_options ALL,1 11,2 # For extra security Debian packages only allow # localhost to use the proxy on new installs diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java index de2f955cec9..975e2d7f0ae 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/Downloader.java @@ -18,11 +18,14 @@ package org.owasp.dependencycheck.utils; import org.apache.hc.client5.http.HttpResponseException; +import org.apache.hc.client5.http.auth.AuthCache; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.Credentials; import org.apache.hc.client5.http.auth.CredentialsStore; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicAuthCache; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.auth.BasicScheme; import org.apache.hc.client5.http.impl.auth.SystemDefaultCredentialsProvider; import org.apache.hc.client5.http.impl.classic.BasicHttpClientResponseHandler; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -34,6 +37,7 @@ import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.Method; import org.apache.hc.core5.http.io.HttpClientResponseHandler; @@ -85,6 +89,17 @@ public final class Downloader { */ private final HttpClientBuilder httpClientBuilderExplicitNoproxy; + /** + * The Authentication cache for pre-emptive authentication. + * This gets filled with credentials from the settings in {@link #configure(Settings)}. + */ + private final AuthCache authCache = new BasicAuthCache(); + + /** + * The credentialsProvider for pre-emptive authentication. + * This gets filled with credentials from the settings in {@link #configure(Settings)}. + */ + private final SystemDefaultCredentialsProvider credentialsProvider = new SystemDefaultCredentialsProvider(); /** * The settings @@ -146,7 +161,6 @@ public static Downloader getInstance() { public void configure(Settings settings) throws InvalidSettingException { this.settings = settings; - final SystemDefaultCredentialsProvider credentialsProvider = new SystemDefaultCredentialsProvider(); if (settings.getString(Settings.KEYS.PROXY_SERVER) != null) { // Legacy proxy configuration present // So don't rely on the system properties for proxy; use the legacy settings configuration @@ -165,86 +179,88 @@ public void configure(Settings settings) throws InvalidSettingException { if (settings.getString(Settings.KEYS.PROXY_USERNAME) != null) { final String proxyuser = settings.getString(Settings.KEYS.PROXY_USERNAME); final char[] proxypass = settings.getString(Settings.KEYS.PROXY_PASSWORD).toCharArray(); + final HttpHost theProxy = new HttpHost(null, proxyHost, proxyPort); + final Credentials creds = new UsernamePasswordCredentials(proxyuser, proxypass); credentialsProvider.setCredentials( - new AuthScope(null, proxyHost, proxyPort, null, null), - new UsernamePasswordCredentials(proxyuser, proxypass) + new AuthScope(theProxy), + creds ); } } - tryAddRetireJSCredentials(settings, credentialsProvider); - tryAddHostedSuppressionCredentials(settings, credentialsProvider); - tryAddKEVCredentials(settings, credentialsProvider); - tryAddNexusAnalyzerCredentials(settings, credentialsProvider); - tryAddCentralAnalyzerCredentials(settings, credentialsProvider); - tryAddCentralContentCredentials(settings, credentialsProvider); - tryAddNVDApiDatafeed(settings, credentialsProvider); + tryAddRetireJSCredentials(settings, credentialsProvider, authCache); + tryAddHostedSuppressionCredentials(settings, credentialsProvider, authCache); + tryAddKEVCredentials(settings, credentialsProvider, authCache); + tryAddNexusAnalyzerCredentials(settings, credentialsProvider, authCache); + tryAddCentralAnalyzerCredentials(settings, credentialsProvider, authCache); + tryAddCentralContentCredentials(settings, credentialsProvider, authCache); + tryAddNVDApiDatafeed(settings, credentialsProvider, authCache); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); httpClientBuilderExplicitNoproxy.setDefaultCredentialsProvider(credentialsProvider); } - private void tryAddRetireJSCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddRetireJSCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, + authCache, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_USER, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_URL, Settings.KEYS.ANALYZER_RETIREJS_REPO_JS_PASSWORD, "RetireJS repo.js"); } } - private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddHostedSuppressionCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.HOSTED_SUPPRESSIONS_USER, + authCache, Settings.KEYS.HOSTED_SUPPRESSIONS_USER, Settings.KEYS.HOSTED_SUPPRESSIONS_URL, Settings.KEYS.HOSTED_SUPPRESSIONS_PASSWORD, "Hosted suppressions"); } } - private void tryAddKEVCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddKEVCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.KEV_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.KEV_USER, + authCache, Settings.KEYS.KEV_USER, Settings.KEYS.KEV_URL, Settings.KEYS.KEV_PASSWORD, "Known Exploited Vulnerabilities"); } } - private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddNexusAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_NEXUS_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.ANALYZER_NEXUS_USER, + authCache, Settings.KEYS.ANALYZER_NEXUS_USER, Settings.KEYS.ANALYZER_NEXUS_URL, Settings.KEYS.ANALYZER_NEXUS_PASSWORD, "Nexus Analyzer"); } } - private void tryAddCentralAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddCentralAnalyzerCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.ANALYZER_CENTRAL_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.ANALYZER_CENTRAL_USER, + authCache, Settings.KEYS.ANALYZER_CENTRAL_USER, Settings.KEYS.ANALYZER_CENTRAL_URL, Settings.KEYS.ANALYZER_CENTRAL_PASSWORD, "Central Analyzer"); } } - private void tryAddCentralContentCredentials(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddCentralContentCredentials(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.CENTRAL_CONTENT_PASSWORD) != null) { addUserPasswordCreds(settings, credentialsStore, - Settings.KEYS.CENTRAL_CONTENT_USER, + authCache, Settings.KEYS.CENTRAL_CONTENT_USER, Settings.KEYS.CENTRAL_CONTENT_URL, Settings.KEYS.CENTRAL_CONTENT_PASSWORD, "Central Content"); } } - private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credentialsStore) throws InvalidSettingException { + private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credentialsStore, AuthCache authCache) throws InvalidSettingException { if (settings.getString(Settings.KEYS.NVD_API_DATAFEED_PASSWORD) != null) { - addUserPasswordCreds(settings, credentialsStore, + addUserPasswordCreds(settings, credentialsStore, authCache, Settings.KEYS.NVD_API_DATAFEED_USER, Settings.KEYS.NVD_API_DATAFEED_URL, Settings.KEYS.NVD_API_DATAFEED_PASSWORD, @@ -255,15 +271,16 @@ private void tryAddNVDApiDatafeed(Settings settings, CredentialsStore credential /** * Add user/password credentials for the host/port of the URL, all configured in the settings, to the credential-store. * - * @param settings The settings to retrieve the values from - * @param store The credentialStore - * @param userKey The key for a configured username credential part + * @param settings The settings to retrieve the values from + * @param store The credentialStore + * @param authCache The authCache to register the authentication for the host of the url + * @param userKey The key for a configured username credential part + * @param urlKey The key for a configured url for which the credentials hold * @param passwordKey The key for a configured password credential part - * @param urlKey The key for a configured url for which the credentials hold - * @param desc A descriptive text for use in error messages for this credential + * @param desc A descriptive text for use in error messages for this credential * @throws InvalidSettingException When the password is empty or one of the other keys are not found in the settings. */ - private void addUserPasswordCreds(Settings settings, CredentialsStore store, String userKey, String urlKey, String passwordKey, String desc) + private void addUserPasswordCreds(Settings settings, CredentialsStore store, AuthCache authCache, String userKey, String urlKey, String passwordKey, String desc) throws InvalidSettingException { final String theUser = settings.getString(userKey); final String theURL = settings.getString(urlKey); @@ -273,30 +290,33 @@ private void addUserPasswordCreds(Settings settings, CredentialsStore store, Str } try { final URL parsedURL = new URL(theURL); - addCredentials(store, desc, parsedURL, theUser, thePass); + final HttpHost scopeHost = new HttpHost(parsedURL.getProtocol(), parsedURL.getHost(), parsedURL.getPort()); + addCredentials(store, scopeHost, desc, theUser, thePass, authCache); } catch (MalformedURLException e) { throw new InvalidSettingException(desc + " URL must be a valid URL", e); } } - private static void addCredentials(CredentialsStore credentialsStore, String messageScope, URL parsedURL, String theUser, char[] thePass) + private static void addCredentials(CredentialsStore credentialsStore, HttpHost scopeHost, String messageScope, String theUser, char[] thePass, + AuthCache authCache) throws InvalidSettingException { - final String theProtocol = parsedURL.getProtocol(); - if ("file".equals(theProtocol)) { + final String schemeName = scopeHost.getSchemeName(); + if ("file".equals(schemeName)) { LOGGER.warn("Credentials are not supported for file-protocol, double-check your configuration options for {}.", messageScope); return; - } else if ("http".equals(theProtocol)) { + } else if ("http".equals(schemeName)) { LOGGER.warn("Insecure configuration: Basic Credentials are configured to be used over a plain http connection for {}. " + "Consider migrating to https to guard the credentials.", messageScope); - } else if (!"https".equals(theProtocol)) { + } else if (!"https".equals(schemeName)) { throw new InvalidSettingException("Unsupported protocol in the " + messageScope + " URL; only file, http and https are supported"); } - final String theHost = parsedURL.getHost(); - final int thePort = parsedURL.getPort(); - final Credentials creds = new UsernamePasswordCredentials(theUser, thePass); - final AuthScope scope = new AuthScope(theProtocol, theHost, thePort, null, null); + final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(theUser, thePass); + final AuthScope scope = new AuthScope(scopeHost, null, null); credentialsStore.setCredentials(scope, creds); + final BasicScheme basicAuth = new BasicScheme(); + basicAuth.initPreemptive(creds); + authCache.put(scopeHost, basicAuth); } /** @@ -337,7 +357,7 @@ public void fetchFile(URL url, File outputPath, boolean useProxy) throws Downloa req = new BasicClassicHttpRequest(Method.GET, url.toURI()); try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath); - hc.execute(req, responseHandler); + hc.execute(req, getPreEmptiveAuthContext(), responseHandler); } } } catch (HttpResponseException hre) { @@ -404,8 +424,11 @@ public void fetchFile(URL url, File outputPath, boolean useProxy, String userKey try { final HttpClientContext context = HttpClientContext.create(); final BasicCredentialsProvider localCredentials = new BasicCredentialsProvider(); - addCredentials(localCredentials, url.toString(), url, settings.getString(userKey), settings.getString(passwordKey).toCharArray()); + final HttpHost scopeHost = new HttpHost(url.getProtocol(), url.getHost(), url.getPort()); + final AuthCache dedicated = new BasicAuthCache(); + addCredentials(localCredentials, scopeHost, url.toString(), settings.getString(userKey), settings.getString(passwordKey).toCharArray(), dedicated); context.setCredentialsProvider(localCredentials); + context.setAuthCache(dedicated); try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); final SaveToFileResponseHandler responseHandler = new SaveToFileResponseHandler(outputPath); @@ -454,7 +477,7 @@ public String postBasedFetchContent(URI url, String payload, ContentType payload } final String result; try (CloseableHttpClient hc = httpClientBuilder.build()) { - result = hc.execute(req, new BasicHttpClientResponseHandler()); + result = hc.execute(req, getPreEmptiveAuthContext(), new BasicHttpClientResponseHandler()); } return result; } @@ -514,8 +537,9 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) final BasicClassicHttpRequest req; req = new BasicClassicHttpRequest(Method.GET, url.toURI()); try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { - final ExplicitEncodingToStringResponseHandler responseHandler = new ExplicitEncodingToStringResponseHandler(charset); - result = hc.execute(req, responseHandler); + req.addHeader(HttpHeaders.ACCEPT_CHARSET, charset.name()); + final ExplicitCharsetToStringResponseHandler responseHandler = new ExplicitCharsetToStringResponseHandler(charset); + result = hc.execute(req, getPreEmptiveAuthContext(), responseHandler); } } return result; @@ -528,6 +552,27 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) } } + /** + * Gets a HttpClientContext that supports pre-emptive authentication. + * @return A HttpClientContext pre-configured with the authentication cache build from the settings. + */ + public HttpClientContext getPreEmptiveAuthContext() { + final HttpClientContext context = HttpClientContext.create(); + context.setCredentialsProvider(credentialsProvider); + context.setAuthCache(authCache); + return context; + } + + /** + * Gets a pre-configured HttpClient. + * Mainly targeted for use in paged resultset scenarios with multiple roundtrips. + * @param useProxy Whether to use the configuration that includes proxy-settings + * @return A HttpClient pre-configured with the settings. + */ + public CloseableHttpClient getHttpClient(boolean useProxy) { + return useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build(); + } + /** * Download a resource from the given URL and have its content handled by the given ResponseHandler. * @@ -540,7 +585,7 @@ public String fetchContent(URL url, boolean useProxy, Charset charset) * @throws ResourceNotFoundException When HTTP status 404 is encountered */ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler) - throws IOException, TooManyRequestsException, ResourceNotFoundException { + throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException { return fetchAndHandle(url, handler, Collections.emptyList(), true); } @@ -557,7 +602,7 @@ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler * @throws ResourceNotFoundException When HTTP status 404 is encountered */ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List
hdr) - throws IOException, TooManyRequestsException, ResourceNotFoundException { + throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException { return fetchAndHandle(url, handler, hdr, true); } @@ -575,33 +620,51 @@ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler * @throws ResourceNotFoundException When HTTP status 404 is encountered */ public T fetchAndHandle(@NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List
hdr, boolean useProxy) + throws IOException, TooManyRequestsException, ResourceNotFoundException, URISyntaxException { + final T data; + if ("file".equals(url.getProtocol())) { + final Path p = Paths.get(url.toURI()); + try (InputStream is = Files.newInputStream(p)) { + final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON); + final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200); + dummyResponse.setEntity(dummyEntity); + data = handler.handleResponse(dummyResponse); + } catch (HttpException e) { + throw new IllegalStateException("HttpException encountered emulating a HTTP response from a file", e); + } + } else { + try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { + return fetchAndHandle(hc, url, handler, hdr); + } + } + return data; + } + /** + * Download a resource from the given URL and have its content handled by the given ResponseHandler. + * + * @param client The HTTP Client to reuse for the request + * @param url The url of the resource + * @param handler The responsehandler to handle the response + * @param hdr Additional headers to add to the HTTP request + * @param The return-type for the responseHandler + * @return The response handler result + * @throws IOException on I/O Exceptions + * @throws TooManyRequestsException When HTTP status 429 is encountered + * @throws ResourceNotFoundException When HTTP status 404 is encountered + */ + public T fetchAndHandle(@NotNull CloseableHttpClient client, @NotNull URL url, @NotNull HttpClientResponseHandler handler, @NotNull List
hdr) throws IOException, TooManyRequestsException, ResourceNotFoundException { try { - final T data; - if ("file".equals(url.getProtocol())) { - final Path p = Paths.get(url.toURI()); - try (InputStream is = Files.newInputStream(p)) { - final HttpEntity dummyEntity = new BasicHttpEntity(is, ContentType.APPLICATION_JSON); - final ClassicHttpResponse dummyResponse = new BasicClassicHttpResponse(200); - dummyResponse.setEntity(dummyEntity); - data = handler.handleResponse(dummyResponse); - } catch (HttpException e) { - throw new IllegalStateException("HttpException encountered without HTTP traffic", e); - } - } else { - final String theProtocol = url.getProtocol(); - if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { - throw new DownloadFailedException("Unsupported protocol in the URL; only file, http and https are supported"); - } - try (CloseableHttpClient hc = useProxy ? httpClientBuilder.build() : httpClientBuilderExplicitNoproxy.build()) { - final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); - for (Header h : hdr) { - req.addHeader(h); - } - data = hc.execute(req, handler); - } + final String theProtocol = url.getProtocol(); + if (!("http".equals(theProtocol) || "https".equals(theProtocol))) { + throw new DownloadFailedException("Unsupported protocol in the URL; only http and https are supported"); + } + final BasicClassicHttpRequest req = new BasicClassicHttpRequest(Method.GET, url.toURI()); + for (Header h : hdr) { + req.addHeader(h); } - return data; + final HttpClientContext context = getPreEmptiveAuthContext(); + return client.execute(req, context, handler); } catch (HttpResponseException hre) { final String messageFormat = "%s - Server status: %d - Server reason: %s"; switch (hre.getStatusCode()) { diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java b/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitCharsetToStringResponseHandler.java similarity index 69% rename from utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java rename to utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitCharsetToStringResponseHandler.java index 1eee19fcc33..038ec8f157a 100644 --- a/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitEncodingToStringResponseHandler.java +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/ExplicitCharsetToStringResponseHandler.java @@ -18,6 +18,7 @@ package org.owasp.dependencycheck.utils; import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.HttpEntity; import org.apache.hc.core5.http.io.entity.EntityUtils; @@ -29,7 +30,7 @@ * * @author Hans Aikema */ -public class ExplicitEncodingToStringResponseHandler extends AbstractHttpClientResponseHandler { +public class ExplicitCharsetToStringResponseHandler extends AbstractHttpClientResponseHandler { /** * The explicit Charset used for interpreting the bytes of the HTTP response entity. @@ -41,12 +42,21 @@ public class ExplicitEncodingToStringResponseHandler extends AbstractHttpClientR * * @param charset The Charset to be used to transform a downloaded file into a String. */ - public ExplicitEncodingToStringResponseHandler(Charset charset) { + public ExplicitCharsetToStringResponseHandler(Charset charset) { this.charset = charset; } @Override public String handleEntity(HttpEntity entity) throws IOException { + final ContentType contentType; + contentType = ContentType.parseLenient(entity.getContentType()); + if (contentType != null) { + final Charset entityCharset = contentType.getCharset(); + if (entityCharset != null && !entityCharset.equals(charset)) { + throw new DownloadFailedException( + String.format("Requested decoding charset %s incompatible with the explicit response charset %s.", charset, entityCharset)); + } + } return new String(EntityUtils.toByteArray(entity), charset); } } diff --git a/utils/src/main/java/org/owasp/dependencycheck/utils/ToXMLDocumentResponseHandler.java b/utils/src/main/java/org/owasp/dependencycheck/utils/ToXMLDocumentResponseHandler.java new file mode 100644 index 00000000000..ffd7a81edb0 --- /dev/null +++ b/utils/src/main/java/org/owasp/dependencycheck/utils/ToXMLDocumentResponseHandler.java @@ -0,0 +1,41 @@ +/* + * This file is part of dependency-check-utils. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright (c) 2024 Hans Aikema. All Rights Reserved. + */ +package org.owasp.dependencycheck.utils; + +import org.apache.hc.client5.http.impl.classic.AbstractHttpClientResponseHandler; +import org.apache.hc.core5.http.HttpEntity; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.ParserConfigurationException; +import java.io.IOException; +import java.io.InputStream; + +public class ToXMLDocumentResponseHandler extends AbstractHttpClientResponseHandler { + @Override + public Document handleEntity(HttpEntity entity) throws IOException { + try (InputStream in = entity.getContent()) { + final DocumentBuilder builder = XmlUtils.buildSecureDocumentBuilder(); + return builder.parse(in); + } catch (ParserConfigurationException | SAXException | IOException e) { + final String errorMessage = "Failed to parse XML Response: " + e.getMessage(); + throw new IOException(errorMessage, e); + } + } +}