From 261417263595c1b4400baa1db8b1a2abcfdae5d7 Mon Sep 17 00:00:00 2001 From: GeorgeC Date: Mon, 4 Nov 2024 10:32:49 -0500 Subject: [PATCH] Reduce default max per route connections and ensure response entity consumption Reduced the default maximum connections per route from 20 to 5 for better resource management. Also, added logic to consume and close the HttpResponse entity properly to avoid potential resource leaks. --- .../AggregateDataSharingResourceRS.java | 186 ++++++++------- .../passthru/PassThroughResourceRS.java | 2 +- .../dbmi/avillach/service/ProxyWebClient.java | 2 +- .../avillach/service/ResourceWebClient.java | 218 ++++++++++-------- 4 files changed, 215 insertions(+), 193 deletions(-) diff --git a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java index b054055e..786fc258 100644 --- a/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java +++ b/pic-sure-resources/pic-sure-aggregate-data-sharing-resource/src/main/java/edu/harvard/hms/dbmi/avillach/AggregateDataSharingResourceRS.java @@ -92,11 +92,11 @@ public AggregateDataSharingResourceRS(ApplicationProperties applicationPropertie variance = properties.getTargetPicsureObfuscationVariance(); randomSalt = properties.getTargetPicsureObfuscationSalt(); - headers = new Header[]{new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + properties.getTargetPicsureToken())}; + headers = new Header[] {new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + properties.getTargetPicsureToken())}; PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(100); // Maximum total connections - connectionManager.setDefaultMaxPerRoute(20); // Maximum connections per route + connectionManager.setDefaultMaxPerRoute(5); // Maximum connections per route httpClientUtil = HttpClientUtil.getInstance(connectionManager); } @@ -120,7 +120,7 @@ public ResourceInfo info(QueryRequest infoRequest) { if (infoRequest != null) { chainRequest.setQuery(infoRequest.getQuery()); chainRequest.setResourceCredentials(infoRequest.getResourceCredentials()); - //set a default value of the existing uuid here (can override in properties file) + // set a default value of the existing uuid here (can override in properties file) chainRequest.setResourceUUID(infoRequest.getResourceUUID()); } if (properties.getTargetResourceId() != null && !properties.getTargetResourceId().isEmpty()) { @@ -131,20 +131,21 @@ public ResourceInfo info(QueryRequest infoRequest) { String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); HttpResponse response = httpClientUtil.retrievePostResponse(composedURL, headers, payload); if (!httpClientUtil.is2xx(response)) { - logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + logger.error( + "{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase() + ); HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); } - //if we are proxying an info request, we need to return our own resource ID + // if we are proxying an info request, we need to return our own resource ID ResourceInfo resourceInfo = readObjectFromResponse(response, ResourceInfo.class); if (infoRequest != null && infoRequest.getResourceUUID() != null) { resourceInfo.setId(infoRequest.getResourceUUID()); } return resourceInfo; } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + infoRequest.getResourceUUID()); + throw new ApplicationException("Error encoding query for resource with id " + infoRequest.getResourceUUID()); } catch (ClassCastException | IllegalArgumentException e) { logger.error(e.getMessage()); throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); @@ -192,9 +193,7 @@ public Response queryResult(@PathParam("resourceQueryId") UUID queryId, QueryReq try { return Response.ok(response.getEntity().getContent()).build(); } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + resultRequest.getResourceUUID() - ); + throw new ApplicationException("Error encoding query for resource with id " + resultRequest.getResourceUUID()); } } @@ -205,16 +204,17 @@ private HttpResponse postRequest(QueryRequest statusRequest, String pathName) { String composedURL = HttpClientUtil.composeURL(properties.getTargetPicsureUrl(), pathName); HttpResponse response = httpClientUtil.retrievePostResponse(composedURL, headers, payload); if (!HttpClientUtil.is2xx(response)) { - logger.error("{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + logger.error( + "{}{} did not return a 200: {} {} ", properties.getTargetPicsureUrl(), pathName, + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase() + ); HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); } return response; } catch (IOException e) { // Note: this shouldn't ever happen logger.error("Error encoding search payload", e); - throw new ApplicationException( - "Error encoding search for resource with id " + statusRequest.getResourceUUID()); + throw new ApplicationException("Error encoding search for resource with id " + statusRequest.getResourceUUID()); } } @@ -236,9 +236,8 @@ public Response querySync(QueryRequest queryRequest) { String expectedResultType = jsonNode.get("expectedResultType").asText(); Set allowedResultTypes = Set.of( - "COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", - "OBSERVATION_CROSS_COUNT", "CATEGORICAL_CROSS_COUNT", "CONTINUOUS_CROSS_COUNT", - "VARIANT_COUNT_FOR_QUERY", "AGGREGATE_VCF_EXCERPT", "VCF_EXCERPT" + "COUNT", "CROSS_COUNT", "INFO_COLUMN_LISTING", "OBSERVATION_COUNT", "OBSERVATION_CROSS_COUNT", "CATEGORICAL_CROSS_COUNT", + "CONTINUOUS_CROSS_COUNT", "VARIANT_COUNT_FOR_QUERY", "AGGREGATE_VCF_EXCERPT", "VCF_EXCERPT" ); if (!allowedResultTypes.contains(expectedResultType)) { @@ -254,7 +253,7 @@ public Response querySync(QueryRequest queryRequest) { responseString = getExpectedResponse(expectedResultType, entityString, responseString, queryRequest); - //propagate any metadata from the back end (e.g., resultId) + // propagate any metadata from the back end (e.g., resultId) if (response.containsHeader(QUERY_METADATA_FIELD)) { Header metadataHeader = ((Header[]) response.getHeaders(QUERY_METADATA_FIELD))[0]; return Response.ok(responseString).header(QUERY_METADATA_FIELD, metadataHeader.getValue()).build(); @@ -263,15 +262,15 @@ public Response querySync(QueryRequest queryRequest) { return Response.ok(responseString).build(); } catch (IOException e) { logger.error(e.getMessage(), e); - throw new ApplicationException( - "Error encoding query for resource with id " + queryRequest.getResourceUUID()); + throw new ApplicationException("Error encoding query for resource with id " + queryRequest.getResourceUUID()); } catch (ClassCastException | IllegalArgumentException e) { logger.error(e.getMessage()); throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); } } - private HttpResponse getHttpResponse(QueryRequest queryRequest, UUID resourceUUID, String pathName, String targetPicsureUrl) throws JsonProcessingException { + private HttpResponse getHttpResponse(QueryRequest queryRequest, UUID resourceUUID, String pathName, String targetPicsureUrl) + throws JsonProcessingException { String queryString = objectMapper.writeValueAsString(queryRequest); String composedURL = HttpClientUtil.composeURL(targetPicsureUrl, pathName); @@ -280,27 +279,26 @@ private HttpResponse getHttpResponse(QueryRequest queryRequest, UUID resourceUUI if (!HttpClientUtil.is2xx(response)) { logger.error("Not 200 status!"); logger.error( - composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase() + ); HttpClientUtil.throwResponseError(response, targetPicsureUrl); } return response; } /** - * This method will process the response from the backend and return the - * expected response based on the expected result type. - * Currently, the only types that are handled are: - * COUNT, CROSS_COUNT, CATEGORICAL_CROSS_COUNT, CONTINUOUS_CROSS_COUNT + * This method will process the response from the backend and return the expected response based on the expected result type. Currently, + * the only types that are handled are: COUNT, CROSS_COUNT, CATEGORICAL_CROSS_COUNT, CONTINUOUS_CROSS_COUNT * * @param expectedResultType The expected result type - * @param entityString The response from the backend that will be processed - * @param responseString The response that will be returned. Will return the passed entityString if - * no cases are matched. + * @param entityString The response from the backend that will be processed + * @param responseString The response that will be returned. Will return the passed entityString if no cases are matched. * @return String The response that will be returned * @throws JsonProcessingException If there is an error processing the response */ - private String getExpectedResponse(String expectedResultType, String entityString, String responseString, QueryRequest queryRequest) throws IOException, JsonProcessingException { + private String getExpectedResponse(String expectedResultType, String entityString, String responseString, QueryRequest queryRequest) + throws IOException, JsonProcessingException { String crossCountResponse; switch (expectedResultType) { case "COUNT": @@ -327,8 +325,8 @@ private String getExpectedResponse(String expectedResultType, String entityStrin } /** - * No matter what the expected result type is we will get the cross count instead. Additionally, - * it will include ALL study consents in the query. + * No matter what the expected result type is we will get the cross count instead. Additionally, it will include ALL study consents in + * the query. * * @param queryRequest The query request * @return String The cross count for the query @@ -336,7 +334,9 @@ private String getExpectedResponse(String expectedResultType, String entityStrin private String getCrossCountForQuery(QueryRequest queryRequest) throws IOException { logger.debug("Calling Aggregate Data Sharing Resource getCrossCountForQuery()"); - HttpResponse response = getHttpResponse(changeQueryToOpenCrossCount(queryRequest), queryRequest.getResourceUUID(), "/query/sync", properties.getTargetPicsureUrl()); + HttpResponse response = getHttpResponse( + changeQueryToOpenCrossCount(queryRequest), queryRequest.getResourceUUID(), "/query/sync", properties.getTargetPicsureUrl() + ); HttpEntity entity = response.getEntity(); return EntityUtils.toString(entity, "UTF-8"); } @@ -356,8 +356,7 @@ private QueryRequest changeQueryToOpenCrossCount(QueryRequest queryRequest) { JsonNode updatedExpectedResulType = setExpectedResultTypeToCrossCount(jsonNode); JsonNode includesStudyConsents = addStudyConsentsToQuery(updatedExpectedResulType); - LinkedHashMap rebuiltQuery = objectMapper.convertValue(includesStudyConsents, new TypeReference<>() { - }); + LinkedHashMap rebuiltQuery = objectMapper.convertValue(includesStudyConsents, new TypeReference<>() {}); queryRequest.setQuery(rebuiltQuery); return queryRequest; } @@ -380,11 +379,9 @@ private JsonNode addStudyConsentsToQuery(JsonNode jsonNode) { logger.debug("Calling Aggregate Data Sharing Resource addStudyConsentsToQuery()"); SearchResults consentResults = getAllStudyConsents(); - LinkedHashMap linkedHashMap = objectMapper.convertValue(consentResults.getResults(), new TypeReference<>() { - }); + LinkedHashMap linkedHashMap = objectMapper.convertValue(consentResults.getResults(), new TypeReference<>() {}); Object phenotypes = linkedHashMap.get("phenotypes"); - LinkedHashMap phenotypesLinkedHashMap = objectMapper.convertValue(phenotypes, new TypeReference<>() { - }); + LinkedHashMap phenotypesLinkedHashMap = objectMapper.convertValue(phenotypes, new TypeReference<>() {}); // get all the keys from phenotypes Set keys = phenotypesLinkedHashMap.keySet(); @@ -426,15 +423,15 @@ public Response queryFormat(QueryRequest queryRequest) { HttpResponse response = httpClientUtil.retrievePostResponse(composedURL, headers, queryString); if (!HttpClientUtil.is2xx(response)) { logger.error( - composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", - response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + composedURL + " calling resource with id " + resourceUUID + " did not return a 200: {} {} ", + response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase() + ); HttpClientUtil.throwResponseError(response, properties.getTargetPicsureUrl()); } return Response.ok(response.getEntity().getContent()).build(); } catch (IOException e) { - throw new ApplicationException( - "Error encoding query for resource with id " + queryRequest.getResourceUUID()); + throw new ApplicationException("Error encoding query for resource with id " + queryRequest.getResourceUUID()); } catch (ClassCastException | IllegalArgumentException e) { logger.error(e.getMessage()); throw new ProtocolException(ProtocolException.INCORRECTLY_FORMATTED_REQUEST); @@ -442,8 +439,7 @@ public Response queryFormat(QueryRequest queryRequest) { } private Map processCrossCounts(String entityString) throws com.fasterxml.jackson.core.JsonProcessingException { - Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>() { - }); + Map crossCounts = objectMapper.readValue(entityString, new TypeReference<>() {}); int requestVariance = generateVarianceWithCrossCounts(crossCounts); crossCounts = obfuscateCrossCounts(crossCounts, requestVariance); @@ -454,7 +450,7 @@ private Map processCrossCounts(String entityString) throws com.f /** * This method will appropriately process the obfuscation of the cross counts. * - * @param crossCounts The cross counts + * @param crossCounts The cross counts * @param requestVariance The variance for the request * @return Map The obfuscated cross counts */ @@ -482,11 +478,9 @@ private Map obfuscateCrossCounts(Map crossCounts } /** - * This method is used to generate a variance for Cross Count queries. - * The variance is generated by taking the cross counts and sorting them by key. - * Then we generate a string with lines like consent:1\n consent:2\ consent:3\n etc. - * Then we generate a variance using the string. This is to give us a random variance that is deterministic for each - * query. + * This method is used to generate a variance for Cross Count queries. The variance is generated by taking the cross counts and sorting + * them by key. Then we generate a string with lines like consent:1\n consent:2\ consent:3\n etc. Then we generate a variance using the + * string. This is to give us a random variance that is deterministic for each query. * * @param crossCounts A map of cross counts * @return int The variance @@ -507,27 +501,27 @@ private int generateVarianceWithCrossCounts(Map crossCounts) { } /** - * This method will return an obfuscated binned count of continuous crosses. Due to the format of a continuous - * cross count, we are unable to directly obfuscate it in its original form. First, we send the continuous - * cross count data to the visualization resource to group it into bins. Once the data is binned, we assess whether - * obfuscation is necessary for this particular continuous cross count. If obfuscation is not required, we return - * the data in string format. However, if obfuscation is needed, we first obfuscate the data and then return it. + * This method will return an obfuscated binned count of continuous crosses. Due to the format of a continuous cross count, we are + * unable to directly obfuscate it in its original form. First, we send the continuous cross count data to the visualization resource to + * group it into bins. Once the data is binned, we assess whether obfuscation is necessary for this particular continuous cross count. + * If obfuscation is not required, we return the data in string format. However, if obfuscation is needed, we first obfuscate the data + * and then return it. * * @param continuousCrossCountResponse The continuous cross count response - * @param crossCountResponse The cross count response - * @param queryRequest The original query request + * @param crossCountResponse The cross count response + * @param queryRequest The original query request * @return String The obfuscated binned continuous cross count * @throws IOException If there is an error processing the JSON */ - protected String processContinuousCrossCounts(String continuousCrossCountResponse, String crossCountResponse, QueryRequest queryRequest) throws IOException { + protected String processContinuousCrossCounts(String continuousCrossCountResponse, String crossCountResponse, QueryRequest queryRequest) + throws IOException { logger.info("Processing continuous cross counts"); if (continuousCrossCountResponse == null || crossCountResponse == null) { return null; } - Map crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>() { - }); + Map crossCounts = objectMapper.readValue(crossCountResponse, new TypeReference<>() {}); int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); boolean mustObfuscate = isCrossCountObfuscated(crossCounts, generatedVariance); @@ -537,8 +531,10 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons // Handle the case where there is no visualization service UUID if (properties.getVisualizationResourceId() != null) { - Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference<>() {}); - Map> binnedContinuousCrossCounts = getBinnedContinuousCrossCount(queryRequest, continuousCrossCounts); + Map> continuousCrossCounts = + objectMapper.readValue(continuousCrossCountResponse, new TypeReference<>() {}); + Map> binnedContinuousCrossCounts = + getBinnedContinuousCrossCount(queryRequest, continuousCrossCounts); // Log the binned continuous cross counts logger.info("Binned continuous cross counts: {}", binnedContinuousCrossCounts); @@ -554,14 +550,17 @@ protected String processContinuousCrossCounts(String continuousCrossCountRespons return continuousCrossCountResponse; } - Map> continuousCrossCounts = objectMapper.readValue(continuousCrossCountResponse, new TypeReference<>() {}); + Map> continuousCrossCounts = + objectMapper.readValue(continuousCrossCountResponse, new TypeReference<>() {}); obfuscatedCrossCount(generatedVariance, continuousCrossCounts); return objectMapper.writeValueAsString(continuousCrossCounts); } } - private Map> getBinnedContinuousCrossCount(QueryRequest queryRequest, Map> continuousCrossCounts) throws IOException { + private Map> getBinnedContinuousCrossCount( + QueryRequest queryRequest, Map> continuousCrossCounts + ) throws IOException { // Create Query for Visualization /bin/continuous QueryRequest visualizationBinRequest = new GeneralQueryRequest(); visualizationBinRequest.setResourceUUID(properties.getVisualizationResourceId()); @@ -574,38 +573,38 @@ private Map> getBinnedContinuousCrossCount(QueryRequ } // call the binning endpoint - HttpResponse httpResponse = getHttpResponse(visualizationBinRequest, visualizationBinRequest.getResourceUUID(), "/bin/continuous", visResource.getResourceRSPath()); + HttpResponse httpResponse = getHttpResponse( + visualizationBinRequest, visualizationBinRequest.getResourceUUID(), "/bin/continuous", visResource.getResourceRSPath() + ); HttpEntity entity = httpResponse.getEntity(); String binResponse = EntityUtils.toString(entity, "UTF-8"); - return objectMapper.readValue(binResponse, new TypeReference>>() { - }); + return objectMapper.readValue(binResponse, new TypeReference>>() {}); } /** - * This method handles the processing of categorical cross counts. It begins by determining whether the cross - * counts require obfuscation. This is accomplished by checking if any of the CROSS_COUNTS must be obfuscated. - * If obfuscation is required, the categorical cross counts will be obfuscated accordingly. Otherwise, - * if no obfuscation is needed, the method can simply return the categorical entity string. + * This method handles the processing of categorical cross counts. It begins by determining whether the cross counts require + * obfuscation. This is accomplished by checking if any of the CROSS_COUNTS must be obfuscated. If obfuscation is required, the + * categorical cross counts will be obfuscated accordingly. Otherwise, if no obfuscation is needed, the method can simply return the + * categorical entity string. * * @param categoricalEntityString The categorical entity string - * @param crossCountEntityString The cross count entity string + * @param crossCountEntityString The cross count entity string * @return String The processed categorical entity string * @throws JsonProcessingException If there is an error processing the JSON */ - protected String processCategoricalCrossCounts(String categoricalEntityString, String crossCountEntityString) throws JsonProcessingException { + protected String processCategoricalCrossCounts(String categoricalEntityString, String crossCountEntityString) + throws JsonProcessingException { logger.info("Processing categorical cross counts"); if (categoricalEntityString == null || crossCountEntityString == null) { return null; } - Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() { - }); + Map crossCounts = objectMapper.readValue(crossCountEntityString, new TypeReference<>() {}); int generatedVariance = this.generateVarianceWithCrossCounts(crossCounts); - Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() { - }); + Map> categoricalCrossCount = objectMapper.readValue(categoricalEntityString, new TypeReference<>() {}); if (categoricalCrossCount == null) { logger.info("Categorical cross count is null. Returning categoricalEntityString: {}", categoricalEntityString); @@ -641,8 +640,7 @@ private boolean doObfuscateData(Map> categoricalCros return false; } - return categoricalCrossCount.values().stream() - .anyMatch(this::checkForObfuscation); + return categoricalCrossCount.values().stream().anyMatch(this::checkForObfuscation); } private boolean checkForObfuscation(Map axisMap) { @@ -678,11 +676,11 @@ private String getRequestSource() { } /** - * This method will obfuscate the cross counts based on the generated variance. We do not have a return because - * we are modifying the passed crossCount object. Java passes objects by reference value, so we do not need to return. + * This method will obfuscate the cross counts based on the generated variance. We do not have a return because we are modifying the + * passed crossCount object. Java passes objects by reference value, so we do not need to return. * * @param generatedVariance The variance for the request - * @param crossCount The cross count that will be obfuscated + * @param crossCount The cross count that will be obfuscated */ private void obfuscatedCrossCount(int generatedVariance, Map> crossCount) { crossCount.forEach((key, value) -> { @@ -698,10 +696,10 @@ private void obfuscatedCrossCount(int generatedVariance, Map crossCounts) { } /** - * This method will generate a random variance for the request based on the passed entityString. The variance - * will be between -variance and +variance. The variance will be generated by adding a random salt to the - * entityString and then taking the hashcode of the result. The variance will be the hashcode mod the - * variance * 2 + 1 - variance. + * This method will generate a random variance for the request based on the passed entityString. The variance will be between -variance + * and +variance. The variance will be generated by adding a random salt to the entityString and then taking the hashcode of the result. + * The variance will be the hashcode mod the variance * 2 + 1 - variance. * * @return int The variance for the request */ @@ -773,9 +770,8 @@ private Stream generateParents(String key) { String[] split = key.split("\\\\"); if (split.length > 1) { - return Arrays.stream(Arrays.copyOfRange(split, 0, split.length - 1)) - .filter(Predicate.not(String::isEmpty)) - .map(segment -> stringJoiner.add(segment).toString()); + return Arrays.stream(Arrays.copyOfRange(split, 0, split.length - 1)).filter(Predicate.not(String::isEmpty)) + .map(segment -> stringJoiner.add(segment).toString()); } return Stream.empty(); } diff --git a/pic-sure-resources/pic-sure-passthrough-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRS.java b/pic-sure-resources/pic-sure-passthrough-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRS.java index 2e0904c3..fdd89cdf 100644 --- a/pic-sure-resources/pic-sure-passthrough-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRS.java +++ b/pic-sure-resources/pic-sure-passthrough-resource/src/main/java/edu/harvard/hms/dbmi/avillach/resource/passthru/PassThroughResourceRS.java @@ -43,7 +43,7 @@ public PoolingHttpClientConnectionManager getConnectionManager() { PoolingHttpClientConnectionManager connectionManager; connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(100); // Maximum total connections - connectionManager.setDefaultMaxPerRoute(20); // Maximum connections per route + connectionManager.setDefaultMaxPerRoute(5); // Maximum connections per route return connectionManager; } diff --git a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ProxyWebClient.java b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ProxyWebClient.java index 17643a3d..cc2cbced 100644 --- a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ProxyWebClient.java +++ b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ProxyWebClient.java @@ -38,7 +38,7 @@ public ProxyWebClient() { connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(100); // Maximum total connections - connectionManager.setDefaultMaxPerRoute(20); // Maximum connections per route + connectionManager.setDefaultMaxPerRoute(5); // Maximum connections per route client = HttpClientUtil.getConfiguredHttpClient(connectionManager); } diff --git a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ResourceWebClient.java b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ResourceWebClient.java index 4767f70c..4ef7aaac 100644 --- a/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ResourceWebClient.java +++ b/pic-sure-resources/pic-sure-resource-api/src/main/java/edu/harvard/dbmi/avillach/service/ResourceWebClient.java @@ -14,6 +14,7 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,10 +28,9 @@ /** * The ResourceWebClient class implements the client side logic for the endpoints specified in IResourceRS. - - The PicsureInfoService, PicsureQueryService and PicsureSearchService would then use this class - to serve calls from their methods to each configured Resource target url - after looking up the target url from the ResourceRepository. + * + * The PicsureInfoService, PicsureQueryService and PicsureSearchService would then use this class to serve calls from their methods to each + * configured Resource target url after looking up the target url from the ResourceRepository. */ @ApplicationScoped public class ResourceWebClient { @@ -42,44 +42,48 @@ public class ResourceWebClient { public static final String QUERY_METADATA_FIELD = "queryMetadata"; private final HttpClientUtil httpClientUtil; - + public ResourceWebClient() { PoolingHttpClientConnectionManager connectionManager; connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxTotal(100); // Maximum total connections - connectionManager.setDefaultMaxPerRoute(20); // Maximum connections per route + connectionManager.setDefaultMaxPerRoute(5); // Maximum connections per route this.httpClientUtil = HttpClientUtil.getInstance(connectionManager); } - public ResourceInfo info(String rsURL, QueryRequest queryRequest){ + public ResourceInfo info(String rsURL, QueryRequest queryRequest) { logger.debug("Calling ResourceWebClient info()"); try { - if (queryRequest == null){ + if (queryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (queryRequest.getResourceCredentials() == null){ + if (queryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException(NotAuthorizedException.MISSING_CREDENTIALS); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } logger.debug("Calling /info at ResourceURL: {}", rsURL); String pathName = "/info"; String body = json.writeValueAsString(queryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return httpClientUtil.readObjectFromResponse(resourcesResponse, ResourceInfo.class); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { throw new NotAuthorizedException("Unable to encode resource credentials", e); } } - public PaginatedSearchResult searchConceptValues(String rsURL, QueryRequest queryRequest, String conceptPath, String query, Integer page, Integer size) { + public PaginatedSearchResult searchConceptValues( + String rsURL, QueryRequest queryRequest, String conceptPath, String query, Integer page, Integer size + ) { try { logger.debug("Calling /search/values at ResourceURL: {}"); URIBuilder uriBuilder = new URIBuilder(rsURL); @@ -94,7 +98,8 @@ public PaginatedSearchResult searchConceptValues(String rsURL, QueryRequest q uriBuilder.addParameter("size", size.toString()); } Map resourceCredentials = queryRequest != null ? queryRequest.getResourceCredentials() : Map.of(); - HttpResponse resourcesResponse = httpClientUtil.retrieveGetResponse(uriBuilder.build().toString(), createHeaders(resourceCredentials)); + HttpResponse resourcesResponse = + httpClientUtil.retrieveGetResponse(uriBuilder.build().toString(), createHeaders(resourceCredentials)); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); @@ -105,199 +110,220 @@ public PaginatedSearchResult searchConceptValues(String rsURL, QueryRequest q } } - public SearchResults search(String rsURL, QueryRequest searchQueryRequest){ + public SearchResults search(String rsURL, QueryRequest searchQueryRequest) { logger.debug("Calling ResourceWebClient search()"); + HttpResponse resourcesResponse = null; try { - if (searchQueryRequest == null || searchQueryRequest.getQuery() == null){ + if (searchQueryRequest == null || searchQueryRequest.getQuery() == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } - if (searchQueryRequest.getResourceCredentials() == null){ + if (searchQueryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException(NotAuthorizedException.MISSING_CREDENTIALS); } String pathName = "/search"; String body = json.writeValueAsString(searchQueryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(searchQueryRequest.getResourceCredentials()), body); + resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(searchQueryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return httpClientUtil.readObjectFromResponse(resourcesResponse, SearchResults.class); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to serialize search query"); - //TODO Write custom exception + // TODO Write custom exception throw new ProtocolException("Unable to serialize search query", e); + } finally { + if (resourcesResponse != null) { + try { + EntityUtils.consume(resourcesResponse.getEntity()); + } catch (IOException e) { + logger.error("Failed to close HttpResponse entity", e); + } + } } } - public QueryStatus query(String rsURL, QueryRequest dataQueryRequest){ + public QueryStatus query(String rsURL, QueryRequest dataQueryRequest) { logger.debug("Calling ResourceWebClient query()"); try { - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } - if (dataQueryRequest == null){ + if (dataQueryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (dataQueryRequest.getResourceCredentials() == null){ + if (dataQueryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException("Missing credentials"); } String pathName = "/query"; String body = json.writeValueAsString(dataQueryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(dataQueryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(dataQueryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return httpClientUtil.readObjectFromResponse(resourcesResponse, QueryStatus.class); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to encode data query"); throw new ProtocolException("Unable to encode data query", e); } } - public QueryStatus queryStatus(String rsURL, String queryId, QueryRequest queryRequest){ + public QueryStatus queryStatus(String rsURL, String queryId, QueryRequest queryRequest) { logger.debug("Calling ResourceWebClient query()"); try { - if (queryRequest == null){ + if (queryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (queryRequest.getResourceCredentials() == null){ + if (queryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException("Missing credentials"); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } - if (queryId == null){ + if (queryId == null) { throw new ProtocolException("Missing query id"); } String pathName = "/query/" + queryId + "/status"; String body = json.writeValueAsString(queryRequest); logger.debug(httpClientUtil.composeURL(rsURL, pathName)); logger.debug(body); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return httpClientUtil.readObjectFromResponse(resourcesResponse, QueryStatus.class); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to encode resource credentials"); throw new ProtocolException("Unable to encode resource credentials", e); } } - public Response queryResult(String rsURL, String queryId, QueryRequest queryRequest){ + public Response queryResult(String rsURL, String queryId, QueryRequest queryRequest) { logger.debug("Calling ResourceWebClient query()"); try { - if (queryRequest == null){ + if (queryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (queryRequest.getResourceCredentials() == null){ + if (queryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException("Missing credentials"); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } - if (queryId == null){ + if (queryId == null) { throw new ProtocolException(ProtocolException.MISSING_QUERY_ID); } String pathName = "/query/" + queryId + "/result"; String body = json.writeValueAsString(queryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return Response.ok(resourcesResponse.getEntity().getContent()).build(); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to encode resource credentials"); throw new NotAuthorizedException("Unable to encode resource credentials", e); - } catch (IOException e){ + } catch (IOException e) { throw new ResourceInterfaceException("Error getting results", e); } } - public Response queryResultSignedUrl(String rsURL, String queryId, QueryRequest queryRequest){ + public Response queryResultSignedUrl(String rsURL, String queryId, QueryRequest queryRequest) { logger.debug("Calling ResourceWebClient querySignedUrl()"); try { - if (queryRequest == null){ + if (queryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (queryRequest.getResourceCredentials() == null){ + if (queryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException("Missing credentials"); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } - if (queryId == null){ + if (queryId == null) { throw new ProtocolException(ProtocolException.MISSING_QUERY_ID); } String pathName = "/query/" + queryId + "/signed-url"; String body = json.writeValueAsString(queryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body + ); if (resourcesResponse.getStatusLine().getStatusCode() != 200) { logger.error("ResourceRS did not return a 200"); httpClientUtil.throwResponseError(resourcesResponse, rsURL); } return Response.ok(resourcesResponse.getEntity().getContent()).build(); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to encode resource credentials"); throw new NotAuthorizedException("Unable to encode resource credentials", e); - } catch (IOException e){ + } catch (IOException e) { throw new ResourceInterfaceException("Error getting results", e); } } - - public Response queryFormat(String rsURL, QueryRequest queryRequest){ + + public Response queryFormat(String rsURL, QueryRequest queryRequest) { logger.debug("Calling ResourceWebClient queryFormat()"); try { - if (queryRequest == null){ + if (queryRequest == null) { throw new ProtocolException(ProtocolException.MISSING_DATA); } - if (queryRequest.getResourceCredentials() == null){ + if (queryRequest.getResourceCredentials() == null) { throw new NotAuthorizedException("Missing credentials"); } - if (rsURL == null){ + if (rsURL == null) { throw new ApplicationException(ApplicationException.MISSING_RESOURCE_PATH); } String pathName = "/query/format"; String body = json.writeValueAsString(queryRequest); - HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body); + HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse( + httpClientUtil.composeURL(rsURL, pathName), createHeaders(queryRequest.getResourceCredentials()), body + ); int status = resourcesResponse.getStatusLine().getStatusCode(); if (status != 200) { logger.error("Query format request did not return a 200: " + resourcesResponse.getStatusLine().getStatusCode()); return Response.status(status).entity(resourcesResponse.getEntity().getContent()).build(); } return Response.ok(resourcesResponse.getEntity().getContent()).build(); - } catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { logger.error("Unable to encode resource credentials"); throw new NotAuthorizedException("Unable to encode resource credentials", e); - } catch (IOException e){ + } catch (IOException e) { throw new ResourceInterfaceException("Error getting results", e); } } - public Response querySync(String rsURL, QueryRequest queryRequest, String requestSource) { - logger.debug("Calling ResourceWebClient querySync()"); - try { - if (queryRequest == null) { - throw new ProtocolException("Missing query data"); - } - if (queryRequest.getResourceCredentials() == null) { - throw new NotAuthorizedException("Missing credentials"); - } - if (rsURL == null) { - throw new ApplicationException("Missing resource URL"); - } + public Response querySync(String rsURL, QueryRequest queryRequest, String requestSource) { + logger.debug("Calling ResourceWebClient querySync()"); + try { + if (queryRequest == null) { + throw new ProtocolException("Missing query data"); + } + if (queryRequest.getResourceCredentials() == null) { + throw new NotAuthorizedException("Missing credentials"); + } + if (rsURL == null) { + throw new ApplicationException("Missing resource URL"); + } - String pathName = "/query/sync"; - String body = json.writeValueAsString(queryRequest); + String pathName = "/query/sync"; + String body = json.writeValueAsString(queryRequest); Header[] headers = createHeaders(queryRequest.getResourceCredentials()); @@ -312,27 +338,27 @@ public Response querySync(String rsURL, QueryRequest queryRequest, String reques } HttpResponse resourcesResponse = httpClientUtil.retrievePostResponse(httpClientUtil.composeURL(rsURL, pathName), headers, body); - if (resourcesResponse.getStatusLine().getStatusCode() != 200) { - throwError(resourcesResponse, rsURL); - } - - if (resourcesResponse.containsHeader(QUERY_METADATA_FIELD)) { - Header metadataHeader = ((Header[]) resourcesResponse.getHeaders(QUERY_METADATA_FIELD))[0]; - return Response.ok(resourcesResponse.getEntity().getContent()) - .header(QUERY_METADATA_FIELD, metadataHeader.getValue()).build(); - } - return Response.ok(resourcesResponse.getEntity().getContent()).build(); - } catch (JsonProcessingException e) { - logger.error("Unable to encode resource credentials"); - throw new NotAuthorizedException("Unable to encode resource credentials", e); - } catch (IOException e) { - throw new ResourceInterfaceException("Error getting results", e); - } - } + if (resourcesResponse.getStatusLine().getStatusCode() != 200) { + throwError(resourcesResponse, rsURL); + } + + if (resourcesResponse.containsHeader(QUERY_METADATA_FIELD)) { + Header metadataHeader = ((Header[]) resourcesResponse.getHeaders(QUERY_METADATA_FIELD))[0]; + return Response.ok(resourcesResponse.getEntity().getContent()).header(QUERY_METADATA_FIELD, metadataHeader.getValue()) + .build(); + } + return Response.ok(resourcesResponse.getEntity().getContent()).build(); + } catch (JsonProcessingException e) { + logger.error("Unable to encode resource credentials"); + throw new NotAuthorizedException("Unable to encode resource credentials", e); + } catch (IOException e) { + throw new ResourceInterfaceException("Error getting results", e); + } + } /** - * This method is used to call the /bin/continuous endpoint on the ResourceRS. The /bin/continuous endpoint is used - * to retrieve binned continuous data from the visualization resource. + * This method is used to call the /bin/continuous endpoint on the ResourceRS. The /bin/continuous endpoint is used to retrieve binned + * continuous data from the visualization resource. * * @param rsURL The URL of the ResourceRS * @param queryRequest The query request object @@ -372,7 +398,7 @@ public Response queryContinuous(String rsURL, QueryRequest queryRequest, String throwError(resourcesResponse, rsURL); } - return Response.ok(resourcesResponse.getEntity().getContent()).build(); + return Response.ok(resourcesResponse.getEntity().getContent()).build(); } catch (JsonProcessingException e) { throw new RuntimeException(e); } catch (IOException e) { @@ -380,15 +406,15 @@ public Response queryContinuous(String rsURL, QueryRequest queryRequest, String } } - private void throwError(HttpResponse response, String baseURL){ + private void throwError(HttpResponse response, String baseURL) { logger.error("ResourceRS did not return a 200"); String errorMessage = baseURL + " " + response.getStatusLine().getStatusCode() + " " + response.getStatusLine().getReasonPhrase(); try { JsonNode responseNode = json.readTree(response.getEntity().getContent()); - if (responseNode != null && responseNode.has("message")){ + if (responseNode != null && responseNode.has("message")) { errorMessage += "/n" + responseNode.get("message").asText(); } - } catch (IOException e ){ + } catch (IOException e) { } if (response.getStatusLine().getStatusCode() == 401) { throw new NotAuthorizedException(errorMessage); @@ -397,7 +423,7 @@ private void throwError(HttpResponse response, String baseURL){ } - private Header[] createHeaders(Map resourceCredentials){ + private Header[] createHeaders(Map resourceCredentials) { Header authorizationHeader = new BasicHeader(HttpHeaders.AUTHORIZATION, BEARER_STRING + resourceCredentials.get(BEARER_TOKEN_KEY)); Header contentTypeHeader = new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"); Header[] headers = {authorizationHeader, contentTypeHeader};