Skip to content

Commit

Permalink
use 128 GB of memory if available but probably just run out of heap m…
Browse files Browse the repository at this point in the history
…emory
  • Loading branch information
Al Niessner authored and Al Niessner committed Nov 3, 2023
1 parent 43fdf0d commit 6a9ce15
Showing 1 changed file with 5 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.logging.log4j.Logger;
import org.opensearch.action.search.SearchRequest;
import org.opensearch.action.search.SearchResponse;
import org.opensearch.client.HttpAsyncResponseConsumerFactory;
import org.opensearch.client.RequestOptions;
import org.opensearch.client.RestClient;
import org.opensearch.client.RestClientBuilder;
Expand All @@ -43,6 +44,7 @@ public class OpensearchDocument implements DocumentInfo, RestClientBuilder.HttpC
final private HashMap<String, List<String>> references = new HashMap<String, List<String>>();
final private LidvidComparator lidvid_compare = new LidvidComparator();
final private Logger log = LogManager.getLogger(OpensearchDocument.class);
final private RequestOptions.Builder hugeMemory = RequestOptions.DEFAULT.toBuilder();

private void load(String lidvid) {
if (!this.documents.containsKey(lidvid)) {
Expand Down Expand Up @@ -149,7 +151,7 @@ protected SearchResponse search(RestHighLevelClient client, SearchRequest reques
try {
if (OpensearchDocument.sourceOverride != null)
return OpensearchDocument.sourceOverride.search(client, request);
return client.search(request, RequestOptions.DEFAULT);
return client.search(request, this.hugeMemory.build());
} catch (ConnectException ce) {
iteration++;
if (iteration < 5)
Expand All @@ -162,6 +164,8 @@ protected SearchResponse search(RestHighLevelClient client, SearchRequest reques

public OpensearchDocument(AuthInformation context) {
this.context = context;
this.hugeMemory.setHttpAsyncResponseConsumerFactory(
new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory(128*1024^4)); // 128 GB
}

@Override
Expand Down

0 comments on commit 6a9ce15

Please sign in to comment.