diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 8fdb16ef2..7c3b593a2 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -1,2 +1,5 @@ # style: Reformat project with Spotless 2dfd69cbc977fa45922485f1d252d81fc0da7447 + +# style: Reformat project +0825cca42191e030ef7c46cc9d86c8d004370d85 diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/AggregationsValidator.java b/batch-index/src/main/java/no/unit/nva/indexingclient/AggregationsValidator.java index dfa2b0c43..ab68d941c 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/AggregationsValidator.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/AggregationsValidator.java @@ -1,7 +1,6 @@ package no.unit.nva.indexingclient; import com.fasterxml.jackson.databind.JsonNode; - import java.util.ArrayList; import java.util.List; diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/BatchIndexer.java b/batch-index/src/main/java/no/unit/nva/indexingclient/BatchIndexer.java index dd1aef373..1ccc456c0 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/BatchIndexer.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/BatchIndexer.java @@ -1,106 +1,102 @@ package no.unit.nva.indexingclient; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.identifiers.SortableIdentifier; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.s3.ListingResult; import no.unit.nva.s3.S3Driver; - import nva.commons.core.paths.UnixPath; - import org.opensearch.action.bulk.BulkItemResponse; import org.opensearch.action.bulk.BulkItemResponse.Failure; import org.opensearch.action.bulk.BulkResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import software.amazon.awssdk.services.s3.S3Client; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - public class BatchIndexer implements IndexingResult { - private static final Logger logger = LoggerFactory.getLogger(BatchIndexer.class); - private final ImportDataRequestEvent importDataRequest; - private final S3Driver s3Driver; - private final IndexingClient openSearchRestClient; - private final int numberOfFilesPerEvent; - private IndexingResultRecord processingResult; - - public BatchIndexer( - ImportDataRequestEvent importDataRequestEvent, - S3Client s3Client, - IndexingClient openSearchRestClient, - int numberOfFilesPerEvent) { - this.importDataRequest = importDataRequestEvent; - this.openSearchRestClient = openSearchRestClient; - this.s3Driver = new S3Driver(s3Client, importDataRequestEvent.getBucket()); - this.numberOfFilesPerEvent = numberOfFilesPerEvent; - } - - public IndexingResult processRequest() { - - ListingResult listFilesResult = fetchNextPageOfFilenames(); - List contents = - fileContents(listFilesResult.getFiles()).collect(Collectors.toList()); - List failedResults = indexFileContents(contents); - this.processingResult = - new IndexingResultRecord<>( - failedResults, - listFilesResult.getListingStartingPoint(), - listFilesResult.isTruncated()); - - return this; - } - - private Stream fileContents(List files) { - return files.stream().map(s3Driver::getFile).map(IndexDocument::fromJsonString); - } - - @Override - public List failedResults() { - return this.processingResult.failedResults(); - } - - @Override - public String nextStartMarker() { - return processingResult.nextStartMarker(); - } - - @Override - public boolean truncated() { - return this.processingResult.truncated(); - } - - private ListingResult fetchNextPageOfFilenames() { - return s3Driver.listFiles( - UnixPath.of(importDataRequest.getS3Path()), - importDataRequest.getStartMarker(), - numberOfFilesPerEvent); - } - - private List indexFileContents(List contents) { - - Stream result = openSearchRestClient.batchInsert(contents.stream()); - List failures = collectFailures(result).collect(Collectors.toList()); - failures.forEach(this::logFailure); - return failures; - } - - private void logFailure(T failureMessage) { - logger.warn("Failed to index resource:{}", failureMessage); - } - - private Stream collectFailures(Stream indexActions) { - return indexActions - .filter(BulkResponse::hasFailures) - .map(BulkResponse::getItems) - .flatMap(Arrays::stream) - .filter(BulkItemResponse::isFailed) - .map(BulkItemResponse::getFailure) - .map(Failure::getId) - .map(SortableIdentifier::new); - } + private static final Logger logger = LoggerFactory.getLogger(BatchIndexer.class); + private final ImportDataRequestEvent importDataRequest; + private final S3Driver s3Driver; + private final IndexingClient openSearchRestClient; + private final int numberOfFilesPerEvent; + private IndexingResultRecord processingResult; + + public BatchIndexer( + ImportDataRequestEvent importDataRequestEvent, + S3Client s3Client, + IndexingClient openSearchRestClient, + int numberOfFilesPerEvent) { + this.importDataRequest = importDataRequestEvent; + this.openSearchRestClient = openSearchRestClient; + this.s3Driver = new S3Driver(s3Client, importDataRequestEvent.getBucket()); + this.numberOfFilesPerEvent = numberOfFilesPerEvent; + } + + public IndexingResult processRequest() { + + ListingResult listFilesResult = fetchNextPageOfFilenames(); + List contents = + fileContents(listFilesResult.getFiles()).collect(Collectors.toList()); + List failedResults = indexFileContents(contents); + this.processingResult = + new IndexingResultRecord<>( + failedResults, + listFilesResult.getListingStartingPoint(), + listFilesResult.isTruncated()); + + return this; + } + + private Stream fileContents(List files) { + return files.stream().map(s3Driver::getFile).map(IndexDocument::fromJsonString); + } + + @Override + public List failedResults() { + return this.processingResult.failedResults(); + } + + @Override + public String nextStartMarker() { + return processingResult.nextStartMarker(); + } + + @Override + public boolean truncated() { + return this.processingResult.truncated(); + } + + private ListingResult fetchNextPageOfFilenames() { + return s3Driver.listFiles( + UnixPath.of(importDataRequest.getS3Path()), + importDataRequest.getStartMarker(), + numberOfFilesPerEvent); + } + + private List indexFileContents(List contents) { + + Stream result = openSearchRestClient.batchInsert(contents.stream()); + List failures = collectFailures(result).collect(Collectors.toList()); + failures.forEach(this::logFailure); + return failures; + } + + private void logFailure(T failureMessage) { + logger.warn("Failed to index resource:{}", failureMessage); + } + + private Stream collectFailures(Stream indexActions) { + return indexActions + .filter(BulkResponse::hasFailures) + .map(BulkResponse::getItems) + .flatMap(Arrays::stream) + .filter(BulkItemResponse::isFailed) + .map(BulkItemResponse::getFailure) + .map(Failure::getId) + .map(SortableIdentifier::new); + } } diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/Constants.java b/batch-index/src/main/java/no/unit/nva/indexingclient/Constants.java index 2b11626cd..cea36f197 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/Constants.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/Constants.java @@ -4,9 +4,7 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; - import nva.commons.core.JacocoGenerated; - import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.eventbridge.EventBridgeClient; diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/EmitEventUtils.java b/batch-index/src/main/java/no/unit/nva/indexingclient/EmitEventUtils.java index f482b2ba3..4c11953ff 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/EmitEventUtils.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/EmitEventUtils.java @@ -4,43 +4,40 @@ import static no.unit.nva.indexingclient.Constants.MANDATORY_UNUSED_SUBTOPIC; import com.amazonaws.services.lambda.runtime.Context; - +import java.time.Instant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequest; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequestEntry; -import java.time.Instant; - public final class EmitEventUtils { - private static final Logger logger = LoggerFactory.getLogger(EmitEventUtils.class); - - private EmitEventUtils() {} - - public static void emitEvent( - EventBridgeClient eventBridgeClient, - ImportDataRequestEvent importDataRequest, - Context context) { - var putEventRequestEntry = eventEntry(importDataRequest, context); - logger.debug("BusName:" + BATCH_INDEX_EVENT_BUS_NAME); - logger.debug("Event:" + putEventRequestEntry.toString()); - var putEventRequest = PutEventsRequest.builder().entries(putEventRequestEntry).build(); - var response = eventBridgeClient.putEvents(putEventRequest); - logger.debug(response.toString()); - } - - private static PutEventsRequestEntry eventEntry( - ImportDataRequestEvent importDataRequest, Context context) { - return PutEventsRequestEntry.builder() - .eventBusName(BATCH_INDEX_EVENT_BUS_NAME) - .detailType(MANDATORY_UNUSED_SUBTOPIC) - .source(EventBasedBatchIndexer.class.getName()) - .time(Instant.now()) - .detail(importDataRequest.toJsonString()) - .resources(context.getInvokedFunctionArn()) - .build(); - } + private static final Logger logger = LoggerFactory.getLogger(EmitEventUtils.class); + + private EmitEventUtils() {} + + public static void emitEvent( + EventBridgeClient eventBridgeClient, + ImportDataRequestEvent importDataRequest, + Context context) { + var putEventRequestEntry = eventEntry(importDataRequest, context); + logger.debug("BusName:" + BATCH_INDEX_EVENT_BUS_NAME); + logger.debug("Event:" + putEventRequestEntry.toString()); + var putEventRequest = PutEventsRequest.builder().entries(putEventRequestEntry).build(); + var response = eventBridgeClient.putEvents(putEventRequest); + logger.debug(response.toString()); + } + + private static PutEventsRequestEntry eventEntry( + ImportDataRequestEvent importDataRequest, Context context) { + return PutEventsRequestEntry.builder() + .eventBusName(BATCH_INDEX_EVENT_BUS_NAME) + .detailType(MANDATORY_UNUSED_SUBTOPIC) + .source(EventBasedBatchIndexer.class.getName()) + .time(Instant.now()) + .detail(importDataRequest.toJsonString()) + .resources(context.getInvokedFunctionArn()) + .build(); + } } diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/EventBasedBatchIndexer.java b/batch-index/src/main/java/no/unit/nva/indexingclient/EventBasedBatchIndexer.java index 7f854e9cf..b53764a70 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/EventBasedBatchIndexer.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/EventBasedBatchIndexer.java @@ -8,83 +8,75 @@ import static no.unit.nva.indexingclient.EmitEventUtils.emitEvent; import com.amazonaws.services.lambda.runtime.Context; - +import java.io.InputStream; +import java.io.OutputStream; import no.unit.nva.events.handlers.EventHandler; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; - import nva.commons.core.JacocoGenerated; import nva.commons.core.ioutils.IoUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.s3.S3Client; -import java.io.InputStream; -import java.io.OutputStream; - public class EventBasedBatchIndexer - extends EventHandler { + extends EventHandler { - private static final Logger logger = LoggerFactory.getLogger(EventBasedBatchIndexer.class); + private static final Logger logger = LoggerFactory.getLogger(EventBasedBatchIndexer.class); - private final S3Client s3Client; - private final IndexingClient openSearchClient; - private final EventBridgeClient eventBridgeClient; - private final int numberOfFilesPerEvent; + private final S3Client s3Client; + private final IndexingClient openSearchClient; + private final EventBridgeClient eventBridgeClient; + private final int numberOfFilesPerEvent; - @JacocoGenerated - public EventBasedBatchIndexer() { - this( - defaultS3Client(), - defaultEsClient(), - defaultEventBridgeClient(), - NUMBER_OF_FILES_PER_EVENT_ENVIRONMENT_VARIABLE); - } + @JacocoGenerated + public EventBasedBatchIndexer() { + this( + defaultS3Client(), + defaultEsClient(), + defaultEventBridgeClient(), + NUMBER_OF_FILES_PER_EVENT_ENVIRONMENT_VARIABLE); + } - protected EventBasedBatchIndexer( - S3Client s3Client, - IndexingClient openSearchClient, - EventBridgeClient eventBridgeClient, - int numberOfFilesPerEvent) { - super(ImportDataRequestEvent.class); - this.s3Client = s3Client; - this.openSearchClient = openSearchClient; - this.eventBridgeClient = eventBridgeClient; - this.numberOfFilesPerEvent = numberOfFilesPerEvent; - } + protected EventBasedBatchIndexer( + S3Client s3Client, + IndexingClient openSearchClient, + EventBridgeClient eventBridgeClient, + int numberOfFilesPerEvent) { + super(ImportDataRequestEvent.class); + this.s3Client = s3Client; + this.openSearchClient = openSearchClient; + this.eventBridgeClient = eventBridgeClient; + this.numberOfFilesPerEvent = numberOfFilesPerEvent; + } - @Override - public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) { - String inputString = IoUtils.streamToString(inputStream); - logger.debug(inputString); - super.handleRequest(IoUtils.stringToStream(inputString), outputStream, context); - } + @Override + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) { + String inputString = IoUtils.streamToString(inputStream); + logger.debug(inputString); + super.handleRequest(IoUtils.stringToStream(inputString), outputStream, context); + } - @Override - protected SortableIdentifier[] processInput( - ImportDataRequestEvent input, - AwsEventBridgeEvent event, - Context context) { - logger.debug("Indexing folder:" + input.getS3Location()); - logger.debug("Indexing lastEvaluatedKey:" + input.getStartMarker()); - IndexingResult result = - new BatchIndexer(input, s3Client, openSearchClient, numberOfFilesPerEvent) - .processRequest(); - if (result.truncated() && RECURSION_ENABLED) { - emitEventToProcessNextBatch(input, context, result); - } - return result.failedResults().toArray(SortableIdentifier[]::new); + @Override + protected SortableIdentifier[] processInput( + ImportDataRequestEvent input, + AwsEventBridgeEvent event, + Context context) { + logger.debug("Indexing folder:" + input.getS3Location()); + logger.debug("Indexing lastEvaluatedKey:" + input.getStartMarker()); + IndexingResult result = + new BatchIndexer(input, s3Client, openSearchClient, numberOfFilesPerEvent).processRequest(); + if (result.truncated() && RECURSION_ENABLED) { + emitEventToProcessNextBatch(input, context, result); } + return result.failedResults().toArray(SortableIdentifier[]::new); + } - private void emitEventToProcessNextBatch( - ImportDataRequestEvent input, - Context context, - IndexingResult result) { - ImportDataRequestEvent newImportDataRequest = - new ImportDataRequestEvent(input.getS3Location(), result.nextStartMarker()); - emitEvent(eventBridgeClient, newImportDataRequest, context); - } + private void emitEventToProcessNextBatch( + ImportDataRequestEvent input, Context context, IndexingResult result) { + ImportDataRequestEvent newImportDataRequest = + new ImportDataRequestEvent(input.getS3Location(), result.nextStartMarker()); + emitEvent(eventBridgeClient, newImportDataRequest, context); + } } diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/ImportDataRequestEvent.java b/batch-index/src/main/java/no/unit/nva/indexingclient/ImportDataRequestEvent.java index e45acc02e..8d28dddcb 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/ImportDataRequestEvent.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/ImportDataRequestEvent.java @@ -7,15 +7,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - -import no.unit.nva.commons.json.JsonSerializable; -import no.unit.nva.events.models.EventBody; - -import nva.commons.core.JacocoGenerated; - import java.net.URI; import java.util.Objects; import java.util.Optional; +import no.unit.nva.commons.json.JsonSerializable; +import no.unit.nva.events.models.EventBody; +import nva.commons.core.JacocoGenerated; public class ImportDataRequestEvent implements EventBody, JsonSerializable { diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/StartBatchIndexingHandler.java b/batch-index/src/main/java/no/unit/nva/indexingclient/StartBatchIndexingHandler.java index 877cffae1..e6ba21028 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/StartBatchIndexingHandler.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/StartBatchIndexingHandler.java @@ -7,42 +7,39 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestStreamHandler; - -import nva.commons.core.JacocoGenerated; - -import software.amazon.awssdk.services.eventbridge.EventBridgeClient; - import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; +import nva.commons.core.JacocoGenerated; +import software.amazon.awssdk.services.eventbridge.EventBridgeClient; public class StartBatchIndexingHandler implements RequestStreamHandler { - private final EventBridgeClient eventBridgeClient; - - @JacocoGenerated - public StartBatchIndexingHandler() { - this(defaultEventBridgeClient()); - } - - public StartBatchIndexingHandler(EventBridgeClient eventBridgeClient) { - this.eventBridgeClient = eventBridgeClient; - } - - @Override - public void handleRequest(InputStream input, OutputStream output, Context context) - throws IOException { - var firstImportRequestEvent = new ImportDataRequestEvent(PERSISTED_RESOURCES_PATH); - emitEvent(eventBridgeClient, firstImportRequestEvent, context); - writeOutput(output); - } - - protected void writeOutput(OutputStream outputStream) throws IOException { - try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { - String outputJson = objectMapperWithEmpty.writeValueAsString("OK"); - writer.write(outputJson); - } + private final EventBridgeClient eventBridgeClient; + + @JacocoGenerated + public StartBatchIndexingHandler() { + this(defaultEventBridgeClient()); + } + + public StartBatchIndexingHandler(EventBridgeClient eventBridgeClient) { + this.eventBridgeClient = eventBridgeClient; + } + + @Override + public void handleRequest(InputStream input, OutputStream output, Context context) + throws IOException { + var firstImportRequestEvent = new ImportDataRequestEvent(PERSISTED_RESOURCES_PATH); + emitEvent(eventBridgeClient, firstImportRequestEvent, context); + writeOutput(output); + } + + protected void writeOutput(OutputStream outputStream) throws IOException { + try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(outputStream))) { + String outputJson = objectMapperWithEmpty.writeValueAsString("OK"); + writer.write(outputJson); } + } } diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandler.java b/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandler.java index 7812f97a1..4a4e67348 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandler.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandler.java @@ -1,5 +1,7 @@ package no.unit.nva.indexingclient.keybatch; +import static java.util.Objects.nonNull; +import static java.util.UUID.randomUUID; import static no.unit.nva.constants.Defaults.ENVIRONMENT; import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.indexingclient.Constants.EVENT_BUS; @@ -7,20 +9,17 @@ import static no.unit.nva.indexingclient.Constants.TOPIC; import static no.unit.nva.indexingclient.Constants.defaultS3Client; -import static java.util.Objects.nonNull; -import static java.util.UUID.randomUUID; - import com.amazonaws.services.lambda.runtime.Context; - +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.List; +import java.util.stream.Collectors; import no.unit.nva.events.handlers.EventHandler; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.indexingclient.EventBasedBatchIndexer; - import nva.commons.core.JacocoGenerated; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.services.eventbridge.EventBridgeClient; @@ -33,11 +32,6 @@ import software.amazon.awssdk.services.s3.model.PutObjectRequest; import software.amazon.awssdk.services.s3.model.S3Object; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.List; -import java.util.stream.Collectors; - public class GenerateKeyBatchesHandler extends EventHandler { public static final String DEFAULT_BATCH_SIZE = "1000"; diff --git a/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandler.java b/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandler.java index 148cc6cf6..15b5aabda 100644 --- a/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandler.java +++ b/batch-index/src/main/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandler.java @@ -1,33 +1,34 @@ package no.unit.nva.indexingclient.keybatch; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Defaults.ENVIRONMENT; import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.indexingclient.Constants.EVENT_BUS; import static no.unit.nva.indexingclient.Constants.MANDATORY_UNUSED_SUBTOPIC; import static no.unit.nva.indexingclient.Constants.TOPIC; import static no.unit.nva.indexingclient.Constants.defaultS3Client; - import static nva.commons.core.attempt.Try.attempt; -import static java.util.Objects.nonNull; - import com.amazonaws.services.lambda.runtime.Context; - +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; import no.unit.nva.events.handlers.EventHandler; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.indexingclient.AggregationsValidator; import no.unit.nva.indexingclient.IndexingClient; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.s3.S3Driver; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.Failure; import nva.commons.core.paths.UnixPath; - import org.opensearch.action.bulk.BulkResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequest; @@ -36,197 +37,187 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ListObjectsV2Response; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.stream.Stream; - public class KeyBasedBatchIndexHandler extends EventHandler { - private static final Logger logger = LoggerFactory.getLogger(KeyBasedBatchIndexHandler.class); - - private static final String LINE_BREAK = "\n"; - private static final String LAST_CONSUMED_BATCH = "Last consumed batch: {}"; - private static final String DEFAULT_PAYLOAD = "3291456"; - private static final int MAX_PAYLOAD = - Integer.parseInt(ENVIRONMENT.readEnvOpt("MAX_PAYLOAD").orElse(DEFAULT_PAYLOAD)); - private static final String PROCESSING_BATCH_MESSAGE = "Processing batch: {}"; - private static final String BULK_HAS_FAILED_MESSAGE = "Bulk has failed: "; - private static final String RESOURCES_BUCKET = - ENVIRONMENT.readEnv("PERSISTED_RESOURCES_BUCKET"); - private static final String KEY_BATCHES_BUCKET = ENVIRONMENT.readEnv("KEY_BATCHES_BUCKET"); - - private final IndexingClient indexingClient; - private final S3Client s3ResourcesClient; - private final S3Client s3BatchesClient; - private final EventBridgeClient eventBridgeClient; - - @JacocoGenerated - public KeyBasedBatchIndexHandler() { - this( - IndexingClient.defaultIndexingClient(), - defaultS3Client(), - defaultS3Client(), - defaultEventBridgeClient()); - } - - public KeyBasedBatchIndexHandler( - IndexingClient indexingClient, - S3Client s3ResourcesClient, - S3Client s3BatchesClient, - EventBridgeClient eventBridgeClient) { - super(KeyBatchRequestEvent.class); - this.indexingClient = indexingClient; - this.s3ResourcesClient = s3ResourcesClient; - this.s3BatchesClient = s3BatchesClient; - this.eventBridgeClient = eventBridgeClient; - } - - @JacocoGenerated - public static EventBridgeClient defaultEventBridgeClient() { - return EventBridgeClient.builder().httpClient(UrlConnectionHttpClient.create()).build(); - } - - private static PutEventsRequestEntry constructRequestEntry( - String lastEvaluatedKey, Context context, String location) { - return PutEventsRequestEntry.builder() - .eventBusName(EVENT_BUS) - .detail(new KeyBatchRequestEvent(lastEvaluatedKey, TOPIC, location).toJsonString()) - .detailType(MANDATORY_UNUSED_SUBTOPIC) - .source(KeyBasedBatchIndexHandler.class.getName()) - .resources(context.getInvokedFunctionArn()) - .time(Instant.now()) - .build(); - } - - private static String getStartMarker(KeyBatchRequestEvent input) { - - return nonNull(input) && nonNull(input.startMarker()) ? input.startMarker() : null; - } - - private static boolean isResource(IndexDocument document) { - return RESOURCES.equals(document.getIndexName()); - } - - @Override - protected Void processInput( - KeyBatchRequestEvent input, - AwsEventBridgeEvent event, - Context context) { - var startMarker = getStartMarker(input); - var location = getLocation(input); - var batchResponse = fetchSingleBatch(startMarker); - - if (batchResponse.isTruncated()) { + private static final Logger logger = LoggerFactory.getLogger(KeyBasedBatchIndexHandler.class); + + private static final String LINE_BREAK = "\n"; + private static final String LAST_CONSUMED_BATCH = "Last consumed batch: {}"; + private static final String DEFAULT_PAYLOAD = "3291456"; + private static final int MAX_PAYLOAD = + Integer.parseInt(ENVIRONMENT.readEnvOpt("MAX_PAYLOAD").orElse(DEFAULT_PAYLOAD)); + private static final String PROCESSING_BATCH_MESSAGE = "Processing batch: {}"; + private static final String BULK_HAS_FAILED_MESSAGE = "Bulk has failed: "; + private static final String RESOURCES_BUCKET = ENVIRONMENT.readEnv("PERSISTED_RESOURCES_BUCKET"); + private static final String KEY_BATCHES_BUCKET = ENVIRONMENT.readEnv("KEY_BATCHES_BUCKET"); + + private final IndexingClient indexingClient; + private final S3Client s3ResourcesClient; + private final S3Client s3BatchesClient; + private final EventBridgeClient eventBridgeClient; + + @JacocoGenerated + public KeyBasedBatchIndexHandler() { + this( + IndexingClient.defaultIndexingClient(), + defaultS3Client(), + defaultS3Client(), + defaultEventBridgeClient()); + } + + public KeyBasedBatchIndexHandler( + IndexingClient indexingClient, + S3Client s3ResourcesClient, + S3Client s3BatchesClient, + EventBridgeClient eventBridgeClient) { + super(KeyBatchRequestEvent.class); + this.indexingClient = indexingClient; + this.s3ResourcesClient = s3ResourcesClient; + this.s3BatchesClient = s3BatchesClient; + this.eventBridgeClient = eventBridgeClient; + } + + @JacocoGenerated + public static EventBridgeClient defaultEventBridgeClient() { + return EventBridgeClient.builder().httpClient(UrlConnectionHttpClient.create()).build(); + } + + private static PutEventsRequestEntry constructRequestEntry( + String lastEvaluatedKey, Context context, String location) { + return PutEventsRequestEntry.builder() + .eventBusName(EVENT_BUS) + .detail(new KeyBatchRequestEvent(lastEvaluatedKey, TOPIC, location).toJsonString()) + .detailType(MANDATORY_UNUSED_SUBTOPIC) + .source(KeyBasedBatchIndexHandler.class.getName()) + .resources(context.getInvokedFunctionArn()) + .time(Instant.now()) + .build(); + } + + private static String getStartMarker(KeyBatchRequestEvent input) { + + return nonNull(input) && nonNull(input.startMarker()) ? input.startMarker() : null; + } + + private static boolean isResource(IndexDocument document) { + return RESOURCES.equals(document.getIndexName()); + } + + @Override + protected Void processInput( + KeyBatchRequestEvent input, + AwsEventBridgeEvent event, + Context context) { + var startMarker = getStartMarker(input); + var location = getLocation(input); + var batchResponse = fetchSingleBatch(startMarker); + + if (batchResponse.isTruncated()) { sendEvent( constructRequestEntry(batchResponse.contents().getFirst().key(), context, location)); - } + } var batchKey = batchResponse.contents().getFirst().key(); - var content = extractContent(batchKey); - var indexDocuments = mapToIndexDocuments(content); + var content = extractContent(batchKey); + var indexDocuments = mapToIndexDocuments(content); - sendDocumentsToIndexInBatches(indexDocuments); + sendDocumentsToIndexInBatches(indexDocuments); logger.info(LAST_CONSUMED_BATCH, batchResponse.contents().getFirst()); - return null; - } - - private String getLocation(KeyBatchRequestEvent input) { - return nonNull(input) && nonNull(input.location()) ? input.location() : null; - } - - private String extractContent(String key) { - var s3Driver = new S3Driver(s3BatchesClient, KEY_BATCHES_BUCKET); - logger.info(PROCESSING_BATCH_MESSAGE, key); - return attempt(() -> s3Driver.getFile(UnixPath.of(key))).orElseThrow(); - } - - private void sendEvent(PutEventsRequestEntry event) { - eventBridgeClient.putEvents(PutEventsRequest.builder().entries(event).build()); - } - - private ListObjectsV2Response fetchSingleBatch(String startMarker) { - return s3BatchesClient.listObjectsV2( - ListObjectsV2Request.builder() - .bucket(KEY_BATCHES_BUCKET) - .startAfter(startMarker) - .maxKeys(1) - .build()); - } - - private List mapToIndexDocuments(String content) { - return extractIdentifiers(content) - .filter(Objects::nonNull) - .map(this::fetchS3Content) - .map(IndexDocument::fromJsonString) - .filter(this::isValid) - .toList(); - } - - @JacocoGenerated - private void sendDocumentsToIndexInBatches(List indexDocuments) { - var documents = new ArrayList(); - var totalSize = 0; - for (IndexDocument indexDocument : indexDocuments) { - var currentFileSize = - indexDocument.toJsonString().getBytes(StandardCharsets.UTF_8).length; - if (totalSize + currentFileSize < MAX_PAYLOAD) { - documents.add(indexDocument); - totalSize += currentFileSize; - } else { - indexDocuments(documents); - totalSize = 0; - documents.clear(); - documents.add(indexDocument); - } - } - if (!documents.isEmpty()) { - indexDocuments(documents); - } - } - - private void indexDocuments(List indexDocuments) { - attempt(() -> indexBatch(indexDocuments)).orElse(this::logFailure); - } - - private List logFailure(Failure> failure) { - logger.error(BULK_HAS_FAILED_MESSAGE, failure.getException()); - return List.of(); - } - - @JacocoGenerated - private List indexBatch(List indexDocuments) { - return indexingClient.batchInsert(indexDocuments.stream()).toList(); - } - - /** - * Resources/Publications only should be validated when indexing. ImportCandidates are not - * validated. - */ - private boolean isValid(IndexDocument document) { - return !isResource(document) || validateResource(document); - } - - private boolean validateResource(IndexDocument document) { - var validator = new AggregationsValidator(document.resource()); - if (!validator.isValid()) { - logger.info(validator.getReport()); - } - return validator.isValid(); - } - - private Stream extractIdentifiers(String value) { - return nonNull(value) && !value.isBlank() - ? Arrays.stream(value.split(LINE_BREAK)) - : Stream.empty(); - } - - private String fetchS3Content(String key) { - var s3Driver = new S3Driver(s3ResourcesClient, RESOURCES_BUCKET); - return attempt(() -> s3Driver.getFile(UnixPath.of(key))).orElseThrow(); - } + return null; + } + + private String getLocation(KeyBatchRequestEvent input) { + return nonNull(input) && nonNull(input.location()) ? input.location() : null; + } + + private String extractContent(String key) { + var s3Driver = new S3Driver(s3BatchesClient, KEY_BATCHES_BUCKET); + logger.info(PROCESSING_BATCH_MESSAGE, key); + return attempt(() -> s3Driver.getFile(UnixPath.of(key))).orElseThrow(); + } + + private void sendEvent(PutEventsRequestEntry event) { + eventBridgeClient.putEvents(PutEventsRequest.builder().entries(event).build()); + } + + private ListObjectsV2Response fetchSingleBatch(String startMarker) { + return s3BatchesClient.listObjectsV2( + ListObjectsV2Request.builder() + .bucket(KEY_BATCHES_BUCKET) + .startAfter(startMarker) + .maxKeys(1) + .build()); + } + + private List mapToIndexDocuments(String content) { + return extractIdentifiers(content) + .filter(Objects::nonNull) + .map(this::fetchS3Content) + .map(IndexDocument::fromJsonString) + .filter(this::isValid) + .toList(); + } + + @JacocoGenerated + private void sendDocumentsToIndexInBatches(List indexDocuments) { + var documents = new ArrayList(); + var totalSize = 0; + for (IndexDocument indexDocument : indexDocuments) { + var currentFileSize = indexDocument.toJsonString().getBytes(StandardCharsets.UTF_8).length; + if (totalSize + currentFileSize < MAX_PAYLOAD) { + documents.add(indexDocument); + totalSize += currentFileSize; + } else { + indexDocuments(documents); + totalSize = 0; + documents.clear(); + documents.add(indexDocument); + } + } + if (!documents.isEmpty()) { + indexDocuments(documents); + } + } + + private void indexDocuments(List indexDocuments) { + attempt(() -> indexBatch(indexDocuments)).orElse(this::logFailure); + } + + private List logFailure(Failure> failure) { + logger.error(BULK_HAS_FAILED_MESSAGE, failure.getException()); + return List.of(); + } + + @JacocoGenerated + private List indexBatch(List indexDocuments) { + return indexingClient.batchInsert(indexDocuments.stream()).toList(); + } + + /** + * Resources/Publications only should be validated when indexing. ImportCandidates are not + * validated. + */ + private boolean isValid(IndexDocument document) { + return !isResource(document) || validateResource(document); + } + + private boolean validateResource(IndexDocument document) { + var validator = new AggregationsValidator(document.resource()); + if (!validator.isValid()) { + logger.info(validator.getReport()); + } + return validator.isValid(); + } + + private Stream extractIdentifiers(String value) { + return nonNull(value) && !value.isBlank() + ? Arrays.stream(value.split(LINE_BREAK)) + : Stream.empty(); + } + + private String fetchS3Content(String key) { + var s3Driver = new S3Driver(s3ResourcesClient, RESOURCES_BUCKET); + return attempt(() -> s3Driver.getFile(UnixPath.of(key))).orElseThrow(); + } } diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/BatchIndexTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/BatchIndexTest.java index 29fcd9c97..00576fbed 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/BatchIndexTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/BatchIndexTest.java @@ -1,57 +1,47 @@ package no.unit.nva.indexingclient; import static no.unit.nva.indexingclient.TestConstants.RESOURCE_INDEX_NAME; - import static org.mockito.Mockito.mock; import com.amazonaws.services.lambda.runtime.Context; - +import java.util.List; +import java.util.Random; +import java.util.stream.Stream; import no.unit.nva.indexing.testutils.FakeIndexingClient; import no.unit.nva.indexingclient.models.IndexDocument; - import org.opensearch.action.DocWriteRequest.OpType; import org.opensearch.action.bulk.BulkItemResponse; import org.opensearch.action.bulk.BulkItemResponse.Failure; import org.opensearch.action.bulk.BulkResponse; -import java.util.List; -import java.util.Random; -import java.util.stream.Stream; - public class BatchIndexTest { - public static final Context CONTEXT = mock(Context.class); - private static final Random RANDOM = new Random(); - private static final int ARBITRARY_QUERY_TIME = 123; - - protected FakeIndexingClient failingOpenSearchClient() { - return new FakeIndexingClient() { - @Override - public Stream batchInsert(Stream indexDocuments) { - List itemResponses = - indexDocuments - .map(IndexDocument::getDocumentIdentifier) - .map(id -> createFailure(id)) - .map( - fail -> - new BulkItemResponse( - randomNumber(), OpType.UPDATE, fail)) - .toList(); - BulkResponse response = - new BulkResponse( - itemResponses.toArray(BulkItemResponse[]::new), - ARBITRARY_QUERY_TIME); - return Stream.of(response); - } - }; - } - - private Failure createFailure(String identifier) { - return new Failure( - RESOURCE_INDEX_NAME, identifier, new Exception("failingBulkIndexMessage")); - } - - private int randomNumber() { - return RANDOM.nextInt(); - } + public static final Context CONTEXT = mock(Context.class); + private static final Random RANDOM = new Random(); + private static final int ARBITRARY_QUERY_TIME = 123; + + protected FakeIndexingClient failingOpenSearchClient() { + return new FakeIndexingClient() { + @Override + public Stream batchInsert(Stream indexDocuments) { + List itemResponses = + indexDocuments + .map(IndexDocument::getDocumentIdentifier) + .map(id -> createFailure(id)) + .map(fail -> new BulkItemResponse(randomNumber(), OpType.UPDATE, fail)) + .toList(); + BulkResponse response = + new BulkResponse(itemResponses.toArray(BulkItemResponse[]::new), ARBITRARY_QUERY_TIME); + return Stream.of(response); + } + }; + } + + private Failure createFailure(String identifier) { + return new Failure(RESOURCE_INDEX_NAME, identifier, new Exception("failingBulkIndexMessage")); + } + + private int randomNumber() { + return RANDOM.nextInt(); + } } diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/EventBasedBatchIndexerTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/EventBasedBatchIndexerTest.java index afec7a083..e187a2e4d 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/EventBasedBatchIndexerTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/EventBasedBatchIndexerTest.java @@ -6,9 +6,7 @@ import static no.unit.nva.indexingclient.Constants.NUMBER_OF_FILES_PER_EVENT_ENVIRONMENT_VARIABLE; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; @@ -20,7 +18,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import no.unit.nva.commons.json.JsonUtils; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; @@ -29,12 +33,10 @@ import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.s3.S3Driver; import no.unit.nva.stubs.FakeS3Client; - import nva.commons.core.attempt.Try; import nva.commons.core.ioutils.IoUtils; import nva.commons.core.paths.UnixPath; import nva.commons.core.paths.UriWrapper; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -42,225 +44,210 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - @SuppressWarnings({"PMD.VariableDeclarationUsageDistance"}) public class EventBasedBatchIndexerTest extends BatchIndexTest { - private static ListAppender appender; - private EventBasedBatchIndexer indexer; - private ByteArrayOutputStream outputStream; - private FakeIndexingClient openSearchClient; - private StubEventBridgeClient eventBridgeClient; - private FakeS3Client s3Client; - private S3Driver s3Driver; - - @BeforeAll - public static void initClass() { - appender = getAppender(BatchIndexer.class); - } - - @BeforeEach - public void init() { - this.outputStream = new ByteArrayOutputStream(); - openSearchClient = mockOsClient(); - eventBridgeClient = new StubEventBridgeClient(); - s3Client = new FakeS3Client(); - s3Driver = new S3Driver(s3Client, "ingoredBucket"); - indexer = - new EventBasedBatchIndexer( - s3Client, - openSearchClient, - eventBridgeClient, - NUMBER_OF_FILES_PER_EVENT_ENVIRONMENT_VARIABLE); - } - - @ParameterizedTest( - name = - "should return all ids for published resources that failed to be indexed. " - + "Input size:{0}") - @ValueSource(ints = {1, 2, 5, 10, 100}) - public void shouldReturnsAllIdsForPublishedResourcesThatFailedToBeIndexed( - int numberOfFilesPerEvent) throws JsonProcessingException { - - indexer = - new EventBasedBatchIndexer( - s3Client, - failingOpenSearchClient(), - eventBridgeClient, - numberOfFilesPerEvent); - var filesFailingToBeIndexed = randomFilesInSingleEvent(s3Driver, numberOfFilesPerEvent); - var importLocation = filesFailingToBeIndexed.get(0).getHost().toString(); - var request = new ImportDataRequestEvent(importLocation); - indexer.handleRequest(eventStream(request), outputStream, CONTEXT); - - var actualIdentifiersOfNonIndexedEntries = - Arrays.stream( - objectMapperWithEmpty.readValue( - outputStream.toString(), String[].class)) - .toList(); - var expectedIdentifiesOfNonIndexedEntries = - Arrays.stream(extractIdentifiersFromFailingFiles(filesFailingToBeIndexed)).toList(); - - assertEquals(actualIdentifiersOfNonIndexedEntries, expectedIdentifiesOfNonIndexedEntries); - - expectedIdentifiesOfNonIndexedEntries.forEach( - expectedIdentifier -> - assertThat(logToString(appender), containsString(expectedIdentifier))); - } - - @Test - void batchIndexerParsesEvent() { - InputStream event = IoUtils.inputStreamFromResources("event.json"); - indexer.handleRequest(event, outputStream, CONTEXT); - } - - @ParameterizedTest(name = "batch indexer processes n files per request:{0}") - @ValueSource(ints = {1, 2, 5, 10, 50, 100}) - void shouldIndexNFilesPerEvent(int numberOfFilesPerEvent) throws IOException { - indexer = - new EventBasedBatchIndexer( - s3Client, openSearchClient, eventBridgeClient, numberOfFilesPerEvent); - var expectedFiles = randomFilesInSingleEvent(s3Driver, numberOfFilesPerEvent); - var unexpectedFile = randomEntryInS3(s3Driver); - - var importLocation = unexpectedFile.getHost().getUri(); // all files are in the same bucket - InputStream event = eventStream(new ImportDataRequestEvent(importLocation.toString())); - indexer.handleRequest(event, outputStream, CONTEXT); - - for (var expectedFile : expectedFiles) { - IndexDocument indexDocument = fetchIndexDocumentFromS3(expectedFile); - assertThat( - openSearchClient.getIndex(indexDocument.getIndexName()), - hasItem(indexDocument.resource())); - } - - IndexDocument notYetIndexedDocument = fetchIndexDocumentFromS3(unexpectedFile); - assertThat( - openSearchClient.getIndex(notYetIndexedDocument.getIndexName()), - not(hasItem(notYetIndexedDocument.resource()))); - } - - @Test - void shouldEmitEventForProcessingNextBatchWhenThereAreMoreFilesToProcess() throws IOException { - var firstFile = randomEntryInS3(s3Driver); - randomEntryInS3(s3Driver); // necessary second file for the emission of the next event - - String bucketUri = firstFile.getHost().getUri().toString(); - ImportDataRequestEvent firstEvent = new ImportDataRequestEvent(bucketUri); - var event = eventStream(firstEvent); - - indexer.handleRequest(event, outputStream, CONTEXT); - assertThat( - eventBridgeClient.getLatestEvent().getStartMarker(), - is(equalTo(firstFile.getLastPathElement()))); - } - - @Test - void shouldNotEmitEventWhenThereAreNoMoreFilesToProcess() throws IOException { - var firstFile = randomEntryInS3(s3Driver); - randomEntryInS3(s3Driver); - var bucketUri = firstFile.getHost().getUri().toString(); - var lastEvent = new ImportDataRequestEvent(bucketUri, firstFile.getLastPathElement()); - var event = eventStream(lastEvent); - - indexer.handleRequest(event, outputStream, CONTEXT); - assertThat(eventBridgeClient.getLatestEvent(), is(nullValue())); - } - - @Test - void shouldIndexFirstFilesInFirstEventAndSubsequentFilesInNextEvent() throws IOException { - var firstFile = randomEntryInS3(s3Driver); - var secondFile = randomEntryInS3(s3Driver); - var firstDocumentToIndex = fetchIndexDocumentFromS3(firstFile); - var secondDocumentIndex = fetchIndexDocumentFromS3(secondFile); - String bucketUri = firstFile.getHost().getUri().toString(); - - var firstEvent = new ImportDataRequestEvent(bucketUri); - indexer.handleRequest(eventStream(firstEvent), outputStream, CONTEXT); - assertThatIndexHasFirstButNotSecondDocument(firstDocumentToIndex, secondDocumentIndex); - - var secondEvent = new ImportDataRequestEvent(bucketUri, firstFile.getLastPathElement()); - indexer.handleRequest(eventStream(secondEvent), outputStream, CONTEXT); - assertThatIndexHasBothDocuments(firstDocumentToIndex, secondDocumentIndex); - } - - private void assertThatIndexHasBothDocuments( - IndexDocument firstDocumentToIndex, IndexDocument secondDocumentIndex) { - assertThat( - openSearchClient.getIndex(firstDocumentToIndex.getIndexName()), - hasItem(firstDocumentToIndex.resource())); - assertThat( - openSearchClient.getIndex(secondDocumentIndex.getIndexName()), - hasItem(secondDocumentIndex.resource())); - } - - private void assertThatIndexHasFirstButNotSecondDocument( - IndexDocument firstDocumentToIndex, IndexDocument secondDocumentIndex) { - assertThat( - openSearchClient.getIndex(firstDocumentToIndex.getIndexName()), - hasItem(firstDocumentToIndex.resource())); - assertThat( - openSearchClient.getIndex(secondDocumentIndex.getIndexName()), - not(hasItem(secondDocumentIndex.resource()))); - } - - private IndexDocument fetchIndexDocumentFromS3(UriWrapper expectedFile) { - String indexDocumentJson = s3Driver.getFile(expectedFile.toS3bucketPath()); - return IndexDocument.fromJsonString(indexDocumentJson); - } - - private String[] extractIdentifiersFromFailingFiles(List filesFailingToBeIndexed) { - return filesFailingToBeIndexed.stream() - .map(UriWrapper::getLastPathElement) - .toList() - .toArray(String[]::new); - } - - private List randomFilesInSingleEvent( - S3Driver s3Driver, int numberOfFilesPerEvent) { - return IntStream.range(0, numberOfFilesPerEvent) - .boxed() - .map(attempt(ignored -> randomEntryInS3(s3Driver))) - .map(Try::orElseThrow) - .collect(Collectors.toList()); - } - - private UriWrapper randomEntryInS3(S3Driver s3Driver) throws IOException { - var randomIndexDocument = randomIndexDocument(); - var filePath = UnixPath.of(randomIndexDocument.getDocumentIdentifier()); - return UriWrapper.fromUri( - s3Driver.insertFile(filePath, randomIndexDocument.toJsonString())); - } - - private IndexDocument randomIndexDocument() { - return new IndexDocument(randomEventConsumptionAttributes(), randomObject()); - } - - private JsonNode randomObject() { - var json = randomJson(); - return attempt(() -> JsonUtils.dtoObjectMapper.readTree(json)).orElseThrow(); - } - - private EventConsumptionAttributes randomEventConsumptionAttributes() { - return new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); - } - - private FakeIndexingClient mockOsClient() { - return new FakeIndexingClient(); - } - - private InputStream eventStream(ImportDataRequestEvent eventDetail) - throws JsonProcessingException { - AwsEventBridgeEvent event = new AwsEventBridgeEvent<>(); - event.setDetail(eventDetail); - String jsonString = objectMapperWithEmpty.writeValueAsString(event); - return IoUtils.stringToStream(jsonString); - } + private static ListAppender appender; + private EventBasedBatchIndexer indexer; + private ByteArrayOutputStream outputStream; + private FakeIndexingClient openSearchClient; + private StubEventBridgeClient eventBridgeClient; + private FakeS3Client s3Client; + private S3Driver s3Driver; + + @BeforeAll + public static void initClass() { + appender = getAppender(BatchIndexer.class); + } + + @BeforeEach + public void init() { + this.outputStream = new ByteArrayOutputStream(); + openSearchClient = mockOsClient(); + eventBridgeClient = new StubEventBridgeClient(); + s3Client = new FakeS3Client(); + s3Driver = new S3Driver(s3Client, "ingoredBucket"); + indexer = + new EventBasedBatchIndexer( + s3Client, + openSearchClient, + eventBridgeClient, + NUMBER_OF_FILES_PER_EVENT_ENVIRONMENT_VARIABLE); + } + + @ParameterizedTest( + name = + "should return all ids for published resources that failed to be indexed. " + + "Input size:{0}") + @ValueSource(ints = {1, 2, 5, 10, 100}) + public void shouldReturnsAllIdsForPublishedResourcesThatFailedToBeIndexed( + int numberOfFilesPerEvent) throws JsonProcessingException { + + indexer = + new EventBasedBatchIndexer( + s3Client, failingOpenSearchClient(), eventBridgeClient, numberOfFilesPerEvent); + var filesFailingToBeIndexed = randomFilesInSingleEvent(s3Driver, numberOfFilesPerEvent); + var importLocation = filesFailingToBeIndexed.get(0).getHost().toString(); + var request = new ImportDataRequestEvent(importLocation); + indexer.handleRequest(eventStream(request), outputStream, CONTEXT); + + var actualIdentifiersOfNonIndexedEntries = + Arrays.stream(objectMapperWithEmpty.readValue(outputStream.toString(), String[].class)) + .toList(); + var expectedIdentifiesOfNonIndexedEntries = + Arrays.stream(extractIdentifiersFromFailingFiles(filesFailingToBeIndexed)).toList(); + + assertEquals(actualIdentifiersOfNonIndexedEntries, expectedIdentifiesOfNonIndexedEntries); + + expectedIdentifiesOfNonIndexedEntries.forEach( + expectedIdentifier -> + assertThat(logToString(appender), containsString(expectedIdentifier))); + } + + @Test + void batchIndexerParsesEvent() { + InputStream event = IoUtils.inputStreamFromResources("event.json"); + indexer.handleRequest(event, outputStream, CONTEXT); + } + + @ParameterizedTest(name = "batch indexer processes n files per request:{0}") + @ValueSource(ints = {1, 2, 5, 10, 50, 100}) + void shouldIndexNFilesPerEvent(int numberOfFilesPerEvent) throws IOException { + indexer = + new EventBasedBatchIndexer( + s3Client, openSearchClient, eventBridgeClient, numberOfFilesPerEvent); + var expectedFiles = randomFilesInSingleEvent(s3Driver, numberOfFilesPerEvent); + var unexpectedFile = randomEntryInS3(s3Driver); + + var importLocation = unexpectedFile.getHost().getUri(); // all files are in the same bucket + InputStream event = eventStream(new ImportDataRequestEvent(importLocation.toString())); + indexer.handleRequest(event, outputStream, CONTEXT); + + for (var expectedFile : expectedFiles) { + IndexDocument indexDocument = fetchIndexDocumentFromS3(expectedFile); + assertThat( + openSearchClient.getIndex(indexDocument.getIndexName()), + hasItem(indexDocument.resource())); + } + + IndexDocument notYetIndexedDocument = fetchIndexDocumentFromS3(unexpectedFile); + assertThat( + openSearchClient.getIndex(notYetIndexedDocument.getIndexName()), + not(hasItem(notYetIndexedDocument.resource()))); + } + + @Test + void shouldEmitEventForProcessingNextBatchWhenThereAreMoreFilesToProcess() throws IOException { + var firstFile = randomEntryInS3(s3Driver); + randomEntryInS3(s3Driver); // necessary second file for the emission of the next event + + String bucketUri = firstFile.getHost().getUri().toString(); + ImportDataRequestEvent firstEvent = new ImportDataRequestEvent(bucketUri); + var event = eventStream(firstEvent); + + indexer.handleRequest(event, outputStream, CONTEXT); + assertThat( + eventBridgeClient.getLatestEvent().getStartMarker(), + is(equalTo(firstFile.getLastPathElement()))); + } + + @Test + void shouldNotEmitEventWhenThereAreNoMoreFilesToProcess() throws IOException { + var firstFile = randomEntryInS3(s3Driver); + randomEntryInS3(s3Driver); + var bucketUri = firstFile.getHost().getUri().toString(); + var lastEvent = new ImportDataRequestEvent(bucketUri, firstFile.getLastPathElement()); + var event = eventStream(lastEvent); + + indexer.handleRequest(event, outputStream, CONTEXT); + assertThat(eventBridgeClient.getLatestEvent(), is(nullValue())); + } + + @Test + void shouldIndexFirstFilesInFirstEventAndSubsequentFilesInNextEvent() throws IOException { + var firstFile = randomEntryInS3(s3Driver); + var secondFile = randomEntryInS3(s3Driver); + var firstDocumentToIndex = fetchIndexDocumentFromS3(firstFile); + var secondDocumentIndex = fetchIndexDocumentFromS3(secondFile); + String bucketUri = firstFile.getHost().getUri().toString(); + + var firstEvent = new ImportDataRequestEvent(bucketUri); + indexer.handleRequest(eventStream(firstEvent), outputStream, CONTEXT); + assertThatIndexHasFirstButNotSecondDocument(firstDocumentToIndex, secondDocumentIndex); + + var secondEvent = new ImportDataRequestEvent(bucketUri, firstFile.getLastPathElement()); + indexer.handleRequest(eventStream(secondEvent), outputStream, CONTEXT); + assertThatIndexHasBothDocuments(firstDocumentToIndex, secondDocumentIndex); + } + + private void assertThatIndexHasBothDocuments( + IndexDocument firstDocumentToIndex, IndexDocument secondDocumentIndex) { + assertThat( + openSearchClient.getIndex(firstDocumentToIndex.getIndexName()), + hasItem(firstDocumentToIndex.resource())); + assertThat( + openSearchClient.getIndex(secondDocumentIndex.getIndexName()), + hasItem(secondDocumentIndex.resource())); + } + + private void assertThatIndexHasFirstButNotSecondDocument( + IndexDocument firstDocumentToIndex, IndexDocument secondDocumentIndex) { + assertThat( + openSearchClient.getIndex(firstDocumentToIndex.getIndexName()), + hasItem(firstDocumentToIndex.resource())); + assertThat( + openSearchClient.getIndex(secondDocumentIndex.getIndexName()), + not(hasItem(secondDocumentIndex.resource()))); + } + + private IndexDocument fetchIndexDocumentFromS3(UriWrapper expectedFile) { + String indexDocumentJson = s3Driver.getFile(expectedFile.toS3bucketPath()); + return IndexDocument.fromJsonString(indexDocumentJson); + } + + private String[] extractIdentifiersFromFailingFiles(List filesFailingToBeIndexed) { + return filesFailingToBeIndexed.stream() + .map(UriWrapper::getLastPathElement) + .toList() + .toArray(String[]::new); + } + + private List randomFilesInSingleEvent(S3Driver s3Driver, int numberOfFilesPerEvent) { + return IntStream.range(0, numberOfFilesPerEvent) + .boxed() + .map(attempt(ignored -> randomEntryInS3(s3Driver))) + .map(Try::orElseThrow) + .collect(Collectors.toList()); + } + + private UriWrapper randomEntryInS3(S3Driver s3Driver) throws IOException { + var randomIndexDocument = randomIndexDocument(); + var filePath = UnixPath.of(randomIndexDocument.getDocumentIdentifier()); + return UriWrapper.fromUri(s3Driver.insertFile(filePath, randomIndexDocument.toJsonString())); + } + + private IndexDocument randomIndexDocument() { + return new IndexDocument(randomEventConsumptionAttributes(), randomObject()); + } + + private JsonNode randomObject() { + var json = randomJson(); + return attempt(() -> JsonUtils.dtoObjectMapper.readTree(json)).orElseThrow(); + } + + private EventConsumptionAttributes randomEventConsumptionAttributes() { + return new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); + } + + private FakeIndexingClient mockOsClient() { + return new FakeIndexingClient(); + } + + private InputStream eventStream(ImportDataRequestEvent eventDetail) + throws JsonProcessingException { + AwsEventBridgeEvent event = new AwsEventBridgeEvent<>(); + event.setDetail(eventDetail); + String jsonString = objectMapperWithEmpty.writeValueAsString(event); + return IoUtils.stringToStream(jsonString); + } } diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/ImportDataRequestEventTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/ImportDataRequestEventTest.java index 30ac59b43..da5aba302 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/ImportDataRequestEventTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/ImportDataRequestEventTest.java @@ -4,7 +4,6 @@ import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.constants.Words.SLASH; import static no.unit.nva.indexingclient.Constants.S3_LOCATION_FIELD; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; @@ -16,7 +15,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.exc.ValueInstantiationException; import com.fasterxml.jackson.databind.node.ObjectNode; - import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/StartBatchIndexingHandlerTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/StartBatchIndexingHandlerTest.java index f52f8c550..3fd07c932 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/StartBatchIndexingHandlerTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/StartBatchIndexingHandlerTest.java @@ -1,44 +1,42 @@ package no.unit.nva.indexingclient; import static no.unit.nva.indexingclient.Constants.PERSISTED_RESOURCES_PATH; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; class StartBatchIndexingHandlerTest extends BatchIndexTest { - private ByteArrayOutputStream outputStream; - private StubEventBridgeClient eventBridgeClient; + private ByteArrayOutputStream outputStream; + private StubEventBridgeClient eventBridgeClient; - @BeforeEach - public void initialize() { - outputStream = new ByteArrayOutputStream(); - eventBridgeClient = new StubEventBridgeClient(); - } + @BeforeEach + public void initialize() { + outputStream = new ByteArrayOutputStream(); + eventBridgeClient = new StubEventBridgeClient(); + } - @Test - void handlerSendsEventToEventBridgeWhenItReceivesAnImportRequest() throws IOException { + @Test + void handlerSendsEventToEventBridgeWhenItReceivesAnImportRequest() throws IOException { - var handler = newHandler(); + var handler = newHandler(); - handler.handleRequest(newImportRequest(), outputStream, CONTEXT); - var expectedImportRequest = new ImportDataRequestEvent(PERSISTED_RESOURCES_PATH); - assertThat(eventBridgeClient.getLatestEvent(), is(equalTo(expectedImportRequest))); - } + handler.handleRequest(newImportRequest(), outputStream, CONTEXT); + var expectedImportRequest = new ImportDataRequestEvent(PERSISTED_RESOURCES_PATH); + assertThat(eventBridgeClient.getLatestEvent(), is(equalTo(expectedImportRequest))); + } - private StartBatchIndexingHandler newHandler() { - return new StartBatchIndexingHandler(eventBridgeClient); - } + private StartBatchIndexingHandler newHandler() { + return new StartBatchIndexingHandler(eventBridgeClient); + } - private InputStream newImportRequest() { - return null; - } + private InputStream newImportRequest() { + return null; + } } diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/StubEventBridgeClient.java b/batch-index/src/test/java/no/unit/nva/indexingclient/StubEventBridgeClient.java index fa18d2215..2947fc00c 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/StubEventBridgeClient.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/StubEventBridgeClient.java @@ -1,11 +1,9 @@ package no.unit.nva.indexingclient; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; - import static nva.commons.core.attempt.Try.attempt; import nva.commons.core.SingletonCollector; - import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequest; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequestEntry; @@ -13,33 +11,31 @@ public class StubEventBridgeClient implements EventBridgeClient { - private ImportDataRequestEvent latestEvent; - - public ImportDataRequestEvent getLatestEvent() { - return latestEvent; - } - - public PutEventsResponse putEvents(PutEventsRequest putEventsRequest) { - this.latestEvent = saveContainedEvent(putEventsRequest); - return PutEventsResponse.builder().failedEntryCount(0).build(); - } - - @Override - public String serviceName() { - return null; - } - - @Override - public void close() {} - - private ImportDataRequestEvent saveContainedEvent(PutEventsRequest putEventsRequest) { - PutEventsRequestEntry eventEntry = - putEventsRequest.entries().stream().collect(SingletonCollector.collect()); - return attempt(eventEntry::detail) - .map( - jsonString -> - objectMapperWithEmpty.readValue( - jsonString, ImportDataRequestEvent.class)) - .orElseThrow(); - } + private ImportDataRequestEvent latestEvent; + + public ImportDataRequestEvent getLatestEvent() { + return latestEvent; + } + + public PutEventsResponse putEvents(PutEventsRequest putEventsRequest) { + this.latestEvent = saveContainedEvent(putEventsRequest); + return PutEventsResponse.builder().failedEntryCount(0).build(); + } + + @Override + public String serviceName() { + return null; + } + + @Override + public void close() {} + + private ImportDataRequestEvent saveContainedEvent(PutEventsRequest putEventsRequest) { + PutEventsRequestEntry eventEntry = + putEventsRequest.entries().stream().collect(SingletonCollector.collect()); + return attempt(eventEntry::detail) + .map( + jsonString -> objectMapperWithEmpty.readValue(jsonString, ImportDataRequestEvent.class)) + .orElseThrow(); + } } diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandlerTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandlerTest.java index 4594b0dca..2438ed72d 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandlerTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/GenerateKeyBatchesHandlerTest.java @@ -3,9 +3,7 @@ import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.indexingclient.TestConstants.RESOURCE_INDEX_NAME; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.core.Is.is; @@ -13,32 +11,27 @@ import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; - +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.net.URI; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; import no.unit.nva.s3.S3Driver; import no.unit.nva.stubs.FakeS3Client; - import nva.commons.core.SingletonCollector; import nva.commons.core.ioutils.IoUtils; import nva.commons.core.paths.UnixPath; import nva.commons.core.paths.UriWrapper; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; - import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequest; import software.amazon.awssdk.services.eventbridge.model.PutEventsResponse; import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.net.URI; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - class GenerateKeyBatchesHandlerTest { private static final String OUTPUT_BUCKET = "outputBucket"; diff --git a/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandlerTest.java b/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandlerTest.java index 45cc51c46..188058584 100644 --- a/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandlerTest.java +++ b/batch-index/src/test/java/no/unit/nva/indexingclient/keybatch/KeyBasedBatchIndexHandlerTest.java @@ -1,13 +1,12 @@ package no.unit.nva.indexingclient.keybatch; +import static java.util.UUID.randomUUID; import static no.unit.nva.LogAppender.getAppender; import static no.unit.nva.LogAppender.logToString; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.indexingclient.TestConstants.RESOURCE_INDEX_NAME; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -18,12 +17,18 @@ import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; -import static java.util.UUID.randomUUID; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; import no.unit.nva.indexingclient.IndexingClient; @@ -31,12 +36,10 @@ import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.s3.S3Driver; import no.unit.nva.stubs.FakeS3Client; - import nva.commons.core.SingletonCollector; import nva.commons.core.StringUtils; import nva.commons.core.ioutils.IoUtils; import nva.commons.core.paths.UnixPath; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -45,22 +48,11 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mockito; import org.opensearch.action.bulk.BulkResponse; - import software.amazon.awssdk.services.eventbridge.EventBridgeClient; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequest; import software.amazon.awssdk.services.eventbridge.model.PutEventsRequestEntry; import software.amazon.awssdk.services.eventbridge.model.PutEventsResponse; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; - class KeyBasedBatchIndexHandlerTest { private static final String LINE_BREAK = "\n"; diff --git a/buildSrc/src/main/groovy/nva.search.service.java-conventions.gradle b/buildSrc/src/main/groovy/nva.search.service.java-conventions.gradle index 5a71d0bb6..7769d92c6 100644 --- a/buildSrc/src/main/groovy/nva.search.service.java-conventions.gradle +++ b/buildSrc/src/main/groovy/nva.search.service.java-conventions.gradle @@ -76,7 +76,7 @@ tasks.named('test', Test) { spotless { java { toggleOffOn() // Ignores sections between `spotless:off` / `spotless:on` - googleJavaFormat().aosp().reflowLongStrings().formatJavadoc(true).reorderImports(true) + googleJavaFormat().reflowLongStrings().formatJavadoc(true).reorderImports(true) } format 'misc', { diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandler.java index 55204563c..90914dd91 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandler.java @@ -1,55 +1,52 @@ package no.unit.nva.indexing.handlers; import com.amazonaws.services.lambda.runtime.Context; - import no.unit.nva.events.handlers.DestinationsEventBridgeEventHandler; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.indexing.model.DeleteImportCandidateEvent; import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DeleteImportCandidateFromIndexHandler - extends DestinationsEventBridgeEventHandler { - - public static final String REMOVING_DOCUMENT_FAILED_MESSAGE = "Removing document failed"; - public static final String REMOVED_FROM_INDEX_MESSAGE = - "Import candidate has been removed from index"; - private static final Logger logger = - LoggerFactory.getLogger(DeleteImportCandidateFromIndexHandler.class); - private final IndexingClient indexingClient; - - @JacocoGenerated - public DeleteImportCandidateFromIndexHandler() { - this(IndexingClient.defaultIndexingClient()); + extends DestinationsEventBridgeEventHandler { + + public static final String REMOVING_DOCUMENT_FAILED_MESSAGE = "Removing document failed"; + public static final String REMOVED_FROM_INDEX_MESSAGE = + "Import candidate has been removed from index"; + private static final Logger logger = + LoggerFactory.getLogger(DeleteImportCandidateFromIndexHandler.class); + private final IndexingClient indexingClient; + + @JacocoGenerated + public DeleteImportCandidateFromIndexHandler() { + this(IndexingClient.defaultIndexingClient()); + } + + public DeleteImportCandidateFromIndexHandler(IndexingClient indexingClient) { + super(DeleteImportCandidateEvent.class); + this.indexingClient = indexingClient; + } + + @Override + protected Void processInputPayload( + DeleteImportCandidateEvent input, + AwsEventBridgeEvent> event, + Context context) { + try { + indexingClient.removeDocumentFromImportCandidateIndex(input.identifier().toString()); + logger.info(REMOVED_FROM_INDEX_MESSAGE); + } catch (Exception e) { + logError(e); + throw new RuntimeException(e); } - public DeleteImportCandidateFromIndexHandler(IndexingClient indexingClient) { - super(DeleteImportCandidateEvent.class); - this.indexingClient = indexingClient; - } + return null; + } - @Override - protected Void processInputPayload( - DeleteImportCandidateEvent input, - AwsEventBridgeEvent> event, - Context context) { - try { - indexingClient.removeDocumentFromImportCandidateIndex(input.identifier().toString()); - logger.info(REMOVED_FROM_INDEX_MESSAGE); - } catch (Exception e) { - logError(e); - throw new RuntimeException(e); - } - - return null; - } - - private void logError(Exception exception) { - logger.warn(REMOVING_DOCUMENT_FAILED_MESSAGE, exception); - } + private void logError(Exception exception) { + logger.warn(REMOVING_DOCUMENT_FAILED_MESSAGE, exception); + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandler.java index 279b23846..7ff56e12e 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandler.java @@ -1,44 +1,40 @@ package no.unit.nva.indexing.handlers; import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; - import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; - import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DeleteImportCandidateIndexHandler implements RequestHandler { - public static final String FINISHED = "FINISHED"; - public static final String INDEX_DELETION_FAILED_MESSAGE = "Index deletion failed"; - private static final Logger logger = LoggerFactory.getLogger(DeleteIndicesHandler.class); - private final IndexingClient indexingClient; - - @JacocoGenerated - public DeleteImportCandidateIndexHandler() { - this(IndexingClient.defaultIndexingClient()); - } - - public DeleteImportCandidateIndexHandler(IndexingClient indexingClient) { - this.indexingClient = indexingClient; - } - - @Override - public String handleRequest(Object input, Context context) { - attempt(() -> indexingClient.deleteIndex(IMPORT_CANDIDATES_INDEX)) - .orElse(fail -> logError(fail.getException())); - return FINISHED; - } - - private Void logError(Exception exception) { - logger.warn(INDEX_DELETION_FAILED_MESSAGE, exception); - return null; - } + public static final String FINISHED = "FINISHED"; + public static final String INDEX_DELETION_FAILED_MESSAGE = "Index deletion failed"; + private static final Logger logger = LoggerFactory.getLogger(DeleteIndicesHandler.class); + private final IndexingClient indexingClient; + + @JacocoGenerated + public DeleteImportCandidateIndexHandler() { + this(IndexingClient.defaultIndexingClient()); + } + + public DeleteImportCandidateIndexHandler(IndexingClient indexingClient) { + this.indexingClient = indexingClient; + } + + @Override + public String handleRequest(Object input, Context context) { + attempt(() -> indexingClient.deleteIndex(IMPORT_CANDIDATES_INDEX)) + .orElse(fail -> logError(fail.getException())); + return FINISHED; + } + + private Void logError(Exception exception) { + logger.warn(INDEX_DELETION_FAILED_MESSAGE, exception); + return null; + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteIndicesHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteIndicesHandler.java index a50ed3185..b9dec20d7 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteIndicesHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteIndicesHandler.java @@ -5,16 +5,12 @@ import static no.unit.nva.constants.Words.PUBLISHING_REQUESTS_INDEX; import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.constants.Words.TICKETS; - import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; - import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceEvent.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceEvent.java index 8f6ee5f19..10b8f8ffa 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceEvent.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceEvent.java @@ -2,54 +2,51 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - +import java.util.Objects; import no.unit.nva.commons.json.JsonSerializable; import no.unit.nva.identifiers.SortableIdentifier; - import nva.commons.core.JacocoGenerated; -import java.util.Objects; - public class DeleteResourceEvent implements JsonSerializable { - public static final String EVENT_TOPIC = "PublicationService.ExpandedEntry.Deleted"; - - private final String topic; - private final SortableIdentifier identifier; - - @JsonCreator - public DeleteResourceEvent( - @JsonProperty("topic") String topic, - @JsonProperty("identifier") SortableIdentifier identifier) { - this.topic = topic; - this.identifier = identifier; + public static final String EVENT_TOPIC = "PublicationService.ExpandedEntry.Deleted"; + + private final String topic; + private final SortableIdentifier identifier; + + @JsonCreator + public DeleteResourceEvent( + @JsonProperty("topic") String topic, + @JsonProperty("identifier") SortableIdentifier identifier) { + this.topic = topic; + this.identifier = identifier; + } + + @JacocoGenerated + public String getTopic() { + return topic; + } + + public SortableIdentifier getIdentifier() { + return identifier; + } + + @JacocoGenerated + @Override + public int hashCode() { + return Objects.hash(getTopic(), getIdentifier()); + } + + @Override + @JacocoGenerated + public boolean equals(Object o) { + if (this == o) { + return true; } - - @JacocoGenerated - public String getTopic() { - return topic; - } - - public SortableIdentifier getIdentifier() { - return identifier; - } - - @JacocoGenerated - @Override - public int hashCode() { - return Objects.hash(getTopic(), getIdentifier()); - } - - @Override - @JacocoGenerated - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DeleteResourceEvent that = (DeleteResourceEvent) o; - return topic.equals(that.topic) && identifier.equals(that.identifier); + if (o == null || getClass() != o.getClass()) { + return false; } + DeleteResourceEvent that = (DeleteResourceEvent) o; + return topic.equals(that.topic) && identifier.equals(that.identifier); + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandler.java index 9d958f336..272602a71 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandler.java @@ -1,14 +1,11 @@ package no.unit.nva.indexing.handlers; import com.amazonaws.services.lambda.runtime.Context; - import no.unit.nva.events.handlers.DestinationsEventBridgeEventHandler; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandler.java index fbf2c6a7d..f522be9e7 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandler.java @@ -1,59 +1,54 @@ package no.unit.nva.indexing.handlers; import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; - import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; - +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; import no.unit.nva.indexing.model.IndexRequest; import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.Failure; import nva.commons.core.ioutils.IoUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.util.concurrent.atomic.AtomicBoolean; - public class ImportCandidateInitHandler implements RequestHandler { - public static final String SUCCESS = "SUCCESS"; - public static final String FAILED = "FAILED. See logs"; - public static final String IMPORT_CANDIDATE_MAPPINGS_JSON = "import_candidate_mappings.json"; - private static final String IMPORT_CANDIDATE_MAPPINGS = - IoUtils.stringFromResources(Path.of(IMPORT_CANDIDATE_MAPPINGS_JSON)); - private static final IndexRequest INDEX = - new IndexRequest(IMPORT_CANDIDATES_INDEX, IMPORT_CANDIDATE_MAPPINGS); - private static final Logger logger = LoggerFactory.getLogger(InitHandler.class); - private final IndexingClient indexingClient; - - @JacocoGenerated - public ImportCandidateInitHandler() { - this(IndexingClient.defaultIndexingClient()); - } - - public ImportCandidateInitHandler(IndexingClient indexingClient) { - this.indexingClient = indexingClient; - } - - @Override - public String handleRequest(Object input, Context context) { - var failState = new AtomicBoolean(false); - - attempt(() -> indexingClient.createIndex(INDEX.getName(), INDEX.getMappings())) - .orElse(fail -> handleFailure(failState, fail)); - - return failState.get() ? FAILED : SUCCESS; - } - - private Void handleFailure(AtomicBoolean failState, Failure failure) { - failState.set(true); - logger.warn("Index creation failed", failure.getException()); - return null; - } + public static final String SUCCESS = "SUCCESS"; + public static final String FAILED = "FAILED. See logs"; + public static final String IMPORT_CANDIDATE_MAPPINGS_JSON = "import_candidate_mappings.json"; + private static final String IMPORT_CANDIDATE_MAPPINGS = + IoUtils.stringFromResources(Path.of(IMPORT_CANDIDATE_MAPPINGS_JSON)); + private static final IndexRequest INDEX = + new IndexRequest(IMPORT_CANDIDATES_INDEX, IMPORT_CANDIDATE_MAPPINGS); + private static final Logger logger = LoggerFactory.getLogger(InitHandler.class); + private final IndexingClient indexingClient; + + @JacocoGenerated + public ImportCandidateInitHandler() { + this(IndexingClient.defaultIndexingClient()); + } + + public ImportCandidateInitHandler(IndexingClient indexingClient) { + this.indexingClient = indexingClient; + } + + @Override + public String handleRequest(Object input, Context context) { + var failState = new AtomicBoolean(false); + + attempt(() -> indexingClient.createIndex(INDEX.getName(), INDEX.getMappings())) + .orElse(fail -> handleFailure(failState, fail)); + + return failState.get() ? FAILED : SUCCESS; + } + + private Void handleFailure(AtomicBoolean failState, Failure failure) { + failState.set(true); + logger.warn("Index creation failed", failure.getException()); + return null; + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandler.java index d372383d9..15dd040c1 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandler.java @@ -3,7 +3,6 @@ import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; - import no.unit.nva.events.handlers.DestinationsEventBridgeEventHandler; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; @@ -11,49 +10,48 @@ import no.unit.nva.indexingclient.IndexingClient; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.s3.S3Driver; - import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; import nva.commons.core.paths.UnixPath; import nva.commons.core.paths.UriWrapper; public class IndexImportCandidateHandler - extends DestinationsEventBridgeEventHandler { - - private static final String EXPANDED_RESOURCES_BUCKET = - new Environment().readEnv("EXPANDED_RESOURCES_BUCKET"); - private final S3Driver s3Driver; - private final IndexingClient indexingClient; - - @JacocoGenerated - public IndexImportCandidateHandler() { - this(new S3Driver(EXPANDED_RESOURCES_BUCKET), defaultIndexingClient()); - } - - public IndexImportCandidateHandler(S3Driver s3Driver, IndexingClient indexingClient) { - super(EventReference.class); - this.s3Driver = s3Driver; - this.indexingClient = indexingClient; - } - - @JacocoGenerated - public static IndexingClient defaultIndexingClient() { - return IndexingClient.defaultIndexingClient(); - } - - @Override - protected Void processInputPayload( - EventReference input, - AwsEventBridgeEvent> event, - Context context) { - var resourceRelativePath = UriWrapper.fromUri(input.getUri()).toS3bucketPath(); - var indexDocument = fetchFileFromS3Bucket(resourceRelativePath).validate(); - attempt(() -> indexingClient.addDocumentToIndex(indexDocument)).orElseThrow(); - return null; - } - - private IndexDocument fetchFileFromS3Bucket(UnixPath resourceRelativePath) { - var resource = s3Driver.getFile(resourceRelativePath); - return IndexDocument.fromJsonString(resource); - } + extends DestinationsEventBridgeEventHandler { + + private static final String EXPANDED_RESOURCES_BUCKET = + new Environment().readEnv("EXPANDED_RESOURCES_BUCKET"); + private final S3Driver s3Driver; + private final IndexingClient indexingClient; + + @JacocoGenerated + public IndexImportCandidateHandler() { + this(new S3Driver(EXPANDED_RESOURCES_BUCKET), defaultIndexingClient()); + } + + public IndexImportCandidateHandler(S3Driver s3Driver, IndexingClient indexingClient) { + super(EventReference.class); + this.s3Driver = s3Driver; + this.indexingClient = indexingClient; + } + + @JacocoGenerated + public static IndexingClient defaultIndexingClient() { + return IndexingClient.defaultIndexingClient(); + } + + @Override + protected Void processInputPayload( + EventReference input, + AwsEventBridgeEvent> event, + Context context) { + var resourceRelativePath = UriWrapper.fromUri(input.getUri()).toS3bucketPath(); + var indexDocument = fetchFileFromS3Bucket(resourceRelativePath).validate(); + attempt(() -> indexingClient.addDocumentToIndex(indexDocument)).orElseThrow(); + return null; + } + + private IndexDocument fetchFileFromS3Bucket(UnixPath resourceRelativePath) { + var resource = s3Driver.getFile(resourceRelativePath); + return IndexDocument.fromJsonString(resource); + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexResourceHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexResourceHandler.java index 57836ea6b..84b504919 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexResourceHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/IndexResourceHandler.java @@ -1,11 +1,9 @@ package no.unit.nva.indexing.handlers; import static no.unit.nva.constants.Defaults.ENVIRONMENT; - import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; - import no.unit.nva.events.handlers.DestinationsEventBridgeEventHandler; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; @@ -16,79 +14,74 @@ import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.indexingclient.models.QueueClient; import no.unit.nva.s3.S3Driver; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.Failure; import nva.commons.core.paths.UnixPath; import nva.commons.core.paths.UriWrapper; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class IndexResourceHandler - extends DestinationsEventBridgeEventHandler { + extends DestinationsEventBridgeEventHandler { - public static final String INDEXING_MESSAGE = "Indexing document with id: {} to {}"; - private static final Logger LOGGER = LoggerFactory.getLogger(IndexResourceHandler.class); - private static final String EXPANDED_RESOURCES_BUCKET = - ENVIRONMENT.readEnv("EXPANDED_RESOURCES_BUCKET"); - private static final String SENT_TO_RECOVERY_QUEUE_MESSAGE = - "IndexDocument for index {} has been sent to recovery queue: {}"; - private final S3Driver resourcesS3Driver; - private final IndexingClient indexingClient; - private final QueueClient queueClient; + public static final String INDEXING_MESSAGE = "Indexing document with id: {} to {}"; + private static final Logger LOGGER = LoggerFactory.getLogger(IndexResourceHandler.class); + private static final String EXPANDED_RESOURCES_BUCKET = + ENVIRONMENT.readEnv("EXPANDED_RESOURCES_BUCKET"); + private static final String SENT_TO_RECOVERY_QUEUE_MESSAGE = + "IndexDocument for index {} has been sent to recovery queue: {}"; + private final S3Driver resourcesS3Driver; + private final IndexingClient indexingClient; + private final QueueClient queueClient; - @JacocoGenerated - public IndexResourceHandler() { - this( - new S3Driver(EXPANDED_RESOURCES_BUCKET), - defaultIndexingClient(), - IndexQueueClient.defaultQueueClient()); - } + @JacocoGenerated + public IndexResourceHandler() { + this( + new S3Driver(EXPANDED_RESOURCES_BUCKET), + defaultIndexingClient(), + IndexQueueClient.defaultQueueClient()); + } - public IndexResourceHandler( - S3Driver resourcesS3Driver, IndexingClient indexingClient, QueueClient queueClient) { - super(EventReference.class); - this.resourcesS3Driver = resourcesS3Driver; - this.indexingClient = indexingClient; - this.queueClient = queueClient; - } + public IndexResourceHandler( + S3Driver resourcesS3Driver, IndexingClient indexingClient, QueueClient queueClient) { + super(EventReference.class); + this.resourcesS3Driver = resourcesS3Driver; + this.indexingClient = indexingClient; + this.queueClient = queueClient; + } - @JacocoGenerated - public static IndexingClient defaultIndexingClient() { - return IndexingClient.defaultIndexingClient(); - } + @JacocoGenerated + public static IndexingClient defaultIndexingClient() { + return IndexingClient.defaultIndexingClient(); + } - @Override - protected Void processInputPayload( - EventReference input, - AwsEventBridgeEvent> event, - Context context) { + @Override + protected Void processInputPayload( + EventReference input, + AwsEventBridgeEvent> event, + Context context) { - var resourceRelativePath = UriWrapper.fromUri(input.getUri()).toS3bucketPath(); - var indexDocument = fetchFileFromS3Bucket(resourceRelativePath).validate(); - attempt(() -> indexingClient.addDocumentToIndex(indexDocument)) - .orElse(failure -> persistRecoveryMessage(failure, indexDocument)); - LOGGER.info( - INDEXING_MESSAGE, - indexDocument.getDocumentIdentifier(), - indexDocument.getIndexName()); - return null; - } + var resourceRelativePath = UriWrapper.fromUri(input.getUri()).toS3bucketPath(); + var indexDocument = fetchFileFromS3Bucket(resourceRelativePath).validate(); + attempt(() -> indexingClient.addDocumentToIndex(indexDocument)) + .orElse(failure -> persistRecoveryMessage(failure, indexDocument)); + LOGGER.info( + INDEXING_MESSAGE, indexDocument.getDocumentIdentifier(), indexDocument.getIndexName()); + return null; + } - private Void persistRecoveryMessage(Failure failure, IndexDocument indexDocument) { - var documentIdentifier = indexDocument.getDocumentIdentifier(); - RecoveryEntry.fromIndexDocument(indexDocument) - .withIdentifier(documentIdentifier) - .withException(failure.getException()) - .persist(queueClient); - LOGGER.error( - SENT_TO_RECOVERY_QUEUE_MESSAGE, indexDocument.getIndexName(), documentIdentifier); - return null; - } + private Void persistRecoveryMessage(Failure failure, IndexDocument indexDocument) { + var documentIdentifier = indexDocument.getDocumentIdentifier(); + RecoveryEntry.fromIndexDocument(indexDocument) + .withIdentifier(documentIdentifier) + .withException(failure.getException()) + .persist(queueClient); + LOGGER.error(SENT_TO_RECOVERY_QUEUE_MESSAGE, indexDocument.getIndexName(), documentIdentifier); + return null; + } - private IndexDocument fetchFileFromS3Bucket(UnixPath resourceRelativePath) { - var resource = resourcesS3Driver.getFile(resourceRelativePath); - return IndexDocument.fromJsonString(resource); - } + private IndexDocument fetchFileFromS3Bucket(UnixPath resourceRelativePath) { + var resource = resourcesS3Driver.getFile(resourceRelativePath); + return IndexDocument.fromJsonString(resource); + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/InitHandler.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/InitHandler.java index ce752d45b..fc85ef057 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/InitHandler.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/handlers/InitHandler.java @@ -6,26 +6,21 @@ import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.constants.Words.TICKETS; import static no.unit.nva.indexingclient.IndexingClient.defaultIndexingClient; - import static nva.commons.core.attempt.Try.attempt; import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.RequestHandler; - +import java.nio.file.Path; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import no.unit.nva.indexing.model.IndexRequest; import no.unit.nva.indexingclient.IndexingClient; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.Failure; import nva.commons.core.ioutils.IoUtils; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.nio.file.Path; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - public class InitHandler implements RequestHandler { public static final String SUCCESS = "SUCCESS"; diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/model/DeleteImportCandidateEvent.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/model/DeleteImportCandidateEvent.java index 2d14a06fe..3e48bddda 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/model/DeleteImportCandidateEvent.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/model/DeleteImportCandidateEvent.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; - import no.unit.nva.commons.json.JsonSerializable; import no.unit.nva.identifiers.SortableIdentifier; diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/model/IndexRequest.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/model/IndexRequest.java index 222649da9..31cda38e9 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/model/IndexRequest.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/model/IndexRequest.java @@ -3,50 +3,48 @@ import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.core.type.TypeReference; - -import no.unit.nva.commons.json.JsonUtils; - import java.util.Collections; import java.util.Map; +import no.unit.nva.commons.json.JsonUtils; public class IndexRequest { - private final String name; - private final Map mappings; - private final Map settings; - - public IndexRequest(String name) { - this.name = name; - this.mappings = Collections.emptyMap(); - this.settings = Collections.emptyMap(); - } - - public IndexRequest(String name, String jsonMappings) { - this.name = name; - this.mappings = jsonToJavaMap(jsonMappings); - this.settings = Collections.emptyMap(); - } - - public IndexRequest(String name, String jsonMappings, String jsonSettings) { - this.name = name; - this.mappings = jsonToJavaMap(jsonMappings); - this.settings = jsonToJavaMap(jsonSettings); - } - - private static Map jsonToJavaMap(String jsonMappings) { - var typeReference = new TypeReference>() {}; - return attempt(() -> JsonUtils.dtoObjectMapper.readValue(jsonMappings, typeReference)) - .orElseThrow(); - } - - public String getName() { - return name; - } - - public Map getMappings() { - return mappings; - } - - public Map getSettings() { - return settings; - } + private final String name; + private final Map mappings; + private final Map settings; + + public IndexRequest(String name) { + this.name = name; + this.mappings = Collections.emptyMap(); + this.settings = Collections.emptyMap(); + } + + public IndexRequest(String name, String jsonMappings) { + this.name = name; + this.mappings = jsonToJavaMap(jsonMappings); + this.settings = Collections.emptyMap(); + } + + public IndexRequest(String name, String jsonMappings, String jsonSettings) { + this.name = name; + this.mappings = jsonToJavaMap(jsonMappings); + this.settings = jsonToJavaMap(jsonSettings); + } + + private static Map jsonToJavaMap(String jsonMappings) { + var typeReference = new TypeReference>() {}; + return attempt(() -> JsonUtils.dtoObjectMapper.readValue(jsonMappings, typeReference)) + .orElseThrow(); + } + + public String getName() { + return name; + } + + public Map getMappings() { + return mappings; + } + + public Map getSettings() { + return settings; + } } diff --git a/indexing-handlers/src/main/java/no/unit/nva/indexing/utils/RecoveryEntry.java b/indexing-handlers/src/main/java/no/unit/nva/indexing/utils/RecoveryEntry.java index 8de354d2b..132f82133 100644 --- a/indexing-handlers/src/main/java/no/unit/nva/indexing/utils/RecoveryEntry.java +++ b/indexing-handlers/src/main/java/no/unit/nva/indexing/utils/RecoveryEntry.java @@ -1,109 +1,103 @@ package no.unit.nva.indexing.utils; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.Map; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.indexingclient.models.QueueClient; - import nva.commons.core.Environment; - import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; import software.amazon.awssdk.services.sqs.model.SendMessageRequest; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.Map; - public final class RecoveryEntry { - public static final String DATA_TYPE_STRING = "String"; - private static final String ID = "id"; - private static final String RECOVERY_QUEUE = "RECOVERY_QUEUE"; - private static final String TYPE = "type"; - private final String identifier; - private final String type; - private final String exception; - - private RecoveryEntry(String identifier, String type, String exception) { - this.identifier = identifier; - this.type = type; - this.exception = exception; - } - - public static RecoveryEntry fromIndexDocument(IndexDocument indexDocument) { - return builder().withType(indexDocument.getType()).build(); - } - - private static Builder builder() { - return new Builder(); - } - - public RecoveryEntry withIdentifier(String identifier) { - return this.copy().withIdentifier(identifier).build(); + public static final String DATA_TYPE_STRING = "String"; + private static final String ID = "id"; + private static final String RECOVERY_QUEUE = "RECOVERY_QUEUE"; + private static final String TYPE = "type"; + private final String identifier; + private final String type; + private final String exception; + + private RecoveryEntry(String identifier, String type, String exception) { + this.identifier = identifier; + this.type = type; + this.exception = exception; + } + + public static RecoveryEntry fromIndexDocument(IndexDocument indexDocument) { + return builder().withType(indexDocument.getType()).build(); + } + + private static Builder builder() { + return new Builder(); + } + + public RecoveryEntry withIdentifier(String identifier) { + return this.copy().withIdentifier(identifier).build(); + } + + public void persist(QueueClient queueClient) { + queueClient.sendMessage(createSendMessageRequest()); + } + + private SendMessageRequest createSendMessageRequest() { + return SendMessageRequest.builder() + .messageAttributes( + Map.of( + ID, convertToMessageAttribute(identifier), + TYPE, convertToMessageAttribute(type))) + .messageBody(exception) + .queueUrl(new Environment().readEnv(RECOVERY_QUEUE)) + .build(); + } + + private MessageAttributeValue convertToMessageAttribute(String value) { + return MessageAttributeValue.builder().stringValue(value).dataType(DATA_TYPE_STRING).build(); + } + + public RecoveryEntry withException(Exception exception) { + return this.copy().withException(getStackTrace(exception)).build(); + } + + private String getStackTrace(Exception exception) { + var stringWriter = new StringWriter(); + exception.printStackTrace(new PrintWriter(stringWriter)); + return stringWriter.toString(); + } + + private Builder copy() { + return new Builder() + .withIdentifier(this.identifier) + .withType(this.type) + .withException(this.exception); + } + + private static final class Builder { + + private String identifier; + private String type; + private String failure; + + private Builder() {} + + public Builder withIdentifier(String identifier) { + this.identifier = identifier; + return this; } - public void persist(QueueClient queueClient) { - queueClient.sendMessage(createSendMessageRequest()); + public Builder withType(String type) { + this.type = type; + return this; } - private SendMessageRequest createSendMessageRequest() { - return SendMessageRequest.builder() - .messageAttributes( - Map.of( - ID, convertToMessageAttribute(identifier), - TYPE, convertToMessageAttribute(type))) - .messageBody(exception) - .queueUrl(new Environment().readEnv(RECOVERY_QUEUE)) - .build(); + public Builder withException(String failure) { + this.failure = failure; + return this; } - private MessageAttributeValue convertToMessageAttribute(String value) { - return MessageAttributeValue.builder() - .stringValue(value) - .dataType(DATA_TYPE_STRING) - .build(); - } - - public RecoveryEntry withException(Exception exception) { - return this.copy().withException(getStackTrace(exception)).build(); - } - - private String getStackTrace(Exception exception) { - var stringWriter = new StringWriter(); - exception.printStackTrace(new PrintWriter(stringWriter)); - return stringWriter.toString(); - } - - private Builder copy() { - return new Builder() - .withIdentifier(this.identifier) - .withType(this.type) - .withException(this.exception); - } - - private static final class Builder { - - private String identifier; - private String type; - private String failure; - - private Builder() {} - - public Builder withIdentifier(String identifier) { - this.identifier = identifier; - return this; - } - - public Builder withType(String type) { - this.type = type; - return this; - } - - public Builder withException(String failure) { - this.failure = failure; - return this; - } - - public RecoveryEntry build() { - return new RecoveryEntry(identifier, type, failure); - } + public RecoveryEntry build() { + return new RecoveryEntry(identifier, type, failure); } + } } diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandlerTest.java index c6d66b49d..713cab3f7 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateFromIndexHandlerTest.java @@ -3,9 +3,7 @@ import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.not; import static org.hamcrest.collection.IsIterableContainingInOrder.contains; @@ -14,7 +12,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; @@ -22,89 +24,79 @@ import no.unit.nva.indexing.testutils.FakeIndexingClient; import no.unit.nva.indexingclient.models.EventConsumptionAttributes; import no.unit.nva.indexingclient.models.IndexDocument; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Set; - public class DeleteImportCandidateFromIndexHandlerTest { - public static final String SOMETHING_BAD_HAPPENED = "Something bad happened"; - private static final Context CONTEXT = Mockito.mock(Context.class); - private FakeIndexingClient indexingClient; - - private ByteArrayOutputStream output; - - private DeleteImportCandidateFromIndexHandler handler; - - private static IndexDocument createSampleResource(SortableIdentifier identifierProvider) { - String randomJson = randomJson(); - ObjectNode objectNode = - attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)) - .orElseThrow(); - EventConsumptionAttributes metadata = - new EventConsumptionAttributes(IMPORT_CANDIDATES_INDEX, identifierProvider); - return new IndexDocument(metadata, objectNode); + public static final String SOMETHING_BAD_HAPPENED = "Something bad happened"; + private static final Context CONTEXT = Mockito.mock(Context.class); + private FakeIndexingClient indexingClient; + + private ByteArrayOutputStream output; + + private DeleteImportCandidateFromIndexHandler handler; + + private static IndexDocument createSampleResource(SortableIdentifier identifierProvider) { + String randomJson = randomJson(); + ObjectNode objectNode = + attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)).orElseThrow(); + EventConsumptionAttributes metadata = + new EventConsumptionAttributes(IMPORT_CANDIDATES_INDEX, identifierProvider); + return new IndexDocument(metadata, objectNode); + } + + @BeforeEach + void init() { + indexingClient = new FakeIndexingClient(); + handler = new DeleteImportCandidateFromIndexHandler(indexingClient); + output = new ByteArrayOutputStream(); + } + + @Test + void shouldThrowRuntimeExceptionWhenIndexingClientIsThrowingException() throws IOException { + indexingClient = + new DeleteImportCandidateFromIndexHandlerTest.FakeIndexingClientThrowingException(); + handler = new DeleteImportCandidateFromIndexHandler(indexingClient); + try (var eventReference = createEventBridgeEvent(SortableIdentifier.next())) { + assertThrows( + RuntimeException.class, () -> handler.handleRequest(eventReference, output, CONTEXT)); } - - @BeforeEach - void init() { - indexingClient = new FakeIndexingClient(); - handler = new DeleteImportCandidateFromIndexHandler(indexingClient); - output = new ByteArrayOutputStream(); - } - - @Test - void shouldThrowRuntimeExceptionWhenIndexingClientIsThrowingException() throws IOException { - indexingClient = - new DeleteImportCandidateFromIndexHandlerTest.FakeIndexingClientThrowingException(); - handler = new DeleteImportCandidateFromIndexHandler(indexingClient); - try (var eventReference = createEventBridgeEvent(SortableIdentifier.next())) { - assertThrows( - RuntimeException.class, - () -> handler.handleRequest(eventReference, output, CONTEXT)); - } + } + + @Test + void shouldRemoveDocumentFromSearchIndexClient() throws IOException { + var resourceIdentifier = SortableIdentifier.next(); + var sampleDocument = createSampleResource(resourceIdentifier); + indexingClient.addDocumentToIndex(sampleDocument); + try (var eventReference = createEventBridgeEvent(resourceIdentifier)) { + handler.handleRequest(eventReference, output, CONTEXT); } + Set allIndexedDocuments = indexingClient.listAllDocuments(IMPORT_CANDIDATES_INDEX); + assertThat(allIndexedDocuments, not(contains(sampleDocument.resource()))); + } - @Test - void shouldRemoveDocumentFromSearchIndexClient() throws IOException { - var resourceIdentifier = SortableIdentifier.next(); - var sampleDocument = createSampleResource(resourceIdentifier); - indexingClient.addDocumentToIndex(sampleDocument); - try (var eventReference = createEventBridgeEvent(resourceIdentifier)) { - handler.handleRequest(eventReference, output, CONTEXT); - } - Set allIndexedDocuments = - indexingClient.listAllDocuments(IMPORT_CANDIDATES_INDEX); - assertThat(allIndexedDocuments, not(contains(sampleDocument.resource()))); - } - - private InputStream createEventBridgeEvent(SortableIdentifier resourceIdentifier) - throws IOException { - DeleteResourceEvent deleteResourceEvent = - new DeleteResourceEvent(DeleteImportCandidateEvent.EVENT_TOPIC, resourceIdentifier); + private InputStream createEventBridgeEvent(SortableIdentifier resourceIdentifier) + throws IOException { + DeleteResourceEvent deleteResourceEvent = + new DeleteResourceEvent(DeleteImportCandidateEvent.EVENT_TOPIC, resourceIdentifier); - AwsEventBridgeDetail detail = new AwsEventBridgeDetail<>(); - detail.setResponsePayload(deleteResourceEvent); + AwsEventBridgeDetail detail = new AwsEventBridgeDetail<>(); + detail.setResponsePayload(deleteResourceEvent); - AwsEventBridgeEvent> event = - new AwsEventBridgeEvent<>(); - event.setDetail(detail); + AwsEventBridgeEvent> event = + new AwsEventBridgeEvent<>(); + event.setDetail(detail); - return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); - } + return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); + } - static class FakeIndexingClientThrowingException extends FakeIndexingClient { + static class FakeIndexingClientThrowingException extends FakeIndexingClient { - @Override - public void removeDocumentFromImportCandidateIndex(String identifier) throws IOException { - throw new IOException(SOMETHING_BAD_HAPPENED); - } + @Override + public void removeDocumentFromImportCandidateIndex(String identifier) throws IOException { + throw new IOException(SOMETHING_BAD_HAPPENED); } + } } diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandlerTest.java index 3b5192df6..7bef69203 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteImportCandidateIndexHandlerTest.java @@ -4,20 +4,17 @@ import static no.unit.nva.LogAppender.logToString; import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.core.StringContains.containsString; +import java.util.ArrayList; import no.unit.nva.indexingclient.IndexingClient; import no.unit.nva.stubs.FakeContext; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.ArrayList; - public class DeleteImportCandidateIndexHandlerTest { private static ListAppender appender; diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteIndicesHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteIndicesHandlerTest.java index 1e768db4b..cfb619e23 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteIndicesHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteIndicesHandlerTest.java @@ -8,21 +8,18 @@ import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.constants.Words.TICKETS; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; import static org.hamcrest.core.StringContains.containsString; +import java.util.ArrayList; +import java.util.List; import no.unit.nva.indexingclient.IndexingClient; import no.unit.nva.stubs.FakeContext; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import java.util.ArrayList; -import java.util.List; - class DeleteIndicesHandlerTest { private static final List ALL_INDICES = diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandlerTest.java index f687f29a0..8db4bde8c 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/DeleteResourceFromIndexHandlerTest.java @@ -4,9 +4,7 @@ import static no.unit.nva.LogAppender.logToString; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.not; @@ -16,7 +14,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Set; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.identifiers.SortableIdentifier; @@ -24,19 +26,12 @@ import no.unit.nva.indexing.testutils.FakeIndexingClient; import no.unit.nva.indexingclient.models.EventConsumptionAttributes; import no.unit.nva.indexingclient.models.IndexDocument; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Set; - public class DeleteResourceFromIndexHandlerTest { public static final String RESOURCES_INDEX = "resource"; diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandlerTest.java index 6e1c20a64..82c9577de 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/ImportCandidateInitHandlerTest.java @@ -5,7 +5,6 @@ import static no.unit.nva.indexing.handlers.InitHandler.FAILED; import static no.unit.nva.indexing.handlers.InitHandler.SUCCESS; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.StringContains.containsString; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -15,51 +14,48 @@ import static org.mockito.Mockito.when; import com.amazonaws.services.lambda.runtime.Context; - +import java.io.IOException; import no.unit.nva.indexingclient.IndexingClient; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.io.IOException; - public class ImportCandidateInitHandlerTest { - private static ListAppender appender; - private ImportCandidateInitHandler initHandler; - private IndexingClient indexingClient; - private Context context; - - @BeforeAll - public static void initClass() { - appender = getAppender(InitHandler.class); - } - - @BeforeEach - void init() { - indexingClient = mock(IndexingClient.class); - initHandler = new ImportCandidateInitHandler(indexingClient); - context = mock(Context.class); - } - - @Test - void shouldNotThrowExceptionIfIndicesClientDoesNotThrowException() throws IOException { - doNothing().when(indexingClient).createIndex(any(String.class)); - var response = initHandler.handleRequest(null, context); + private static ListAppender appender; + private ImportCandidateInitHandler initHandler; + private IndexingClient indexingClient; + private Context context; + + @BeforeAll + public static void initClass() { + appender = getAppender(InitHandler.class); + } + + @BeforeEach + void init() { + indexingClient = mock(IndexingClient.class); + initHandler = new ImportCandidateInitHandler(indexingClient); + context = mock(Context.class); + } + + @Test + void shouldNotThrowExceptionIfIndicesClientDoesNotThrowException() throws IOException { + doNothing().when(indexingClient).createIndex(any(String.class)); + var response = initHandler.handleRequest(null, context); assertEquals(SUCCESS, response); - } - - @Test - void shouldLogWarningAndReturnFailedWhenIndexingClientFailedToCreateIndex() throws IOException { - String expectedMessage = randomString(); - when(indexingClient.createIndex(Mockito.anyString(), Mockito.anyMap())) - .thenThrow(new IOException(expectedMessage)); - var response = initHandler.handleRequest(null, context); - assertEquals(FAILED, response); - - assertThat(logToString(appender), containsString(expectedMessage)); - } + } + + @Test + void shouldLogWarningAndReturnFailedWhenIndexingClientFailedToCreateIndex() throws IOException { + String expectedMessage = randomString(); + when(indexingClient.createIndex(Mockito.anyString(), Mockito.anyMap())) + .thenThrow(new IOException(expectedMessage)); + var response = initHandler.handleRequest(null, context); + assertEquals(FAILED, response); + + assertThat(logToString(appender), containsString(expectedMessage)); + } } diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandlerTest.java index 20b381c78..3052aed6f 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexImportCandidateHandlerTest.java @@ -6,9 +6,7 @@ import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.stringContainsInOrder; import static org.hamcrest.collection.IsIterableContainingInOrder.contains; @@ -17,7 +15,11 @@ import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.events.models.EventReference; @@ -28,143 +30,126 @@ import no.unit.nva.s3.S3Driver; import no.unit.nva.stubs.FakeS3Client; import no.unit.nva.testutils.RandomDataGenerator; - import nva.commons.core.paths.UriWrapper; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; - import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; - public class IndexImportCandidateHandlerTest { - public static final IndexDocument SAMPLE_RESOURCE = - createSampleResource(SortableIdentifier.next(), IMPORT_CANDIDATES_INDEX); - public static final String FILE_DOES_NOT_EXIST = "File does not exist"; - public static final String IGNORED_TOPIC = "ignoredValue"; - private static final IndexDocument SAMPLE_IMPORT_CANDIDATE_MISSING_IDENTIFIER = - createSampleResource(null, IMPORT_CANDIDATES_INDEX); - private static final IndexDocument IMPORT_CANDIDATE_MISSING_INDEX_NAME = - createSampleResource(SortableIdentifier.next(), null); - private S3Driver s3Driver; - private IndexImportCandidateHandler handler; - private Context context; - private ByteArrayOutputStream output; - private FakeIndexingClient indexingClient; - - private static IndexDocument createSampleResource( - SortableIdentifier identifierProvider, String indexName) { - var randomJson = randomJson(); - var objectNode = - attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)) - .orElseThrow(); - var metadata = new EventConsumptionAttributes(indexName, identifierProvider); - return new IndexDocument(metadata, objectNode); - } - - @BeforeEach - void init() { - FakeS3Client fakeS3Client = new FakeS3Client(); - s3Driver = new S3Driver(fakeS3Client, "ignored"); - indexingClient = new FakeIndexingClient(); - handler = new IndexImportCandidateHandler(s3Driver, indexingClient); - - context = Mockito.mock(Context.class); - output = new ByteArrayOutputStream(); - } - - @Test - void shouldAddDocumentToIndexWhenResourceExistsInResourcesStorage() throws Exception { - var resourceLocation = prepareEventStorageResourceFile(); - var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); - handler.handleRequest(input, output, context); - var allIndexedDocuments = indexingClient.listAllDocuments(SAMPLE_RESOURCE.getIndexName()); - - assertThat(allIndexedDocuments, contains(SAMPLE_RESOURCE.resource())); - } - - @Test - void shouldThrowExceptionOnCommunicationProblemWithService() throws Exception { - final var expectedErrorMessage = randomString(); - indexingClient = indexingClientThrowingException(expectedErrorMessage); - handler = new IndexImportCandidateHandler(s3Driver, indexingClient); - var resourceLocation = prepareEventStorageResourceFile(); - var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); + public static final IndexDocument SAMPLE_RESOURCE = + createSampleResource(SortableIdentifier.next(), IMPORT_CANDIDATES_INDEX); + public static final String FILE_DOES_NOT_EXIST = "File does not exist"; + public static final String IGNORED_TOPIC = "ignoredValue"; + private static final IndexDocument SAMPLE_IMPORT_CANDIDATE_MISSING_IDENTIFIER = + createSampleResource(null, IMPORT_CANDIDATES_INDEX); + private static final IndexDocument IMPORT_CANDIDATE_MISSING_INDEX_NAME = + createSampleResource(SortableIdentifier.next(), null); + private S3Driver s3Driver; + private IndexImportCandidateHandler handler; + private Context context; + private ByteArrayOutputStream output; + private FakeIndexingClient indexingClient; + + private static IndexDocument createSampleResource( + SortableIdentifier identifierProvider, String indexName) { + var randomJson = randomJson(); + var objectNode = + attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)).orElseThrow(); + var metadata = new EventConsumptionAttributes(indexName, identifierProvider); + return new IndexDocument(metadata, objectNode); + } + + @BeforeEach + void init() { + FakeS3Client fakeS3Client = new FakeS3Client(); + s3Driver = new S3Driver(fakeS3Client, "ignored"); + indexingClient = new FakeIndexingClient(); + handler = new IndexImportCandidateHandler(s3Driver, indexingClient); + + context = Mockito.mock(Context.class); + output = new ByteArrayOutputStream(); + } + + @Test + void shouldAddDocumentToIndexWhenResourceExistsInResourcesStorage() throws Exception { + var resourceLocation = prepareEventStorageResourceFile(); + var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); + handler.handleRequest(input, output, context); + var allIndexedDocuments = indexingClient.listAllDocuments(SAMPLE_RESOURCE.getIndexName()); + + assertThat(allIndexedDocuments, contains(SAMPLE_RESOURCE.resource())); + } + + @Test + void shouldThrowExceptionOnCommunicationProblemWithService() throws Exception { + final var expectedErrorMessage = randomString(); + indexingClient = indexingClientThrowingException(expectedErrorMessage); + handler = new IndexImportCandidateHandler(s3Driver, indexingClient); + var resourceLocation = prepareEventStorageResourceFile(); + var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); + + assertThrows(RuntimeException.class, () -> handler.handleRequest(input, output, context)); + } + + @Test + void shouldThrowExceptionWhenResourceIsMissingIdentifier() throws Exception { + var resourceLocation = + prepareEventStorageResourceFile(SAMPLE_IMPORT_CANDIDATE_MISSING_IDENTIFIER); + var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); + var exception = + assertThrows(RuntimeException.class, () -> handler.handleRequest(input, output, context)); + + assertThat(exception.getMessage(), stringContainsInOrder(MISSING_IDENTIFIER_IN_RESOURCE)); + } + + @Test + void shouldThrowNoSuchKeyExceptionWhenResourceIsMissingFromEventStorage() { + var missingResourceLocation = RandomDataGenerator.randomUri(); + var input = attempt(() -> createEventBridgeEvent(missingResourceLocation)).get(); + var exception = + assertThrows(NoSuchKeyException.class, () -> handler.handleRequest(input, output, context)); + assertThat(exception.getMessage(), stringContainsInOrder(FILE_DOES_NOT_EXIST)); + } + + @Test + void shouldThrowExceptionWhenEventConsumptionAttributesIsMissingIndexName() throws Exception { + var resourceLocation = prepareEventStorageResourceFile(IMPORT_CANDIDATE_MISSING_INDEX_NAME); + var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); + var exception = assertThrows(RuntimeException.class, () -> handler.handleRequest(input, output, context)); - } - - @Test - void shouldThrowExceptionWhenResourceIsMissingIdentifier() throws Exception { - var resourceLocation = - prepareEventStorageResourceFile(SAMPLE_IMPORT_CANDIDATE_MISSING_IDENTIFIER); - var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); - var exception = - assertThrows( - RuntimeException.class, - () -> handler.handleRequest(input, output, context)); - - assertThat(exception.getMessage(), stringContainsInOrder(MISSING_IDENTIFIER_IN_RESOURCE)); - } - - @Test - void shouldThrowNoSuchKeyExceptionWhenResourceIsMissingFromEventStorage() { - var missingResourceLocation = RandomDataGenerator.randomUri(); - var input = attempt(() -> createEventBridgeEvent(missingResourceLocation)).get(); - var exception = - assertThrows( - NoSuchKeyException.class, - () -> handler.handleRequest(input, output, context)); - - assertThat(exception.getMessage(), stringContainsInOrder(FILE_DOES_NOT_EXIST)); - } - - @Test - void shouldThrowExceptionWhenEventConsumptionAttributesIsMissingIndexName() throws Exception { - var resourceLocation = prepareEventStorageResourceFile(IMPORT_CANDIDATE_MISSING_INDEX_NAME); - var input = attempt(() -> createEventBridgeEvent(resourceLocation)).get(); - var exception = - assertThrows( - RuntimeException.class, - () -> handler.handleRequest(input, output, context)); - - assertThat(exception.getMessage(), stringContainsInOrder(MISSING_INDEX_NAME_IN_RESOURCE)); - } - - private FakeIndexingClient indexingClientThrowingException(String expectedErrorMessage) { - return new FakeIndexingClient() { - @Override - public Void addDocumentToIndex(IndexDocument indexDocument) throws IOException { - throw new IOException(expectedErrorMessage); - } - }; - } - - private URI prepareEventStorageResourceFile() throws IOException { - return prepareEventStorageResourceFile(SAMPLE_RESOURCE); - } - - private URI prepareEventStorageResourceFile(IndexDocument resource) throws IOException { - var resourceLocation = RandomDataGenerator.randomUri(); - var resourcePath = UriWrapper.fromUri(resourceLocation).toS3bucketPath(); - s3Driver.insertFile(resourcePath, resource.toJsonString()); - return resourceLocation; - } - - private InputStream createEventBridgeEvent(URI resourceLocation) - throws JsonProcessingException { - var indexResourceEvent = new EventReference(IGNORED_TOPIC, resourceLocation); - var detail = new AwsEventBridgeDetail<>(); - detail.setResponsePayload(indexResourceEvent); - var event = new AwsEventBridgeEvent<>(); - event.setDetail(detail); - return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); - } + + assertThat(exception.getMessage(), stringContainsInOrder(MISSING_INDEX_NAME_IN_RESOURCE)); + } + + private FakeIndexingClient indexingClientThrowingException(String expectedErrorMessage) { + return new FakeIndexingClient() { + @Override + public Void addDocumentToIndex(IndexDocument indexDocument) throws IOException { + throw new IOException(expectedErrorMessage); + } + }; + } + + private URI prepareEventStorageResourceFile() throws IOException { + return prepareEventStorageResourceFile(SAMPLE_RESOURCE); + } + + private URI prepareEventStorageResourceFile(IndexDocument resource) throws IOException { + var resourceLocation = RandomDataGenerator.randomUri(); + var resourcePath = UriWrapper.fromUri(resourceLocation).toS3bucketPath(); + s3Driver.insertFile(resourcePath, resource.toJsonString()); + return resourceLocation; + } + + private InputStream createEventBridgeEvent(URI resourceLocation) throws JsonProcessingException { + var indexResourceEvent = new EventReference(IGNORED_TOPIC, resourceLocation); + var detail = new AwsEventBridgeDetail<>(); + detail.setResponsePayload(indexResourceEvent); + var event = new AwsEventBridgeEvent<>(); + event.setDetail(detail); + return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); + } } diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexResourceHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexResourceHandlerTest.java index 6653efd66..18ff90406 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexResourceHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/IndexResourceHandlerTest.java @@ -7,9 +7,7 @@ import static no.unit.nva.constants.Words.TICKETS; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -22,7 +20,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.Set; import no.unit.nva.events.models.AwsEventBridgeDetail; import no.unit.nva.events.models.AwsEventBridgeEvent; import no.unit.nva.events.models.EventReference; @@ -34,195 +37,175 @@ import no.unit.nva.s3.S3Driver; import no.unit.nva.stubs.FakeS3Client; import no.unit.nva.testutils.RandomDataGenerator; - import nva.commons.core.paths.UnixPath; import nva.commons.core.paths.UriWrapper; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; - import software.amazon.awssdk.services.s3.model.NoSuchKeyException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.util.Set; - public class IndexResourceHandlerTest { - public static final IndexDocument SAMPLE_RESOURCE = - createSampleResource(SortableIdentifier.next(), RESOURCES); - public static final IndexDocument SAMPLE_TICKET = - createSampleResource(SortableIdentifier.next(), TICKETS); - public static final String FILE_DOES_NOT_EXIST = "File does not exist"; - public static final String IGNORED_TOPIC = "ignoredValue"; - - private static final IndexDocument SAMPLE_RESOURCE_MISSING_IDENTIFIER = - createSampleResource(null, RESOURCES); - private static final IndexDocument SAMPLE_RESOURCE_MISSING_INDEX_NAME = - createSampleResource(SortableIdentifier.next(), null); - private S3Driver resourcesS3Driver; - private IndexResourceHandler indexResourceHandler; - private Context context; - private ByteArrayOutputStream output; - private FakeIndexingClient indexingClient; - private FakeSqsClient sqsClient; - - private static IndexDocument createSampleResource( - SortableIdentifier identifierProvider, String indexName) { - String randomJson = randomJson(); - ObjectNode objectNode = - attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)) - .orElseThrow(); - EventConsumptionAttributes metadata = - new EventConsumptionAttributes(indexName, identifierProvider); - return new IndexDocument(metadata, objectNode); - } - - @BeforeEach - void init() { - FakeS3Client fakeS3Client = new FakeS3Client(); - resourcesS3Driver = new S3Driver(fakeS3Client, "resources"); - indexingClient = new FakeIndexingClient(); - sqsClient = new FakeSqsClient(); - indexResourceHandler = - new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); - context = Mockito.mock(Context.class); - output = new ByteArrayOutputStream(); - } - - @Test - void shouldAddDocumentToIndexWhenResourceExistsInResourcesStorage() throws Exception { - URI resourceLocation = prepareEventStorageResourceFile(); - InputStream input = createEventBridgeEvent(resourceLocation); - indexResourceHandler.handleRequest(input, output, context); - Set allIndexedDocuments = - indexingClient.listAllDocuments(SAMPLE_RESOURCE.getIndexName()); - assertThat(allIndexedDocuments, contains(SAMPLE_RESOURCE.resource())); - } - - @Test - void shouldSendMessageToRecoveryQueueWhenIndexingResourceIsFailing() throws Exception { - indexingClient = indexingClientThrowingException(randomString()); - indexResourceHandler = - new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); - var resourceLocation = prepareEventStorageResourceFile(); - var input = createEventBridgeEvent(resourceLocation); - indexResourceHandler.handleRequest(input, output, context); + public static final IndexDocument SAMPLE_RESOURCE = + createSampleResource(SortableIdentifier.next(), RESOURCES); + public static final IndexDocument SAMPLE_TICKET = + createSampleResource(SortableIdentifier.next(), TICKETS); + public static final String FILE_DOES_NOT_EXIST = "File does not exist"; + public static final String IGNORED_TOPIC = "ignoredValue"; + + private static final IndexDocument SAMPLE_RESOURCE_MISSING_IDENTIFIER = + createSampleResource(null, RESOURCES); + private static final IndexDocument SAMPLE_RESOURCE_MISSING_INDEX_NAME = + createSampleResource(SortableIdentifier.next(), null); + private S3Driver resourcesS3Driver; + private IndexResourceHandler indexResourceHandler; + private Context context; + private ByteArrayOutputStream output; + private FakeIndexingClient indexingClient; + private FakeSqsClient sqsClient; + + private static IndexDocument createSampleResource( + SortableIdentifier identifierProvider, String indexName) { + String randomJson = randomJson(); + ObjectNode objectNode = + attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(randomJson)).orElseThrow(); + EventConsumptionAttributes metadata = + new EventConsumptionAttributes(indexName, identifierProvider); + return new IndexDocument(metadata, objectNode); + } + + @BeforeEach + void init() { + FakeS3Client fakeS3Client = new FakeS3Client(); + resourcesS3Driver = new S3Driver(fakeS3Client, "resources"); + indexingClient = new FakeIndexingClient(); + sqsClient = new FakeSqsClient(); + indexResourceHandler = new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); + context = Mockito.mock(Context.class); + output = new ByteArrayOutputStream(); + } + + @Test + void shouldAddDocumentToIndexWhenResourceExistsInResourcesStorage() throws Exception { + URI resourceLocation = prepareEventStorageResourceFile(); + InputStream input = createEventBridgeEvent(resourceLocation); + indexResourceHandler.handleRequest(input, output, context); + Set allIndexedDocuments = + indexingClient.listAllDocuments(SAMPLE_RESOURCE.getIndexName()); + assertThat(allIndexedDocuments, contains(SAMPLE_RESOURCE.resource())); + } + + @Test + void shouldSendMessageToRecoveryQueueWhenIndexingResourceIsFailing() throws Exception { + indexingClient = indexingClientThrowingException(randomString()); + indexResourceHandler = new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); + var resourceLocation = prepareEventStorageResourceFile(); + var input = createEventBridgeEvent(resourceLocation); + indexResourceHandler.handleRequest(input, output, context); var deliveredMessage = sqsClient.getDeliveredMessages().getFirst(); - assertThat( - deliveredMessage.messageAttributes().get("id").stringValue(), is(notNullValue())); - assertThat( - deliveredMessage.messageAttributes().get("type").stringValue(), - is(equalTo("Resource"))); - } + assertThat(deliveredMessage.messageAttributes().get("id").stringValue(), is(notNullValue())); + assertThat( + deliveredMessage.messageAttributes().get("type").stringValue(), is(equalTo("Resource"))); + } - @Test - void shouldSendMessageToRecoveryQueueWhenIndexingTicketIsFailing() throws Exception { - indexingClient = indexingClientThrowingException(randomString()); - indexResourceHandler = - new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); - var resourceLocation = prepareEventStorageTicketFile(); - var input = createEventBridgeEvent(resourceLocation); - indexResourceHandler.handleRequest(input, output, context); + @Test + void shouldSendMessageToRecoveryQueueWhenIndexingTicketIsFailing() throws Exception { + indexingClient = indexingClientThrowingException(randomString()); + indexResourceHandler = new IndexResourceHandler(resourcesS3Driver, indexingClient, sqsClient); + var resourceLocation = prepareEventStorageTicketFile(); + var input = createEventBridgeEvent(resourceLocation); + indexResourceHandler.handleRequest(input, output, context); var deliveredMessage = sqsClient.getDeliveredMessages().getFirst(); - assertThat( - deliveredMessage.messageAttributes().get("id").stringValue(), is(notNullValue())); - assertThat( - deliveredMessage.messageAttributes().get("type").stringValue(), - is(equalTo("Ticket"))); - } - - @Test - void shouldThrowExceptionWhenResourceIsMissingIdentifier() throws Exception { - URI resourceLocation = prepareEventStorageResourceFile(SAMPLE_RESOURCE_MISSING_IDENTIFIER); + assertThat(deliveredMessage.messageAttributes().get("id").stringValue(), is(notNullValue())); + assertThat( + deliveredMessage.messageAttributes().get("type").stringValue(), is(equalTo("Ticket"))); + } - RuntimeException exception; - try (InputStream input = createEventBridgeEvent(resourceLocation)) { + @Test + void shouldThrowExceptionWhenResourceIsMissingIdentifier() throws Exception { + URI resourceLocation = prepareEventStorageResourceFile(SAMPLE_RESOURCE_MISSING_IDENTIFIER); - exception = - assertThrows( - RuntimeException.class, - () -> indexResourceHandler.handleRequest(input, output, context)); - } + RuntimeException exception; + try (InputStream input = createEventBridgeEvent(resourceLocation)) { - assertThat(exception.getMessage(), stringContainsInOrder(MISSING_IDENTIFIER_IN_RESOURCE)); + exception = + assertThrows( + RuntimeException.class, + () -> indexResourceHandler.handleRequest(input, output, context)); } - @Test - void shouldThrowNoSuchKeyExceptionWhenResourceIsMissingFromEventStorage() throws Exception { - URI missingResourceLocation = RandomDataGenerator.randomUri(); + assertThat(exception.getMessage(), stringContainsInOrder(MISSING_IDENTIFIER_IN_RESOURCE)); + } - NoSuchKeyException exception; - try (InputStream input = createEventBridgeEvent(missingResourceLocation)) { + @Test + void shouldThrowNoSuchKeyExceptionWhenResourceIsMissingFromEventStorage() throws Exception { + URI missingResourceLocation = RandomDataGenerator.randomUri(); - exception = - assertThrows( - NoSuchKeyException.class, - () -> indexResourceHandler.handleRequest(input, output, context)); - } + NoSuchKeyException exception; + try (InputStream input = createEventBridgeEvent(missingResourceLocation)) { - assertThat(exception.getMessage(), stringContainsInOrder(FILE_DOES_NOT_EXIST)); + exception = + assertThrows( + NoSuchKeyException.class, + () -> indexResourceHandler.handleRequest(input, output, context)); } - @Test - void shouldThrowExceptionWhenEventConsumptionAttributesIsMissingIndexName() throws Exception { - URI resourceLocation = prepareEventStorageResourceFile(SAMPLE_RESOURCE_MISSING_INDEX_NAME); + assertThat(exception.getMessage(), stringContainsInOrder(FILE_DOES_NOT_EXIST)); + } - RuntimeException exception; - try (InputStream input = createEventBridgeEvent(resourceLocation)) { + @Test + void shouldThrowExceptionWhenEventConsumptionAttributesIsMissingIndexName() throws Exception { + URI resourceLocation = prepareEventStorageResourceFile(SAMPLE_RESOURCE_MISSING_INDEX_NAME); - exception = - assertThrows( - RuntimeException.class, - () -> indexResourceHandler.handleRequest(input, output, context)); - } + RuntimeException exception; + try (InputStream input = createEventBridgeEvent(resourceLocation)) { - assertThat(exception.getMessage(), stringContainsInOrder(MISSING_INDEX_NAME_IN_RESOURCE)); + exception = + assertThrows( + RuntimeException.class, + () -> indexResourceHandler.handleRequest(input, output, context)); } - private FakeIndexingClient indexingClientThrowingException(String expectedErrorMessage) { - return new FakeIndexingClient() { - @Override - public Void addDocumentToIndex(IndexDocument indexDocument) throws IOException { - throw new IOException(expectedErrorMessage); - } - }; - } + assertThat(exception.getMessage(), stringContainsInOrder(MISSING_INDEX_NAME_IN_RESOURCE)); + } - private URI prepareEventStorageTicketFile() throws IOException { - return prepareEventStorageResourceFile(SAMPLE_TICKET); - } + private FakeIndexingClient indexingClientThrowingException(String expectedErrorMessage) { + return new FakeIndexingClient() { + @Override + public Void addDocumentToIndex(IndexDocument indexDocument) throws IOException { + throw new IOException(expectedErrorMessage); + } + }; + } - private URI prepareEventStorageResourceFile() throws IOException { - return prepareEventStorageResourceFile(SAMPLE_RESOURCE); - } + private URI prepareEventStorageTicketFile() throws IOException { + return prepareEventStorageResourceFile(SAMPLE_TICKET); + } - private URI prepareEventStorageResourceFile(IndexDocument resource) throws IOException { - URI resourceLocation = RandomDataGenerator.randomUri(); - UnixPath resourcePath = UriWrapper.fromUri(resourceLocation).toS3bucketPath(); - resourcesS3Driver.insertFile(resourcePath, resource.toJsonString()); - return resourceLocation; - } + private URI prepareEventStorageResourceFile() throws IOException { + return prepareEventStorageResourceFile(SAMPLE_RESOURCE); + } - private InputStream createEventBridgeEvent(URI resourceLocation) - throws JsonProcessingException { - EventReference indexResourceEvent = new EventReference(IGNORED_TOPIC, resourceLocation); + private URI prepareEventStorageResourceFile(IndexDocument resource) throws IOException { + URI resourceLocation = RandomDataGenerator.randomUri(); + UnixPath resourcePath = UriWrapper.fromUri(resourceLocation).toS3bucketPath(); + resourcesS3Driver.insertFile(resourcePath, resource.toJsonString()); + return resourceLocation; + } - AwsEventBridgeDetail detail = new AwsEventBridgeDetail<>(); - detail.setResponsePayload(indexResourceEvent); + private InputStream createEventBridgeEvent(URI resourceLocation) throws JsonProcessingException { + EventReference indexResourceEvent = new EventReference(IGNORED_TOPIC, resourceLocation); - AwsEventBridgeEvent> event = - new AwsEventBridgeEvent<>(); - event.setDetail(detail); + AwsEventBridgeDetail detail = new AwsEventBridgeDetail<>(); + detail.setResponsePayload(indexResourceEvent); - return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); - } + AwsEventBridgeEvent> event = new AwsEventBridgeEvent<>(); + event.setDetail(detail); + + return new ByteArrayInputStream(objectMapperWithEmpty.writeValueAsBytes(event)); + } } diff --git a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/InitHandlerTest.java b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/InitHandlerTest.java index 2bc0ee273..11169f0c9 100644 --- a/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/InitHandlerTest.java +++ b/indexing-handlers/src/test/java/no/unit/nva/indexing/handlers/InitHandlerTest.java @@ -5,7 +5,6 @@ import static no.unit.nva.indexing.handlers.InitHandler.FAILED; import static no.unit.nva.indexing.handlers.InitHandler.SUCCESS; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.StringContains.containsString; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -15,17 +14,14 @@ import static org.mockito.Mockito.when; import com.amazonaws.services.lambda.runtime.Context; - +import java.io.IOException; import no.unit.nva.indexingclient.IndexingClient; - import org.apache.logging.log4j.core.test.appender.ListAppender; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; -import java.io.IOException; - class InitHandlerTest { private static ListAppender appender; diff --git a/search-commons/src/main/java/no/unit/nva/constants/Defaults.java b/search-commons/src/main/java/no/unit/nva/constants/Defaults.java index d758f4c90..180e96ce8 100644 --- a/search-commons/src/main/java/no/unit/nva/constants/Defaults.java +++ b/search-commons/src/main/java/no/unit/nva/constants/Defaults.java @@ -2,16 +2,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.net.MediaType; - +import java.net.URI; +import java.util.List; import no.unit.nva.commons.json.JsonUtils; - import nva.commons.apigateway.MediaTypes; import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; -import java.net.URI; -import java.util.List; - /** * Default values for the search service. * diff --git a/search-commons/src/main/java/no/unit/nva/constants/ErrorMessages.java b/search-commons/src/main/java/no/unit/nva/constants/ErrorMessages.java index 03a0f21df..ff3c7da3e 100644 --- a/search-commons/src/main/java/no/unit/nva/constants/ErrorMessages.java +++ b/search-commons/src/main/java/no/unit/nva/constants/ErrorMessages.java @@ -3,14 +3,12 @@ import static no.unit.nva.constants.Words.PREFIX; import static no.unit.nva.constants.Words.QUOTE; import static no.unit.nva.constants.Words.SUFFIX; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import nva.commons.core.JacocoGenerated; - import java.util.Collection; import java.util.Set; import java.util.stream.Collectors; +import nva.commons.core.JacocoGenerated; /** * Error messages used in the search service. @@ -19,55 +17,54 @@ */ public final class ErrorMessages { - public static final String HANDLER_NOT_DEFINED = "handler NOT defined -> "; - public static final String INVALID_VALUE = "Parameter [%s] has invalid value [%s]"; - public static final String INVALID_VALUE_WITH_SORT = - "Sort has invalid field value [%s]. Valid values are: %s"; - public static final String INVALID_NUMBER = - "Parameter '%s' has invalid value. Must be a number."; - public static final String INVALID_DATE = "Parameter '%s' has invalid value. Must be a date."; - public static final String INVALID_BOOLEAN = - "Parameter '%s' has invalid value. Must be a boolean."; - public static final String MISSING_PARAMETER = "Parameter(s) -> [%s] -> is/are required."; - public static final String NOT_IMPLEMENTED_FOR = "Not implemented for "; + public static final String HANDLER_NOT_DEFINED = "handler NOT defined -> "; + public static final String INVALID_VALUE = "Parameter [%s] has invalid value [%s]"; + public static final String INVALID_VALUE_WITH_SORT = + "Sort has invalid field value [%s]. Valid values are: %s"; + public static final String INVALID_NUMBER = "Parameter '%s' has invalid value. Must be a number."; + public static final String INVALID_DATE = "Parameter '%s' has invalid value. Must be a date."; + public static final String INVALID_BOOLEAN = + "Parameter '%s' has invalid value. Must be a boolean."; + public static final String MISSING_PARAMETER = "Parameter(s) -> [%s] -> is/are required."; + public static final String NOT_IMPLEMENTED_FOR = "Not implemented for "; - public static final String OPERATOR_NOT_SUPPORTED = "Operator not supported"; - public static final String RELEVANCE_SEARCH_AFTER_ARE_MUTUAL_EXCLUSIVE = - "Sorted by relevance & searchAfter are mutual exclusive."; - public static final String TOO_MANY_ARGUMENTS = "too many arguments: "; + public static final String OPERATOR_NOT_SUPPORTED = "Operator not supported"; + public static final String RELEVANCE_SEARCH_AFTER_ARE_MUTUAL_EXCLUSIVE = + "Sorted by relevance & searchAfter are mutual exclusive."; + public static final String TOO_MANY_ARGUMENTS = "too many arguments: "; - public static final String TEMPLATE_INVALID_QUERY_PARAMETERS = - "Invalid query parameter supplied %s. Valid parameters: %s Also pass through to" - + " OpenSearch:[page & per_page | offset & results, sort (& sortOrder), fields, " - + "search_after]"; + public static final String TEMPLATE_INVALID_QUERY_PARAMETERS = + "Invalid query parameter supplied %s. Valid parameters: %s Also pass through to" + + " OpenSearch:[page & per_page | offset & results, sort (& sortOrder), fields, " + + "search_after]"; - public static final String MISSING_IDENTIFIER_IN_RESOURCE = "Missing identifier in resource"; - public static final String MISSING_INDEX_NAME_IN_RESOURCE = "Missing index name in resource"; + public static final String MISSING_IDENTIFIER_IN_RESOURCE = "Missing identifier in resource"; + public static final String MISSING_INDEX_NAME_IN_RESOURCE = "Missing index name in resource"; - @JacocoGenerated - public ErrorMessages() {} + @JacocoGenerated + public ErrorMessages() {} - /** - * Formats and emits a message with valid parameter names. - * - * @param invalidKeys list of invalid parameter names - * @param queryParameters list of valid parameter names - * @return formatted string containing a list of valid parameters - */ - public static String validQueryParameterNamesMessage( - Set invalidKeys, Collection queryParameters) { - return TEMPLATE_INVALID_QUERY_PARAMETERS.formatted(invalidKeys, queryParameters); - } + /** + * Formats and emits a message with valid parameter names. + * + * @param invalidKeys list of invalid parameter names + * @param queryParameters list of valid parameter names + * @return formatted string containing a list of valid parameters + */ + public static String validQueryParameterNamesMessage( + Set invalidKeys, Collection queryParameters) { + return TEMPLATE_INVALID_QUERY_PARAMETERS.formatted(invalidKeys, queryParameters); + } - public static String requiredMissingMessage(Set missingKeys) { - return String.format(MISSING_PARAMETER, prettifyList(missingKeys)); - } + public static String requiredMissingMessage(Set missingKeys) { + return String.format(MISSING_PARAMETER, prettifyList(missingKeys)); + } - private static String prettifyList(Set queryParameters) { - return queryParameters.size() > 1 - ? queryParameters.stream() - .map(parameterName -> QUOTE + parameterName + QUOTE) - .collect(Collectors.joining(", ", PREFIX, SUFFIX)) - : queryParameters.stream().collect(Collectors.joining(EMPTY_STRING, QUOTE, QUOTE)); - } + private static String prettifyList(Set queryParameters) { + return queryParameters.size() > 1 + ? queryParameters.stream() + .map(parameterName -> QUOTE + parameterName + QUOTE) + .collect(Collectors.joining(", ", PREFIX, SUFFIX)) + : queryParameters.stream().collect(Collectors.joining(EMPTY_STRING, QUOTE, QUOTE)); + } } diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/IndexQueueClient.java b/search-commons/src/main/java/no/unit/nva/indexingclient/IndexQueueClient.java index ccc684cce..5176f9721 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/IndexQueueClient.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/IndexQueueClient.java @@ -1,57 +1,54 @@ package no.unit.nva.indexingclient; +import java.time.Duration; import no.unit.nva.indexingclient.models.QueueClient; - import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; - import software.amazon.awssdk.http.SdkHttpClient; import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.SendMessageRequest; -import java.time.Duration; - @JacocoGenerated public final class IndexQueueClient implements QueueClient { - public static final String AWS_REGION = "AWS_REGION"; - private static final int MAX_CONNECTIONS = 10_000; - private static final long IDLE_TIME = 30; - private static final long TIMEOUT_TIME = 30; - private final SqsClient sqsClient; - - private IndexQueueClient(SqsClient sqsClient) { - this.sqsClient = sqsClient; - } - - public static IndexQueueClient defaultQueueClient() { - return new IndexQueueClient(defaultClient()); - } - - private static SqsClient defaultClient() { - return SqsClient.builder() - .region(getRegion()) - .httpClient(httpClientForConcurrentQueries()) - .build(); - } - - private static Region getRegion() { - return new Environment().readEnvOpt(AWS_REGION).map(Region::of).orElse(Region.EU_WEST_1); - } - - private static SdkHttpClient httpClientForConcurrentQueries() { - return ApacheHttpClient.builder() - .useIdleConnectionReaper(true) - .maxConnections(MAX_CONNECTIONS) - .connectionMaxIdleTime(Duration.ofSeconds(IDLE_TIME)) - .connectionTimeout(Duration.ofSeconds(TIMEOUT_TIME)) - .build(); - } - - @Override - public void sendMessage(SendMessageRequest sendMessageRequest) { - sqsClient.sendMessage(sendMessageRequest); - } + public static final String AWS_REGION = "AWS_REGION"; + private static final int MAX_CONNECTIONS = 10_000; + private static final long IDLE_TIME = 30; + private static final long TIMEOUT_TIME = 30; + private final SqsClient sqsClient; + + private IndexQueueClient(SqsClient sqsClient) { + this.sqsClient = sqsClient; + } + + public static IndexQueueClient defaultQueueClient() { + return new IndexQueueClient(defaultClient()); + } + + private static SqsClient defaultClient() { + return SqsClient.builder() + .region(getRegion()) + .httpClient(httpClientForConcurrentQueries()) + .build(); + } + + private static Region getRegion() { + return new Environment().readEnvOpt(AWS_REGION).map(Region::of).orElse(Region.EU_WEST_1); + } + + private static SdkHttpClient httpClientForConcurrentQueries() { + return ApacheHttpClient.builder() + .useIdleConnectionReaper(true) + .maxConnections(MAX_CONNECTIONS) + .connectionMaxIdleTime(Duration.ofSeconds(IDLE_TIME)) + .connectionTimeout(Duration.ofSeconds(TIMEOUT_TIME)) + .build(); + } + + @Override + public void sendMessage(SendMessageRequest sendMessageRequest) { + sqsClient.sendMessage(sendMessageRequest); + } } diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/IndexingClient.java b/search-commons/src/main/java/no/unit/nva/indexingclient/IndexingClient.java index 71003a2c5..c61d7f2e8 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/IndexingClient.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/IndexingClient.java @@ -4,24 +4,27 @@ import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.indexingclient.models.RestHighLevelClientWrapper.defaultRestHighLevelClientWrapper; - import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Iterators; import com.google.common.collect.UnmodifiableIterator; - +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import no.unit.nva.commons.json.JsonUtils; import no.unit.nva.indexingclient.models.AuthenticatedOpenSearchClientWrapper; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.indexingclient.models.RestHighLevelClientWrapper; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.jwt.CognitoAuthenticator; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.Try; import nva.commons.secrets.SecretsReader; - import org.opensearch.action.DocWriteResponse; import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.bulk.BulkRequest; @@ -39,14 +42,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - public class IndexingClient extends AuthenticatedOpenSearchClientWrapper { public static final int BULK_SIZE = 100; diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/models/AuthenticatedOpenSearchClientWrapper.java b/search-commons/src/main/java/no/unit/nva/indexingclient/models/AuthenticatedOpenSearchClientWrapper.java index 3d2c30d12..86ec65b06 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/models/AuthenticatedOpenSearchClientWrapper.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/models/AuthenticatedOpenSearchClientWrapper.java @@ -3,55 +3,52 @@ import static no.unit.nva.constants.Defaults.ENVIRONMENT; import static no.unit.nva.constants.Words.AUTHORIZATION; +import java.net.URI; import no.unit.nva.auth.CognitoCredentials; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.records.UsernamePasswordWrapper; - import nva.commons.secrets.SecretsReader; - import org.opensearch.client.RequestOptions; -import java.net.URI; - public class AuthenticatedOpenSearchClientWrapper { - static final String SEARCH_INFRASTRUCTURE_CREDENTIALS = "SearchInfrastructureCredentials"; - static final String SEARCH_INFRASTRUCTURE_AUTH_URI = - ENVIRONMENT.readEnv("SEARCH_INFRASTRUCTURE_AUTH_URI"); - - protected final RestHighLevelClientWrapper openSearchClient; - protected final CachedJwtProvider cachedJwtProvider; - - /** - * Creates a new OpensearchClient. - * - * @param openSearchClient client to use for access to external search infrastructure - * @param cachedJwtProvider provider for JWT tokens - */ - public AuthenticatedOpenSearchClientWrapper( - RestHighLevelClientWrapper openSearchClient, CachedJwtProvider cachedJwtProvider) { - this.openSearchClient = openSearchClient; - this.cachedJwtProvider = cachedJwtProvider; - } - - /** - * Creates a new OpensearchClient. - * - * @param secretsReader reader for secrets (used to fetch the username and password for the - * search infrastructure) - * @return CognitoCredentials for the search infrastructure - */ - protected static CognitoCredentials createCognitoCredentials(SecretsReader secretsReader) { - var credentials = - secretsReader.fetchClassSecret( - SEARCH_INFRASTRUCTURE_CREDENTIALS, UsernamePasswordWrapper.class); - var uri = URI.create(SEARCH_INFRASTRUCTURE_AUTH_URI); - - return new CognitoCredentials(credentials::getUsername, credentials::getPassword, uri); - } - - protected RequestOptions getRequestOptions() { - var token = "Bearer " + cachedJwtProvider.getValue().getToken(); - return RequestOptions.DEFAULT.toBuilder().addHeader(AUTHORIZATION, token).build(); - } + static final String SEARCH_INFRASTRUCTURE_CREDENTIALS = "SearchInfrastructureCredentials"; + static final String SEARCH_INFRASTRUCTURE_AUTH_URI = + ENVIRONMENT.readEnv("SEARCH_INFRASTRUCTURE_AUTH_URI"); + + protected final RestHighLevelClientWrapper openSearchClient; + protected final CachedJwtProvider cachedJwtProvider; + + /** + * Creates a new OpensearchClient. + * + * @param openSearchClient client to use for access to external search infrastructure + * @param cachedJwtProvider provider for JWT tokens + */ + public AuthenticatedOpenSearchClientWrapper( + RestHighLevelClientWrapper openSearchClient, CachedJwtProvider cachedJwtProvider) { + this.openSearchClient = openSearchClient; + this.cachedJwtProvider = cachedJwtProvider; + } + + /** + * Creates a new OpensearchClient. + * + * @param secretsReader reader for secrets (used to fetch the username and password for the search + * infrastructure) + * @return CognitoCredentials for the search infrastructure + */ + protected static CognitoCredentials createCognitoCredentials(SecretsReader secretsReader) { + var credentials = + secretsReader.fetchClassSecret( + SEARCH_INFRASTRUCTURE_CREDENTIALS, UsernamePasswordWrapper.class); + var uri = URI.create(SEARCH_INFRASTRUCTURE_AUTH_URI); + + return new CognitoCredentials(credentials::getUsername, credentials::getPassword, uri); + } + + protected RequestOptions getRequestOptions() { + var token = "Bearer " + cachedJwtProvider.getValue().getToken(); + return RequestOptions.DEFAULT.toBuilder().addHeader(AUTHORIZATION, token).build(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndexDocument.java b/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndexDocument.java index a05d7eae4..4a0ef8b4f 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndexDocument.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndexDocument.java @@ -9,82 +9,76 @@ import static no.unit.nva.constants.Words.IMPORT_CANDIDATES_INDEX; import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.constants.Words.TICKETS; - import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; - +import java.util.Objects; +import java.util.Optional; import no.unit.nva.commons.json.JsonSerializable; import no.unit.nva.identifiers.SortableIdentifier; - import nva.commons.core.StringUtils; - import org.opensearch.action.index.IndexRequest; import org.opensearch.common.xcontent.XContentType; -import java.util.Objects; -import java.util.Optional; - public record IndexDocument( - @JsonProperty(CONSUMPTION_ATTRIBUTES) EventConsumptionAttributes consumptionAttributes, - @JsonProperty(BODY) JsonNode resource) - implements JsonSerializable { - - static final String IMPORT_CANDIDATE = "ImportCandidate"; - static final String TICKET = "Ticket"; - static final String RESOURCE = "Resource"; - - public static IndexDocument fromJsonString(String json) { - return attempt(() -> objectMapperWithEmpty.readValue(json, IndexDocument.class)) - .orElseThrow(); + @JsonProperty(CONSUMPTION_ATTRIBUTES) EventConsumptionAttributes consumptionAttributes, + @JsonProperty(BODY) JsonNode resource) + implements JsonSerializable { + + static final String IMPORT_CANDIDATE = "ImportCandidate"; + static final String TICKET = "Ticket"; + static final String RESOURCE = "Resource"; + + public static IndexDocument fromJsonString(String json) { + return attempt(() -> objectMapperWithEmpty.readValue(json, IndexDocument.class)).orElseThrow(); + } + + public IndexDocument validate() { + Objects.requireNonNull(getIndexName()); + Objects.requireNonNull(getDocumentIdentifier()); + return this; + } + + @JsonIgnore + public String getType() { + var indexName = consumptionAttributes.index(); + if (RESOURCES.equals(indexName)) { + return RESOURCE; } - - public IndexDocument validate() { - Objects.requireNonNull(getIndexName()); - Objects.requireNonNull(getDocumentIdentifier()); - return this; - } - - @JsonIgnore - public String getType() { - var indexName = consumptionAttributes.index(); - if (RESOURCES.equals(indexName)) { - return RESOURCE; - } - if (TICKETS.equals(indexName)) { - return TICKET; - } - if (IMPORT_CANDIDATES_INDEX.equals(indexName)) { - return IMPORT_CANDIDATE; - } else { - throw new IllegalArgumentException("Unknown type!"); - } - } - - @JsonIgnore - public String getIndexName() { - return Optional.ofNullable(consumptionAttributes.index()) - .filter(StringUtils::isNotBlank) - .orElseThrow(() -> new RuntimeException(MISSING_INDEX_NAME_IN_RESOURCE)); + if (TICKETS.equals(indexName)) { + return TICKET; } - - @JsonIgnore - public String getDocumentIdentifier() { - return Optional.ofNullable(consumptionAttributes.documentIdentifier()) - .map(SortableIdentifier::toString) - .orElseThrow(() -> new RuntimeException(MISSING_IDENTIFIER_IN_RESOURCE)); - } - - public IndexRequest toIndexRequest() { - return new IndexRequest(getIndexName()) - .source(serializeResource(), XContentType.JSON) - .routing(DEFAULT_SHARD_ID) - .id(getDocumentIdentifier()); - } - - private String serializeResource() { - return attempt(() -> objectMapperWithEmpty.writeValueAsString(resource)).orElseThrow(); + if (IMPORT_CANDIDATES_INDEX.equals(indexName)) { + return IMPORT_CANDIDATE; + } else { + throw new IllegalArgumentException("Unknown type!"); } + } + + @JsonIgnore + public String getIndexName() { + return Optional.ofNullable(consumptionAttributes.index()) + .filter(StringUtils::isNotBlank) + .orElseThrow(() -> new RuntimeException(MISSING_INDEX_NAME_IN_RESOURCE)); + } + + @JsonIgnore + public String getDocumentIdentifier() { + return Optional.ofNullable(consumptionAttributes.documentIdentifier()) + .map(SortableIdentifier::toString) + .orElseThrow(() -> new RuntimeException(MISSING_IDENTIFIER_IN_RESOURCE)); + } + + public IndexRequest toIndexRequest() { + return new IndexRequest(getIndexName()) + .source(serializeResource(), XContentType.JSON) + .routing(DEFAULT_SHARD_ID) + .id(getDocumentIdentifier()); + } + + private String serializeResource() { + return attempt(() -> objectMapperWithEmpty.writeValueAsString(resource)).orElseThrow(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndicesClientWrapper.java b/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndicesClientWrapper.java index b283ee93c..8f7b894b8 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndicesClientWrapper.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/models/IndicesClientWrapper.java @@ -1,7 +1,7 @@ package no.unit.nva.indexingclient.models; +import java.io.IOException; import nva.commons.core.JacocoGenerated; - import org.opensearch.action.admin.indices.delete.DeleteIndexRequest; import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; import org.opensearch.action.support.master.AcknowledgedResponse; @@ -12,8 +12,6 @@ import org.opensearch.client.indices.GetIndexRequest; import org.opensearch.client.indices.GetIndexResponse; -import java.io.IOException; - /** Wrapper class for being able to test calls to the final class IndicesClient. */ @JacocoGenerated public class IndicesClientWrapper { diff --git a/search-commons/src/main/java/no/unit/nva/indexingclient/models/RestHighLevelClientWrapper.java b/search-commons/src/main/java/no/unit/nva/indexingclient/models/RestHighLevelClientWrapper.java index f67aedb7a..d866b20ba 100644 --- a/search-commons/src/main/java/no/unit/nva/indexingclient/models/RestHighLevelClientWrapper.java +++ b/search-commons/src/main/java/no/unit/nva/indexingclient/models/RestHighLevelClientWrapper.java @@ -2,8 +2,8 @@ import static no.unit.nva.constants.Defaults.ENVIRONMENT; +import java.io.IOException; import nva.commons.core.JacocoGenerated; - import org.apache.http.HttpHost; import org.opensearch.action.bulk.BulkRequest; import org.opensearch.action.bulk.BulkResponse; @@ -22,8 +22,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - /** Class for avoiding mocking/spying the ES final classes. */ public class RestHighLevelClientWrapper { diff --git a/search-commons/src/main/java/no/unit/nva/search/common/AggregationFormat.java b/search-commons/src/main/java/no/unit/nva/search/common/AggregationFormat.java index a57164bb4..dbbafd2b3 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/AggregationFormat.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/AggregationFormat.java @@ -1,5 +1,6 @@ package no.unit.nva.search.common; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.BUCKETS; import static no.unit.nva.constants.Words.ENGLISH_CODE; import static no.unit.nva.constants.Words.KEY; @@ -10,22 +11,16 @@ import static no.unit.nva.constants.Words.ZERO; import static no.unit.nva.search.common.AggregationFormat.Constants.DOC_COUNT; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_WORD_ENDING_WITH_HASHTAG; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import static java.util.Objects.nonNull; - import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Streams; - -import no.unit.nva.commons.json.JsonUtils; - -import nva.commons.core.JacocoGenerated; - import java.util.Map; import java.util.Map.Entry; import java.util.stream.Stream; +import no.unit.nva.commons.json.JsonUtils; +import nva.commons.core.JacocoGenerated; /** * Formats the aggregation response from OpenSearch. diff --git a/search-commons/src/main/java/no/unit/nva/search/common/AsType.java b/search-commons/src/main/java/no/unit/nva/search/common/AsType.java index 26156d72c..55d997877 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/AsType.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/AsType.java @@ -1,20 +1,17 @@ package no.unit.nva.search.common; -import static nva.commons.core.StringUtils.EMPTY_STRING; -import static nva.commons.core.attempt.Try.attempt; - import static java.util.Objects.isNull; import static java.util.Objects.nonNull; - -import no.unit.nva.search.common.enums.ParameterKey; -import no.unit.nva.search.common.enums.ParameterKind; - -import org.joda.time.DateTime; +import static nva.commons.core.StringUtils.EMPTY_STRING; +import static nva.commons.core.attempt.Try.attempt; import java.util.Arrays; import java.util.Locale; import java.util.Optional; import java.util.stream.Stream; +import no.unit.nva.search.common.enums.ParameterKey; +import no.unit.nva.search.common.enums.ParameterKind; +import org.joda.time.DateTime; /** * AutoConvert value to Date, Number (or String). diff --git a/search-commons/src/main/java/no/unit/nva/search/common/ContentTypeUtils.java b/search-commons/src/main/java/no/unit/nva/search/common/ContentTypeUtils.java index a9b4a0571..1b5e588af 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/ContentTypeUtils.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/ContentTypeUtils.java @@ -2,40 +2,38 @@ import static org.apache.http.HttpHeaders.ACCEPT; +import java.util.Optional; import nva.commons.apigateway.RequestInfo; - import org.apache.http.entity.ContentType; -import java.util.Optional; - public class ContentTypeUtils { - public static final String VERSION = "version"; - - /** Extract the version field value if present from a header or else null. */ - private static String extractVersion(String headerValue) { - var contentType = ContentType.parse(headerValue); - return contentType.getParameter(VERSION); - } - - /** Extract mimetype field value if present from a header or else null. */ - private static ContentType extractContentType(String headerValue) { - return ContentType.parse(headerValue); - } - - public static String extractVersionFromRequestInfo(RequestInfo requestInfo) { - return Optional.of(requestInfo) - .map(RequestInfo::getHeaders) - .map(map -> map.get(ACCEPT)) - .map(ContentTypeUtils::extractVersion) - .orElse(null); - } - - public static ContentType extractContentTypeFromRequestInfo(RequestInfo requestInfo) { - return Optional.of(requestInfo) - .map(RequestInfo::getHeaders) - .map(map -> map.get(ACCEPT)) - .map(ContentTypeUtils::extractContentType) - .orElse(null); - } + public static final String VERSION = "version"; + + /** Extract the version field value if present from a header or else null. */ + private static String extractVersion(String headerValue) { + var contentType = ContentType.parse(headerValue); + return contentType.getParameter(VERSION); + } + + /** Extract mimetype field value if present from a header or else null. */ + private static ContentType extractContentType(String headerValue) { + return ContentType.parse(headerValue); + } + + public static String extractVersionFromRequestInfo(RequestInfo requestInfo) { + return Optional.of(requestInfo) + .map(RequestInfo::getHeaders) + .map(map -> map.get(ACCEPT)) + .map(ContentTypeUtils::extractVersion) + .orElse(null); + } + + public static ContentType extractContentTypeFromRequestInfo(RequestInfo requestInfo) { + return Optional.of(requestInfo) + .map(RequestInfo::getHeaders) + .map(map -> map.get(ACCEPT)) + .map(ContentTypeUtils::extractContentType) + .orElse(null); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/OpenSearchClient.java b/search-commons/src/main/java/no/unit/nva/search/common/OpenSearchClient.java index e75ad9b93..a4817e56f 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/OpenSearchClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/OpenSearchClient.java @@ -1,29 +1,15 @@ package no.unit.nva.search.common; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.util.stream.Collectors.joining; import static no.unit.nva.auth.AuthorizedBackendClient.AUTHORIZATION_HEADER; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.constants.Words.AMPERSAND; import static no.unit.nva.constants.Words.CONTENT_TYPE; - import static nva.commons.core.attempt.Try.attempt; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.util.stream.Collectors.joining; - import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.net.MediaType; - -import no.unit.nva.commons.json.JsonSerializable; -import no.unit.nva.search.common.jwt.CachedJwtProvider; -import no.unit.nva.search.common.records.QueryContentWrapper; -import no.unit.nva.search.common.records.ResponseLogInfo; -import no.unit.nva.search.common.records.SwsResponse; - -import nva.commons.core.attempt.FunctionWithException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -35,6 +21,14 @@ import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.function.BinaryOperator; +import no.unit.nva.commons.json.JsonSerializable; +import no.unit.nva.search.common.jwt.CachedJwtProvider; +import no.unit.nva.search.common.records.QueryContentWrapper; +import no.unit.nva.search.common.records.ResponseLogInfo; +import no.unit.nva.search.common.records.SwsResponse; +import nva.commons.core.attempt.FunctionWithException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Abstract class for OpenSearch clients. @@ -47,115 +41,111 @@ */ public abstract class OpenSearchClient> { - protected static final Logger logger = LoggerFactory.getLogger(OpenSearchClient.class); + protected static final Logger logger = LoggerFactory.getLogger(OpenSearchClient.class); - protected final HttpClient httpClient; - protected final BodyHandler bodyHandler; - protected final CachedJwtProvider jwtProvider; - protected Instant queryBuilderStart; - protected long fetchDuration; - protected String queryParameters; + protected final HttpClient httpClient; + protected final BodyHandler bodyHandler; + protected final CachedJwtProvider jwtProvider; + protected Instant queryBuilderStart; + protected long fetchDuration; + protected String queryParameters; - public OpenSearchClient(HttpClient httpClient, CachedJwtProvider jwtProvider) { - this.bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); - this.httpClient = httpClient; - this.jwtProvider = jwtProvider; - } + public OpenSearchClient(HttpClient httpClient, CachedJwtProvider jwtProvider) { + this.bodyHandler = HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8); + this.httpClient = httpClient; + this.jwtProvider = jwtProvider; + } - public R doSearch(Q query) { - if (query.filters().hasContent()) { - logger.info(query.filters().toString()); - } - queryBuilderStart = query.getStartTime(); - queryParameters = - query.parameters().asMap().entrySet().stream() - .map(Object::toString) - .collect(joining(AMPERSAND)); - - var completableFutures = - query.assemble() - .map(this::createRequest) - .map(this::fetch) - .map(this::handleResponse) - .toList(); - - return combineAndReturn(completableFutures); + public R doSearch(Q query) { + if (query.filters().hasContent()) { + logger.info(query.filters().toString()); } - - protected CompletableFuture handleResponse( - CompletableFuture> completableFuture) { - return completableFuture.thenApplyAsync( - response -> { - if (response.statusCode() != HTTP_OK) { - throw new RuntimeException(response.body()); - } - return attempt(() -> jsonToResponse(response)) - .map(logAndReturnResult()) - .orElseThrow(); - }); - } - - protected abstract R jsonToResponse(HttpResponse response) - throws JsonProcessingException; - - protected abstract BinaryOperator responseAccumulator(); - - protected abstract FunctionWithException logAndReturnResult(); - - protected R combineAndReturn(List> completableFutures) { - return completableFutures.size() == 2 - ? completableFutures - .get(0) - .thenCombineAsync(completableFutures.get(1), responseAccumulator()) - .join() - : completableFutures.get(0).join(); - } - - protected CompletableFuture> fetch(HttpRequest request) { - var fetchStart = Instant.now(); - return httpClient - .sendAsync(request, bodyHandler) - .thenApplyAsync( - response -> { - fetchDuration = Duration.between(fetchStart, Instant.now()).toMillis(); - return response; - }) - .exceptionallyAsync( - responseFailure -> { - fetchDuration = Duration.between(fetchStart, Instant.now()).toMillis(); - var error = - new ErrorEntry( - request.uri(), - responseFailure.getCause().getMessage()) - .toJsonString(); - logger.error(error); - return null; - }); - } - - protected HttpRequest createRequest(QueryContentWrapper qbs) { - logger.debug(qbs.body()); - return HttpRequest.newBuilder(qbs.uri()) - .headers( - ACCEPT, MediaType.JSON_UTF_8.toString(), - CONTENT_TYPE, MediaType.JSON_UTF_8.toString(), - AUTHORIZATION_HEADER, jwtProvider.getValue().getToken()) - .POST(HttpRequest.BodyPublishers.ofString(qbs.body())) - .build(); - } - - protected String buildLogInfo(SwsResponse result) { - return ResponseLogInfo.builder() - .withTotalTime(totalDuration()) - .withFetchTime(fetchDuration) - .withSwsResponse(result) - .withSearchQuery(queryParameters) - .toJsonString(); - } - - protected long totalDuration() { - return Duration.between(queryBuilderStart, Instant.now()).toMillis(); - } - - protected record ErrorEntry(URI requestUri, String exception) implements JsonSerializable {} + queryBuilderStart = query.getStartTime(); + queryParameters = + query.parameters().asMap().entrySet().stream() + .map(Object::toString) + .collect(joining(AMPERSAND)); + + var completableFutures = + query + .assemble() + .map(this::createRequest) + .map(this::fetch) + .map(this::handleResponse) + .toList(); + + return combineAndReturn(completableFutures); + } + + protected CompletableFuture handleResponse( + CompletableFuture> completableFuture) { + return completableFuture.thenApplyAsync( + response -> { + if (response.statusCode() != HTTP_OK) { + throw new RuntimeException(response.body()); + } + return attempt(() -> jsonToResponse(response)).map(logAndReturnResult()).orElseThrow(); + }); + } + + protected abstract R jsonToResponse(HttpResponse response) throws JsonProcessingException; + + protected abstract BinaryOperator responseAccumulator(); + + protected abstract FunctionWithException logAndReturnResult(); + + protected R combineAndReturn(List> completableFutures) { + return completableFutures.size() == 2 + ? completableFutures + .get(0) + .thenCombineAsync(completableFutures.get(1), responseAccumulator()) + .join() + : completableFutures.get(0).join(); + } + + protected CompletableFuture> fetch(HttpRequest request) { + var fetchStart = Instant.now(); + return httpClient + .sendAsync(request, bodyHandler) + .thenApplyAsync( + response -> { + fetchDuration = Duration.between(fetchStart, Instant.now()).toMillis(); + return response; + }) + .exceptionallyAsync( + responseFailure -> { + fetchDuration = Duration.between(fetchStart, Instant.now()).toMillis(); + var error = + new ErrorEntry(request.uri(), responseFailure.getCause().getMessage()) + .toJsonString(); + logger.error(error); + return null; + }); + } + + protected HttpRequest createRequest(QueryContentWrapper qbs) { + logger.debug(qbs.body()); + return HttpRequest.newBuilder(qbs.uri()) + .headers( + ACCEPT, MediaType.JSON_UTF_8.toString(), + CONTENT_TYPE, MediaType.JSON_UTF_8.toString(), + AUTHORIZATION_HEADER, jwtProvider.getValue().getToken()) + .POST(HttpRequest.BodyPublishers.ofString(qbs.body())) + .build(); + } + + protected String buildLogInfo(SwsResponse result) { + return ResponseLogInfo.builder() + .withTotalTime(totalDuration()) + .withFetchTime(fetchDuration) + .withSwsResponse(result) + .withSearchQuery(queryParameters) + .toJsonString(); + } + + protected long totalDuration() { + return Duration.between(queryBuilderStart, Instant.now()).toMillis(); + } + + protected record ErrorEntry(URI requestUri, String exception) implements JsonSerializable {} } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/ParameterValidator.java b/search-commons/src/main/java/no/unit/nva/search/common/ParameterValidator.java index 13a96e9d4..80c306aed 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/ParameterValidator.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/ParameterValidator.java @@ -1,5 +1,6 @@ package no.unit.nva.search.common; +import static java.util.Objects.isNull; import static no.unit.nva.constants.ErrorMessages.RELEVANCE_SEARCH_AFTER_ARE_MUTUAL_EXCLUSIVE; import static no.unit.nva.constants.ErrorMessages.requiredMissingMessage; import static no.unit.nva.constants.ErrorMessages.validQueryParameterNamesMessage; @@ -10,21 +11,8 @@ import static no.unit.nva.search.common.ContentTypeUtils.extractContentTypeFromRequestInfo; import static no.unit.nva.search.common.constant.Functions.decodeUTF; import static no.unit.nva.search.common.constant.Functions.mergeWithColonOrComma; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import static java.util.Objects.isNull; - -import no.unit.nva.search.common.constant.Patterns; -import no.unit.nva.search.common.enums.ParameterKey; -import no.unit.nva.search.common.enums.ValueEncoding; - -import nva.commons.apigateway.RequestInfo; -import nva.commons.apigateway.exceptions.BadRequestException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.net.URI; import java.util.Arrays; import java.util.Collection; @@ -33,6 +21,13 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import no.unit.nva.search.common.constant.Patterns; +import no.unit.nva.search.common.enums.ParameterKey; +import no.unit.nva.search.common.enums.ValueEncoding; +import nva.commons.apigateway.RequestInfo; +import nva.commons.apigateway.exceptions.BadRequestException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Builder for OpenSearchQuery. @@ -43,270 +38,267 @@ */ @SuppressWarnings("PMD.GodClass") public abstract class ParameterValidator< - K extends Enum & ParameterKey, Q extends SearchQuery> { - - protected static final Logger logger = LoggerFactory.getLogger(ParameterValidator.class); - - protected final transient Set invalidKeys = new HashSet<>(0); - protected final transient SearchQuery query; - protected transient boolean notValidated = true; - - /** - * Constructor of QueryBuilder. - * - *

Usage: Query.builder()
- * .fromRequestInfo(requestInfo)
- * .withRequiredParameters(FROM, SIZE)
- * .build() - */ - public ParameterValidator(SearchQuery query) { - this.query = query; + K extends Enum & ParameterKey, Q extends SearchQuery> { + + protected static final Logger logger = LoggerFactory.getLogger(ParameterValidator.class); + + protected final transient Set invalidKeys = new HashSet<>(0); + protected final transient SearchQuery query; + protected transient boolean notValidated = true; + + /** + * Constructor of QueryBuilder. + * + *

Usage: Query.builder()
+ * .fromRequestInfo(requestInfo)
+ * .withRequiredParameters(FROM, SIZE)
+ * .build() + */ + public ParameterValidator(SearchQuery query) { + this.query = query; + } + + /** + * Builder of Query. + * + * @throws BadRequestException if parameters are invalid or missing + */ + @SuppressWarnings("unchecked") + public Q build() throws BadRequestException { + if (notValidated) { + validate(); } - - /** - * Builder of Query. - * - * @throws BadRequestException if parameters are invalid or missing - */ - @SuppressWarnings("unchecked") - public Q build() throws BadRequestException { - if (notValidated) { - validate(); - } - return (Q) query; + return (Q) query; + } + + /** + * Validator of QueryBuilder. + * + * @throws BadRequestException if parameters are invalid or missing + */ + public ParameterValidator validate() throws BadRequestException { + assignDefaultValues(); + for (var entry : query.parameters().getSearchEntries()) { + validatesEntrySet(entry); } - - /** - * Validator of QueryBuilder. - * - * @throws BadRequestException if parameters are invalid or missing - */ - public ParameterValidator validate() throws BadRequestException { - assignDefaultValues(); - for (var entry : query.parameters().getSearchEntries()) { - validatesEntrySet(entry); - } - for (var entry : query.parameters().getPageEntries()) { - validatesEntrySet(entry); - } - if (!requiredMissing().isEmpty()) { - throw new BadRequestException(requiredMissingMessage(getMissingKeys())); - } - if (!invalidKeys.isEmpty()) { - throw new BadRequestException( - validQueryParameterNamesMessage(invalidKeys, validKeys())); - } - validatedSort(); - - if (hasSearchAfterAndSortByRelevance()) { - throw new BadRequestException(RELEVANCE_SEARCH_AFTER_ARE_MUTUAL_EXCLUSIVE); - } - applyRulesAfterValidation(); - notValidated = false; - return this; + for (var entry : query.parameters().getPageEntries()) { + validatesEntrySet(entry); } - - /** - * DefaultValues are only assigned if they are set as required, otherwise ignored. - * - *

Usage: requiredMissing().forEach(key -> {
- * switch (key) {
- * case LANGUAGE -> query.setValue(key, DEFAULT_LANGUAGE_CODE);
- * default -> { // do nothing }
- * }});
- * - */ - protected abstract void assignDefaultValues(); - - protected abstract void applyRulesAfterValidation(); - - protected abstract Collection validKeys(); - - protected abstract boolean isKeyValid(String keyName); - - protected abstract void validateSortKeyName(String name); - - /** - * Sample code for setValue. - * - *

Usage: var qpKey = keyFromString(key,value);
- * if(qpKey.equals(INVALID)) {
- * invalidKeys.add(key);
- * } else {
- * query.setValue(qpKey, value);
- * }
- * - */ - protected abstract void setValue(String key, String value); - - /** - * Validate sort keys. - * - * @throws BadRequestException if sort key is invalid - */ - protected void validatedSort() throws BadRequestException { - try { - query.sort().asSplitStream(COMMA).forEach(this::validateSortKeyName); - } catch (IllegalArgumentException e) { - throw new BadRequestException(e.getMessage()); - } + if (!requiredMissing().isEmpty()) { + throw new BadRequestException(requiredMissingMessage(getMissingKeys())); } - - protected Set getMissingKeys() { - return requiredMissing().stream() - .map(ParameterKey::asCamelCase) - .collect(Collectors.toSet()); - } - - protected Set requiredMissing() { - return required().stream() - .filter(key -> !query.parameters().isPresent(key)) - .collect(Collectors.toSet()); + if (!invalidKeys.isEmpty()) { + throw new BadRequestException(validQueryParameterNamesMessage(invalidKeys, validKeys())); } + validatedSort(); - protected Set required() { - return query.parameters().otherRequired; + if (hasSearchAfterAndSortByRelevance()) { + throw new BadRequestException(RELEVANCE_SEARCH_AFTER_ARE_MUTUAL_EXCLUSIVE); } - - protected void validatesEntrySet(Map.Entry entry) throws BadRequestException { - final var key = entry.getKey(); - final var value = entry.getValue(); - if (invalidQueryParameter(key, value)) { - final var keyName = key.asCamelCase(); - final var errorMessage = key.errorMessage().formatted(keyName, value); - throw new BadRequestException(errorMessage); - } + applyRulesAfterValidation(); + notValidated = false; + return this; + } + + /** + * DefaultValues are only assigned if they are set as required, otherwise ignored. + * + *

Usage: requiredMissing().forEach(key -> {
+ * switch (key) {
+ * case LANGUAGE -> query.setValue(key, DEFAULT_LANGUAGE_CODE);
+ * default -> { // do nothing }
+ * }});
+ * + */ + protected abstract void assignDefaultValues(); + + protected abstract void applyRulesAfterValidation(); + + protected abstract Collection validKeys(); + + protected abstract boolean isKeyValid(String keyName); + + protected abstract void validateSortKeyName(String name); + + /** + * Sample code for setValue. + * + *

Usage: var qpKey = keyFromString(key,value);
+ * if(qpKey.equals(INVALID)) {
+ * invalidKeys.add(key);
+ * } else {
+ * query.setValue(qpKey, value);
+ * }
+ * + */ + protected abstract void setValue(String key, String value); + + /** + * Validate sort keys. + * + * @throws BadRequestException if sort key is invalid + */ + protected void validatedSort() throws BadRequestException { + try { + query.sort().asSplitStream(COMMA).forEach(this::validateSortKeyName); + } catch (IllegalArgumentException e) { + throw new BadRequestException(e.getMessage()); } - - protected String getDecodedValue(ParameterKey qpKey, String value) { - return (qpKey.valueEncoding() == ValueEncoding.NONE ? value : decodeUTF(value)) - .replaceAll(Patterns.PATTERN_IS_NON_PRINTABLE_CHARACTERS, EMPTY_STRING); - } - - /** Adds query and path parameters from requestInfo. */ - public ParameterValidator fromRequestInfo(RequestInfo requestInfo) { - var contentType = extractContentTypeFromRequestInfo(requestInfo); - query.setMediaType(isNull(contentType) ? null : contentType.getMimeType()); - var uri = URI.create(HTTPS + requestInfo.getDomainName() + requestInfo.getPath()); - query.setAccessRights(requestInfo.getAccessRights()); - query.setNvaSearchApiUri(uri); - return fromMultiValueParameters(requestInfo.getMultiValueQueryStringParameters()); - } - - /** - * Adds testParameters from query. - * - * @apiNote This is intended to be used in runtime - */ - public ParameterValidator fromMultiValueParameters(Map> parameters) { - parameters.forEach((k, v) -> v.forEach(value -> setValue(k, value))); - return this; - } - - /** - * Adds testParameters from query. - * - * @apiNote This is intended to be used when setting up tests, or from {@link #fromRequestInfo} - */ - public ParameterValidator fromTestParameterMap(Map parameters) { - parameters.forEach(this::setValue); - return this; - } - - /** - * Adds testParameters from query. - * - * @apiNote This is intended to be used when setting up tests. - */ - public ParameterValidator fromTestQueryParameters( - Collection> testParameters) { - testParameters.forEach(this::setEntryValue); - return this; - } - - private void setEntryValue(Map.Entry entry) { - setValue(entry.getKey(), entry.getValue()); - } - - /** - * Defines which parameters are required. - * - *

In order to improve ease of use, you can add a default value to each required parameter, - * and it will be used, if it is not proved by the requester. Implement default values in {@link - * #assignDefaultValues()} - * - * @param requiredParameters comma seperated QueryParameterKeys - */ - @SafeVarargs - public final ParameterValidator withRequiredParameters(K... requiredParameters) { - var tmpSet = Set.of(requiredParameters); - query.parameters().otherRequired.addAll(tmpSet); - return this; - } - - public final ParameterValidator withMediaType(String mediaType) { - query.setMediaType(mediaType); - return this; - } - - /** - * Overrides/set a key value pair - * - * @param key to assign value to - * @param value assigned - * @return builder - */ - public ParameterValidator withParameter(K key, String value) { - query.parameters().set(key, value); - return this; - } - - /** - * When running docker tests, the current host needs to be specified. - * - * @param uri URI to local docker test instance - * @apiNote This is intended to be used when setting up tests. - */ - public final ParameterValidator withDockerHostUri(URI uri) { - query.setOpenSearchUri(uri); - return this; - } - - public final ParameterValidator withAlwaysIncludedFields(List includedFields) { - query.setAlwaysIncludedFields(includedFields); - return this; - } - - public final ParameterValidator withAlwaysExcludedFields(List excludedFields) { - query.setAlwaysExcludedFields(excludedFields); - return this; - } - - public final ParameterValidator withAlwaysExcludedFields(String... excludedFields) { - query.setAlwaysExcludedFields(List.of(excludedFields)); - return this; - } - - protected void mergeToKey(K key, String value) { - query.parameters().set(key, mergeWithColonOrComma(query.parameters().get(key).as(), value)); - } - - protected String ignoreInvalidFields(String value) { - return ALL.equalsIgnoreCase(value) || isNull(value) - ? ALL - : Arrays.stream(value.split(COMMA)) - .filter(this::isKeyValid) // ignoring invalid keys - .collect(Collectors.joining(COMMA)); - } - - protected boolean invalidQueryParameter(K key, String value) { - return isNull(value) - || Arrays.stream(value.split(COMMA)) - .noneMatch(singleValue -> singleValue.matches(key.valuePattern())); - } - - private boolean hasSearchAfterAndSortByRelevance() { - return query.parameters().isPresent(query.keySearchAfter()) - && query.sort().toString().contains(RELEVANCE_KEY_NAME); + } + + protected Set getMissingKeys() { + return requiredMissing().stream().map(ParameterKey::asCamelCase).collect(Collectors.toSet()); + } + + protected Set requiredMissing() { + return required().stream() + .filter(key -> !query.parameters().isPresent(key)) + .collect(Collectors.toSet()); + } + + protected Set required() { + return query.parameters().otherRequired; + } + + protected void validatesEntrySet(Map.Entry entry) throws BadRequestException { + final var key = entry.getKey(); + final var value = entry.getValue(); + if (invalidQueryParameter(key, value)) { + final var keyName = key.asCamelCase(); + final var errorMessage = key.errorMessage().formatted(keyName, value); + throw new BadRequestException(errorMessage); } + } + + protected String getDecodedValue(ParameterKey qpKey, String value) { + return (qpKey.valueEncoding() == ValueEncoding.NONE ? value : decodeUTF(value)) + .replaceAll(Patterns.PATTERN_IS_NON_PRINTABLE_CHARACTERS, EMPTY_STRING); + } + + /** Adds query and path parameters from requestInfo. */ + public ParameterValidator fromRequestInfo(RequestInfo requestInfo) { + var contentType = extractContentTypeFromRequestInfo(requestInfo); + query.setMediaType(isNull(contentType) ? null : contentType.getMimeType()); + var uri = URI.create(HTTPS + requestInfo.getDomainName() + requestInfo.getPath()); + query.setAccessRights(requestInfo.getAccessRights()); + query.setNvaSearchApiUri(uri); + return fromMultiValueParameters(requestInfo.getMultiValueQueryStringParameters()); + } + + /** + * Adds testParameters from query. + * + * @apiNote This is intended to be used in runtime + */ + public ParameterValidator fromMultiValueParameters(Map> parameters) { + parameters.forEach((k, v) -> v.forEach(value -> setValue(k, value))); + return this; + } + + /** + * Adds testParameters from query. + * + * @apiNote This is intended to be used when setting up tests, or from {@link #fromRequestInfo} + */ + public ParameterValidator fromTestParameterMap(Map parameters) { + parameters.forEach(this::setValue); + return this; + } + + /** + * Adds testParameters from query. + * + * @apiNote This is intended to be used when setting up tests. + */ + public ParameterValidator fromTestQueryParameters( + Collection> testParameters) { + testParameters.forEach(this::setEntryValue); + return this; + } + + private void setEntryValue(Map.Entry entry) { + setValue(entry.getKey(), entry.getValue()); + } + + /** + * Defines which parameters are required. + * + *

In order to improve ease of use, you can add a default value to each required parameter, and + * it will be used, if it is not proved by the requester. Implement default values in {@link + * #assignDefaultValues()} + * + * @param requiredParameters comma seperated QueryParameterKeys + */ + @SafeVarargs + public final ParameterValidator withRequiredParameters(K... requiredParameters) { + var tmpSet = Set.of(requiredParameters); + query.parameters().otherRequired.addAll(tmpSet); + return this; + } + + public final ParameterValidator withMediaType(String mediaType) { + query.setMediaType(mediaType); + return this; + } + + /** + * Overrides/set a key value pair + * + * @param key to assign value to + * @param value assigned + * @return builder + */ + public ParameterValidator withParameter(K key, String value) { + query.parameters().set(key, value); + return this; + } + + /** + * When running docker tests, the current host needs to be specified. + * + * @param uri URI to local docker test instance + * @apiNote This is intended to be used when setting up tests. + */ + public final ParameterValidator withDockerHostUri(URI uri) { + query.setOpenSearchUri(uri); + return this; + } + + public final ParameterValidator withAlwaysIncludedFields(List includedFields) { + query.setAlwaysIncludedFields(includedFields); + return this; + } + + public final ParameterValidator withAlwaysExcludedFields(List excludedFields) { + query.setAlwaysExcludedFields(excludedFields); + return this; + } + + public final ParameterValidator withAlwaysExcludedFields(String... excludedFields) { + query.setAlwaysExcludedFields(List.of(excludedFields)); + return this; + } + + protected void mergeToKey(K key, String value) { + query.parameters().set(key, mergeWithColonOrComma(query.parameters().get(key).as(), value)); + } + + protected String ignoreInvalidFields(String value) { + return ALL.equalsIgnoreCase(value) || isNull(value) + ? ALL + : Arrays.stream(value.split(COMMA)) + .filter(this::isKeyValid) // ignoring invalid keys + .collect(Collectors.joining(COMMA)); + } + + protected boolean invalidQueryParameter(K key, String value) { + return isNull(value) + || Arrays.stream(value.split(COMMA)) + .noneMatch(singleValue -> singleValue.matches(key.valuePattern())); + } + + private boolean hasSearchAfterAndSortByRelevance() { + return query.parameters().isPresent(query.keySearchAfter()) + && query.sort().toString().contains(RELEVANCE_KEY_NAME); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/Query.java b/search-commons/src/main/java/no/unit/nva/search/common/Query.java index a4f62da62..74375079b 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/Query.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/Query.java @@ -2,13 +2,12 @@ import static no.unit.nva.search.common.constant.Functions.readSearchInfrastructureApiUri; -import no.unit.nva.search.common.enums.ParameterKey; -import no.unit.nva.search.common.records.HttpResponseFormatter; -import no.unit.nva.search.common.records.QueryContentWrapper; - import java.net.URI; import java.time.Instant; import java.util.stream.Stream; +import no.unit.nva.search.common.enums.ParameterKey; +import no.unit.nva.search.common.records.HttpResponseFormatter; +import no.unit.nva.search.common.records.QueryContentWrapper; /** * Query is a class that represents a query to the search service. @@ -18,37 +17,37 @@ * define the parameters that can be used in the query. */ public abstract class Query & ParameterKey> { - private final transient Instant startTime; - protected transient URI infrastructureApiUri = URI.create(readSearchInfrastructureApiUri()); - protected transient QueryKeys queryKeys; - protected transient QueryFilter queryFilter = new QueryFilter(); - - protected Query() { - startTime = Instant.now(); - } - - public abstract Stream assemble(); - - /** - * Method to mimic Domain driven design. - * - * @param queryClient simple service to do i/o (http) - * @return HttpResponseFormatter a formatter for the response - */ - public abstract > HttpResponseFormatter doSearch( - OpenSearchClient queryClient); - - protected abstract URI openSearchUri(); - - public QueryKeys parameters() { - return queryKeys; - } - - public QueryFilter filters() { - return queryFilter; - } - - public Instant getStartTime() { - return startTime; - } + private final transient Instant startTime; + protected transient URI infrastructureApiUri = URI.create(readSearchInfrastructureApiUri()); + protected transient QueryKeys queryKeys; + protected transient QueryFilter queryFilter = new QueryFilter(); + + protected Query() { + startTime = Instant.now(); + } + + public abstract Stream assemble(); + + /** + * Method to mimic Domain driven design. + * + * @param queryClient simple service to do i/o (http) + * @return HttpResponseFormatter a formatter for the response + */ + public abstract > HttpResponseFormatter doSearch( + OpenSearchClient queryClient); + + protected abstract URI openSearchUri(); + + public QueryKeys parameters() { + return queryKeys; + } + + public QueryFilter filters() { + return queryFilter; + } + + public Instant getStartTime() { + return startTime; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/QueryFilter.java b/search-commons/src/main/java/no/unit/nva/search/common/QueryFilter.java index b61037f96..73d09b98e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/QueryFilter.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/QueryFilter.java @@ -2,14 +2,13 @@ import static no.unit.nva.constants.Words.COMMA; -import org.opensearch.index.query.BoolQueryBuilder; -import org.opensearch.index.query.QueryBuilder; -import org.opensearch.index.query.QueryBuilders; - import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; /** * QueryFilter is a class that represents a query filter to the search service. @@ -17,42 +16,42 @@ * @author Stig Norland */ public class QueryFilter { - private static final String SUFFIX = "]"; - private static final String PREFIX = "["; - - private final transient Map filters = new HashMap<>(); - - public QueryFilter() {} - - public BoolQueryBuilder get() { - var boolQueryBuilder = QueryBuilders.boolQuery(); - filters.values().forEach(boolQueryBuilder::must); - return boolQueryBuilder; - } - - /** - * Clears filter and sets new filters. - * - * @param filters QueryBuilder - */ - public void set(QueryBuilder... filters) { - this.filters.clear(); - Arrays.stream(filters).forEach(this::add); - } - - public boolean hasContent() { - return !filters.isEmpty(); - } - - public QueryFilter add(QueryBuilder builder) { - this.filters.put(builder.queryName(), builder); - return this; - } - - @Override - public String toString() { - return filters.values().stream() - .map(QueryBuilder::toString) - .collect(Collectors.joining(COMMA, PREFIX, SUFFIX)); - } + private static final String SUFFIX = "]"; + private static final String PREFIX = "["; + + private final transient Map filters = new HashMap<>(); + + public QueryFilter() {} + + public BoolQueryBuilder get() { + var boolQueryBuilder = QueryBuilders.boolQuery(); + filters.values().forEach(boolQueryBuilder::must); + return boolQueryBuilder; + } + + /** + * Clears filter and sets new filters. + * + * @param filters QueryBuilder + */ + public void set(QueryBuilder... filters) { + this.filters.clear(); + Arrays.stream(filters).forEach(this::add); + } + + public boolean hasContent() { + return !filters.isEmpty(); + } + + public QueryFilter add(QueryBuilder builder) { + this.filters.put(builder.queryName(), builder); + return this; + } + + @Override + public String toString() { + return filters.values().stream() + .map(QueryBuilder::toString) + .collect(Collectors.joining(COMMA, PREFIX, SUFFIX)); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/QueryKeys.java b/search-commons/src/main/java/no/unit/nva/search/common/QueryKeys.java index 2ebd555d9..81c489f85 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/QueryKeys.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/QueryKeys.java @@ -1,14 +1,10 @@ package no.unit.nva.search.common; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.PLUS; import static no.unit.nva.constants.Words.SPACE; import static no.unit.nva.search.common.constant.Functions.decodeUTF; -import static java.util.Objects.nonNull; - -import no.unit.nva.search.common.enums.ParameterKey; -import no.unit.nva.search.common.enums.ValueEncoding; - import java.util.Comparator; import java.util.HashSet; import java.util.LinkedHashMap; @@ -16,6 +12,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Stream; +import no.unit.nva.search.common.enums.ParameterKey; +import no.unit.nva.search.common.enums.ValueEncoding; /** * This class operates on the queryKeys that a request provides. @@ -26,92 +24,92 @@ */ public class QueryKeys & ParameterKey> { - protected final transient Set otherRequired; - private final transient Map page; - private final transient Map search; - private final K fields; - - public QueryKeys(K fields) { - this.fields = fields; - search = new ConcurrentHashMap<>(); - page = new ConcurrentHashMap<>(); - otherRequired = new HashSet<>(); + protected final transient Set otherRequired; + private final transient Map page; + private final transient Map search; + private final K fields; + + public QueryKeys(K fields) { + this.fields = fields; + search = new ConcurrentHashMap<>(); + page = new ConcurrentHashMap<>(); + otherRequired = new HashSet<>(); + } + + public Stream getSearchKeys() { + return search.keySet().stream(); + } + + public Set> getSearchEntries() { + return search.entrySet(); + } + + public Set> getPageEntries() { + return page.entrySet(); + } + + /** + * Query Parameters with string Keys. + * + * @return Map of String and String + */ + public Map asMap() { + var results = new LinkedHashMap(); + Stream.of(search.entrySet(), page.entrySet()) + .flatMap(Set::stream) + .sorted(Comparator.comparingInt(o -> o.getKey().ordinal())) + .forEach(entry -> results.put(toApiKey(entry), toApiValue(entry))); + return results; + } + + private String toApiKey(Map.Entry entry) { + return entry.getKey().asCamelCase(); + } + + private String toApiValue(Map.Entry entry) { + return entry.getValue().replace(SPACE, PLUS); + } + + /** + * Get value from Query Parameter Map with key. + * + * @param key to look up. + * @return String content raw + */ + public AsType get(K key) { + return new AsType<>(search.containsKey(key) ? search.get(key) : page.get(key), key); + } + + /** + * Add a key value pair to Parameters. + * + * @param key to add to. + * @param value to assign + */ + public void set(K key, String value) { + if (nonNull(value)) { + var decodedValue = key.valueEncoding() == ValueEncoding.NONE ? value : decodeUTF(value); + if (isPagingValue(key)) { + page.put(key, decodedValue); + } else { + search.put(key, decodedValue); + } } + } - public Stream getSearchKeys() { - return search.keySet().stream(); - } + private boolean isPagingValue(K key) { + return key.ordinal() >= fields.ordinal(); + } - public Set> getSearchEntries() { - return search.entrySet(); - } + public AsType remove(K key) { + return new AsType<>(search.containsKey(key) ? search.remove(key) : page.remove(key), key); + } - public Set> getPageEntries() { - return page.entrySet(); - } + public boolean isPresent(K key) { + return search.containsKey(key) || page.containsKey(key); + } - /** - * Query Parameters with string Keys. - * - * @return Map of String and String - */ - public Map asMap() { - var results = new LinkedHashMap(); - Stream.of(search.entrySet(), page.entrySet()) - .flatMap(Set::stream) - .sorted(Comparator.comparingInt(o -> o.getKey().ordinal())) - .forEach(entry -> results.put(toApiKey(entry), toApiValue(entry))); - return results; - } - - private String toApiKey(Map.Entry entry) { - return entry.getKey().asCamelCase(); - } - - private String toApiValue(Map.Entry entry) { - return entry.getValue().replace(SPACE, PLUS); - } - - /** - * Get value from Query Parameter Map with key. - * - * @param key to look up. - * @return String content raw - */ - public AsType get(K key) { - return new AsType<>(search.containsKey(key) ? search.get(key) : page.get(key), key); - } - - /** - * Add a key value pair to Parameters. - * - * @param key to add to. - * @param value to assign - */ - public void set(K key, String value) { - if (nonNull(value)) { - var decodedValue = key.valueEncoding() == ValueEncoding.NONE ? value : decodeUTF(value); - if (isPagingValue(key)) { - page.put(key, decodedValue); - } else { - search.put(key, decodedValue); - } - } - } - - private boolean isPagingValue(K key) { - return key.ordinal() >= fields.ordinal(); - } - - public AsType remove(K key) { - return new AsType<>(search.containsKey(key) ? search.remove(key) : page.remove(key), key); - } - - public boolean isPresent(K key) { - return search.containsKey(key) || page.containsKey(key); - } - - public AsType ifPresent(K key) { - return isPresent(key) ? get(key) : null; - } + public AsType ifPresent(K key) { + return isPresent(key) ? get(key) : null; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/SearchQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/SearchQuery.java index d3c8c5393..fd8d73976 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/SearchQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/SearchQuery.java @@ -2,7 +2,7 @@ import static com.google.common.net.MediaType.CSV_UTF_8; import static com.google.common.net.MediaType.JSON_UTF_8; - +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Defaults.DEFAULT_SORT_ORDER; import static no.unit.nva.constants.Defaults.ZERO_RESULTS_AGGREGATION_ONLY; import static no.unit.nva.constants.Words.ALL; @@ -15,13 +15,19 @@ import static no.unit.nva.search.common.constant.Patterns.COLON_OR_SPACE; import static no.unit.nva.search.common.enums.FieldOperator.NOT_ALL_OF; import static no.unit.nva.search.common.enums.FieldOperator.NOT_ANY_OF; - import static nva.commons.core.attempt.Try.attempt; -import static java.util.Objects.nonNull; - import com.google.common.net.MediaType; - +import java.net.URI; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.constants.ErrorMessages; import no.unit.nva.constants.Words; import no.unit.nva.search.common.builder.AcrossFieldsQuery; @@ -38,10 +44,8 @@ import no.unit.nva.search.common.records.HttpResponseFormatter; import no.unit.nva.search.common.records.QueryContentWrapper; import no.unit.nva.search.common.records.SwsResponse; - import nva.commons.apigateway.AccessRight; import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder.Type; import org.opensearch.index.query.Operator; @@ -56,17 +60,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.util.ArrayList; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * SearchQuery is a class that represents a query to the search service. * @@ -75,308 +68,303 @@ * define the parameters that can be used in the query. */ @SuppressWarnings({ - "PMD.CouplingBetweenObjects", - "PMD.GodClass", - "PMD.ConstructorCallsOverridableMethod" + "PMD.CouplingBetweenObjects", + "PMD.GodClass", + "PMD.ConstructorCallsOverridableMethod" }) public abstract class SearchQuery & ParameterKey> extends Query { - protected static final Logger logger = LoggerFactory.getLogger(SearchQuery.class); - private final transient Set accessRights; - private transient MediaType mediaType; - private transient Set excludedFields = Set.of(); - private transient Set includedFields = Set.of("*"); - - /** - * Always set at runtime by ParameterValidator.fromRequestInfo(RequestInfo requestInfo); This - * value only used in debug and tests. - */ - private transient URI gatewayUri = - URI.create("https://api.dev.nva.aws.unit.no/resource/search"); - - protected SearchQuery() { - super(); - queryKeys = new QueryKeys<>(keyFields()); - accessRights = EnumSet.noneOf(AccessRight.class); - setMediaType(JSON_UTF_8.toString()); - } - - protected abstract K keyFields(); - - protected abstract AsType sort(); - - protected abstract String[] exclude(); - - protected abstract String[] include(); - - protected abstract K keyAggregation(); - - protected abstract K keySearchAfter(); - - protected abstract K toKey(String keyName); - - protected abstract SortKey toSortKey(String sortName); - - protected abstract List builderAggregations(); - - protected abstract Stream> builderCustomQueryStream(K key); - - public boolean hasAccessRights(AccessRight... rights) { - return accessRights.containsAll(List.of(rights)); - } - - protected void setAccessRights(List accessRights) { - this.accessRights.clear(); - this.accessRights.addAll(accessRights); - } - - public void setAlwaysExcludedFields(List fieldNames) { - this.excludedFields = new HashSet<>(fieldNames); - } - - public void setAlwaysIncludedFields(List fieldNames) { - this.includedFields = new HashSet<>(fieldNames); - } - - protected Set getExcludedFields() { - return excludedFields; - } - - protected Set getIncludedFields() { - return includedFields; - } - - protected void setOpenSearchUri(URI openSearchUri) { - this.infrastructureApiUri = openSearchUri; - } - - @JacocoGenerated // default can only be tested if we add a new fieldtype not in use.... - protected Stream> builderStreamDefaultQuery(K key) { - final var value = parameters().get(key).toString(); - return switch (key.fieldType()) { - case FUZZY_KEYWORD -> new FuzzyKeywordQuery().buildQuery(key, value); - case KEYWORD -> new KeywordQuery().buildQuery(key, value); - case TEXT -> new TextQuery().buildQuery(key, value); - case FLAG -> Stream.empty(); - case CUSTOM -> builderCustomQueryStream(key); - case NUMBER, DATE -> new RangeQuery().buildQuery(key, value); - case ACROSS_FIELDS -> new AcrossFieldsQuery().buildQuery(key, value); - case EXISTS -> new ExistsQuery().buildQuery(key, value); - case FREE_TEXT -> Functions.queryToEntry(key, builderSearchAllQuery(key)); - case HAS_PARTS -> new HasPartsQuery().buildQuery(key, value); - case PART_OF -> new PartOfQuery().buildQuery(key, value); - default -> throw new RuntimeException(ErrorMessages.HANDLER_NOT_DEFINED + key.name()); - }; - } - - @Override - public Stream assemble() { - // TODO extract builderDefaultSearchSource() and this content into a separate class? - var contentWrappers = new ArrayList(numberOfRequests()); - var builder = builderDefaultSearchSource(); - - handleFetchSource(builder); - handleAggregation(builder, contentWrappers); - handleSearchAfter(builder); - handleSorting(builder); - contentWrappers.add(new QueryContentWrapper(builder.toString(), this.openSearchUri())); - return contentWrappers.stream(); - } - - @Override - public > HttpResponseFormatter doSearch( - OpenSearchClient queryClient) { - return new HttpResponseFormatter<>( - (SwsResponse) queryClient.doSearch((Q) this), - getMediaType(), - getNvaSearchApiUri(), - from().as(), - size().as(), - facetPaths(), - parameters()); - } - - protected abstract AsType from(); - - protected abstract AsType size(); - - /** - * Path to each and every facet defined in builderAggregations(). - * - * @return MapOf(Name, Path) - */ - protected abstract Map facetPaths(); - - public MediaType getMediaType() { - return mediaType; - } - - final void setMediaType(String mediaType) { - if (nonNull(mediaType) && mediaType.contains(Words.TEXT_CSV)) { - this.mediaType = CSV_UTF_8; - } else { - this.mediaType = JSON_UTF_8; - } - } - - public URI getNvaSearchApiUri() { - return gatewayUri; - } - - @JacocoGenerated - public void setNvaSearchApiUri(URI gatewayUri) { - this.gatewayUri = gatewayUri; - } - - /** - * Creates a boolean query, with all the search parameters. - * - * @return a BoolQueryBuilder - */ - protected BoolQueryBuilder builderMainQuery() { - var boolQueryBuilder = QueryBuilders.boolQuery(); - parameters() - .getSearchKeys() - .flatMap(this::builderStreamDefaultQuery) - .forEach( - entry -> { - if (isMustNot(entry.getKey())) { - boolQueryBuilder.mustNot(entry.getValue()); - } else { - boolQueryBuilder.must(entry.getValue()); - } - }); - return boolQueryBuilder; - } - - protected Stream> builderStreamFieldSort() { - return sort().asSplitStream(COMMA) - .flatMap( - item -> { - final var parts = item.split(COLON_OR_SPACE); - final var order = - SortOrder.fromString( - attempt(() -> parts[1]) - .orElse((f) -> DEFAULT_SORT_ORDER)); - final var sortKey = toSortKey(parts[0]); - - return RELEVANCE_KEY_NAME.equalsIgnoreCase(sortKey.name()) - ? Stream.of(SortBuilders.scoreSort().order(order)) - : sortKey.jsonPaths() - .map( - path -> - SortBuilders.fieldSort(path) - .order(order) - .missing(SORT_LAST)); - }); - } - - protected AggregationBuilder builderAggregationsWithFilter() { - var aggrFilter = AggregationBuilders.filter(POST_FILTER, filters().get()); - builderAggregations().forEach(aggrFilter::subAggregation); - return aggrFilter; - } - - protected SearchSourceBuilder builderDefaultSearchSource() { - var queryBuilder = - parameters().getSearchKeys().findAny().isEmpty() - ? QueryBuilders.matchAllQuery() - : builderMainQuery(); - - return new SearchSourceBuilder() - .query(queryBuilder) - .size(size().as()) - .from(from().as()) - .postFilter(filters().get()) - .trackTotalHits(true); - } - - /** - * Creates a multi match query, all words needs to be present, within a document. - * - * @return a MultiMatchQueryBuilder - */ - protected QueryBuilder builderSearchAllQuery(K searchAllKey) { - var fields = mapOfPathAndBoost(parameters().get(keyFields())); - var value = parameters().get(searchAllKey).toString(); - return QueryBuilders.multiMatchQuery(value) - .fields(fields) - .type(Type.CROSS_FIELDS) - .operator(Operator.AND); - } - - protected Map mapOfPathAndBoost(AsType fieldValue) { - return fieldValue.isEmpty() || fieldValue.asLowerCase().contains(ALL) - ? Map.of(ASTERISK, 1F) // NONE or ALL -> <'*',1.0> - : fieldValue - .asSplitStream(COMMA) - .map(this::toKey) - .flatMap(this::entryStreamOfPathAndBoost) - .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); - } - - private void handleAggregation( - SearchSourceBuilder builder, List contentWrappers) { - if (hasAggregation()) { - var aggregationBuilder = builder.shallowCopy(); - aggregationBuilder.size(ZERO_RESULTS_AGGREGATION_ONLY); - aggregationBuilder.aggregation(builderAggregationsWithFilter()); - contentWrappers.add( - new QueryContentWrapper(aggregationBuilder.toString(), this.openSearchUri())); - } - } - - private void handleFetchSource(SearchSourceBuilder builder) { - if (isFetchSource()) { - builder.fetchSource(include(), exclude()); - } else { - builder.fetchSource(true); - } - } - - private void handleSearchAfter(SearchSourceBuilder builder) { - var sortKeys = parameters().remove(keySearchAfter()).split(COMMA); - if (nonNull(sortKeys)) { - builder.searchAfter(sortKeys); - } - } - - private void handleSorting(SearchSourceBuilder builder) { - if (hasSortBy(RELEVANCE_KEY_NAME)) { - // Not very well documented. - // This allows sorting on relevance together with other fields. - builder.trackScores(true); - } - builderStreamFieldSort().forEach(builder::sort); - } - - private Stream> entryStreamOfPathAndBoost(K key) { - return key.searchFields(KEYWORD_FALSE).map(jsonPath -> entryOfPathAndBoost(key, jsonPath)); - } - - private Entry entryOfPathAndBoost(K key, String jsonPath) { - return Map.entry(jsonPath, key.fieldBoost()); - } - - private int numberOfRequests() { - return hasAggregation() ? 2 : 1; - } - - protected boolean hasSortBy(String sortKeyName) { - var sorts = sort().toString(); - return nonNull(sorts) && sorts.split(COMMA).length > 1 && sorts.contains(sortKeyName); - } - - private boolean isMustNot(K key) { - return NOT_ALL_OF.equals(key.searchOperator()) || NOT_ANY_OF.equals(key.searchOperator()); - } - - private boolean isFetchSource() { - return nonNull(exclude()) || nonNull(include()); - } - - private boolean hasAggregation() { - return getMediaType().is(JSON_UTF_8) - && ALL.equalsIgnoreCase(parameters().get(keyAggregation()).as()); - } + protected static final Logger logger = LoggerFactory.getLogger(SearchQuery.class); + private final transient Set accessRights; + private transient MediaType mediaType; + private transient Set excludedFields = Set.of(); + private transient Set includedFields = Set.of("*"); + + /** + * Always set at runtime by ParameterValidator.fromRequestInfo(RequestInfo requestInfo); This + * value only used in debug and tests. + */ + private transient URI gatewayUri = URI.create("https://api.dev.nva.aws.unit.no/resource/search"); + + protected SearchQuery() { + super(); + queryKeys = new QueryKeys<>(keyFields()); + accessRights = EnumSet.noneOf(AccessRight.class); + setMediaType(JSON_UTF_8.toString()); + } + + protected abstract K keyFields(); + + protected abstract AsType sort(); + + protected abstract String[] exclude(); + + protected abstract String[] include(); + + protected abstract K keyAggregation(); + + protected abstract K keySearchAfter(); + + protected abstract K toKey(String keyName); + + protected abstract SortKey toSortKey(String sortName); + + protected abstract List builderAggregations(); + + protected abstract Stream> builderCustomQueryStream(K key); + + public boolean hasAccessRights(AccessRight... rights) { + return accessRights.containsAll(List.of(rights)); + } + + protected void setAccessRights(List accessRights) { + this.accessRights.clear(); + this.accessRights.addAll(accessRights); + } + + public void setAlwaysExcludedFields(List fieldNames) { + this.excludedFields = new HashSet<>(fieldNames); + } + + public void setAlwaysIncludedFields(List fieldNames) { + this.includedFields = new HashSet<>(fieldNames); + } + + protected Set getExcludedFields() { + return excludedFields; + } + + protected Set getIncludedFields() { + return includedFields; + } + + protected void setOpenSearchUri(URI openSearchUri) { + this.infrastructureApiUri = openSearchUri; + } + + @JacocoGenerated // default can only be tested if we add a new fieldtype not in use.... + protected Stream> builderStreamDefaultQuery(K key) { + final var value = parameters().get(key).toString(); + return switch (key.fieldType()) { + case FUZZY_KEYWORD -> new FuzzyKeywordQuery().buildQuery(key, value); + case KEYWORD -> new KeywordQuery().buildQuery(key, value); + case TEXT -> new TextQuery().buildQuery(key, value); + case FLAG -> Stream.empty(); + case CUSTOM -> builderCustomQueryStream(key); + case NUMBER, DATE -> new RangeQuery().buildQuery(key, value); + case ACROSS_FIELDS -> new AcrossFieldsQuery().buildQuery(key, value); + case EXISTS -> new ExistsQuery().buildQuery(key, value); + case FREE_TEXT -> Functions.queryToEntry(key, builderSearchAllQuery(key)); + case HAS_PARTS -> new HasPartsQuery().buildQuery(key, value); + case PART_OF -> new PartOfQuery().buildQuery(key, value); + default -> throw new RuntimeException(ErrorMessages.HANDLER_NOT_DEFINED + key.name()); + }; + } + + @Override + public Stream assemble() { + // TODO extract builderDefaultSearchSource() and this content into a separate class? + var contentWrappers = new ArrayList(numberOfRequests()); + var builder = builderDefaultSearchSource(); + + handleFetchSource(builder); + handleAggregation(builder, contentWrappers); + handleSearchAfter(builder); + handleSorting(builder); + contentWrappers.add(new QueryContentWrapper(builder.toString(), this.openSearchUri())); + return contentWrappers.stream(); + } + + @Override + public > HttpResponseFormatter doSearch( + OpenSearchClient queryClient) { + return new HttpResponseFormatter<>( + (SwsResponse) queryClient.doSearch((Q) this), + getMediaType(), + getNvaSearchApiUri(), + from().as(), + size().as(), + facetPaths(), + parameters()); + } + + protected abstract AsType from(); + + protected abstract AsType size(); + + /** + * Path to each and every facet defined in builderAggregations(). + * + * @return MapOf(Name, Path) + */ + protected abstract Map facetPaths(); + + public MediaType getMediaType() { + return mediaType; + } + + final void setMediaType(String mediaType) { + if (nonNull(mediaType) && mediaType.contains(Words.TEXT_CSV)) { + this.mediaType = CSV_UTF_8; + } else { + this.mediaType = JSON_UTF_8; + } + } + + public URI getNvaSearchApiUri() { + return gatewayUri; + } + + @JacocoGenerated + public void setNvaSearchApiUri(URI gatewayUri) { + this.gatewayUri = gatewayUri; + } + + /** + * Creates a boolean query, with all the search parameters. + * + * @return a BoolQueryBuilder + */ + protected BoolQueryBuilder builderMainQuery() { + var boolQueryBuilder = QueryBuilders.boolQuery(); + parameters() + .getSearchKeys() + .flatMap(this::builderStreamDefaultQuery) + .forEach( + entry -> { + if (isMustNot(entry.getKey())) { + boolQueryBuilder.mustNot(entry.getValue()); + } else { + boolQueryBuilder.must(entry.getValue()); + } + }); + return boolQueryBuilder; + } + + protected Stream> builderStreamFieldSort() { + return sort() + .asSplitStream(COMMA) + .flatMap( + item -> { + final var parts = item.split(COLON_OR_SPACE); + final var order = + SortOrder.fromString(attempt(() -> parts[1]).orElse((f) -> DEFAULT_SORT_ORDER)); + final var sortKey = toSortKey(parts[0]); + + return RELEVANCE_KEY_NAME.equalsIgnoreCase(sortKey.name()) + ? Stream.of(SortBuilders.scoreSort().order(order)) + : sortKey + .jsonPaths() + .map(path -> SortBuilders.fieldSort(path).order(order).missing(SORT_LAST)); + }); + } + + protected AggregationBuilder builderAggregationsWithFilter() { + var aggrFilter = AggregationBuilders.filter(POST_FILTER, filters().get()); + builderAggregations().forEach(aggrFilter::subAggregation); + return aggrFilter; + } + + protected SearchSourceBuilder builderDefaultSearchSource() { + var queryBuilder = + parameters().getSearchKeys().findAny().isEmpty() + ? QueryBuilders.matchAllQuery() + : builderMainQuery(); + + return new SearchSourceBuilder() + .query(queryBuilder) + .size(size().as()) + .from(from().as()) + .postFilter(filters().get()) + .trackTotalHits(true); + } + + /** + * Creates a multi match query, all words needs to be present, within a document. + * + * @return a MultiMatchQueryBuilder + */ + protected QueryBuilder builderSearchAllQuery(K searchAllKey) { + var fields = mapOfPathAndBoost(parameters().get(keyFields())); + var value = parameters().get(searchAllKey).toString(); + return QueryBuilders.multiMatchQuery(value) + .fields(fields) + .type(Type.CROSS_FIELDS) + .operator(Operator.AND); + } + + protected Map mapOfPathAndBoost(AsType fieldValue) { + return fieldValue.isEmpty() || fieldValue.asLowerCase().contains(ALL) + ? Map.of(ASTERISK, 1F) // NONE or ALL -> <'*',1.0> + : fieldValue + .asSplitStream(COMMA) + .map(this::toKey) + .flatMap(this::entryStreamOfPathAndBoost) + .collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + } + + private void handleAggregation( + SearchSourceBuilder builder, List contentWrappers) { + if (hasAggregation()) { + var aggregationBuilder = builder.shallowCopy(); + aggregationBuilder.size(ZERO_RESULTS_AGGREGATION_ONLY); + aggregationBuilder.aggregation(builderAggregationsWithFilter()); + contentWrappers.add( + new QueryContentWrapper(aggregationBuilder.toString(), this.openSearchUri())); + } + } + + private void handleFetchSource(SearchSourceBuilder builder) { + if (isFetchSource()) { + builder.fetchSource(include(), exclude()); + } else { + builder.fetchSource(true); + } + } + + private void handleSearchAfter(SearchSourceBuilder builder) { + var sortKeys = parameters().remove(keySearchAfter()).split(COMMA); + if (nonNull(sortKeys)) { + builder.searchAfter(sortKeys); + } + } + + private void handleSorting(SearchSourceBuilder builder) { + if (hasSortBy(RELEVANCE_KEY_NAME)) { + // Not very well documented. + // This allows sorting on relevance together with other fields. + builder.trackScores(true); + } + builderStreamFieldSort().forEach(builder::sort); + } + + private Stream> entryStreamOfPathAndBoost(K key) { + return key.searchFields(KEYWORD_FALSE).map(jsonPath -> entryOfPathAndBoost(key, jsonPath)); + } + + private Entry entryOfPathAndBoost(K key, String jsonPath) { + return Map.entry(jsonPath, key.fieldBoost()); + } + + private int numberOfRequests() { + return hasAggregation() ? 2 : 1; + } + + protected boolean hasSortBy(String sortKeyName) { + var sorts = sort().toString(); + return nonNull(sorts) && sorts.split(COMMA).length > 1 && sorts.contains(sortKeyName); + } + + private boolean isMustNot(K key) { + return NOT_ALL_OF.equals(key.searchOperator()) || NOT_ANY_OF.equals(key.searchOperator()); + } + + private boolean isFetchSource() { + return nonNull(exclude()) || nonNull(include()); + } + + private boolean hasAggregation() { + return getMediaType().is(JSON_UTF_8) + && ALL.equalsIgnoreCase(parameters().get(keyAggregation()).as()); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/AbstractBuilder.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/AbstractBuilder.java index 1e63635f8..d887ac96d 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/AbstractBuilder.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/AbstractBuilder.java @@ -5,17 +5,14 @@ import static no.unit.nva.search.common.enums.FieldOperator.BETWEEN; import static no.unit.nva.search.common.enums.FieldOperator.NOT_ANY_OF; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.enums.ParameterKey; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Abstract class for building OpenSearch queries. * @@ -33,59 +30,46 @@ */ public abstract class AbstractBuilder & ParameterKey> { - @JacocoGenerated - abstract Stream> buildMatchAnyValueQuery(K key, String... values); + @JacocoGenerated + abstract Stream> buildMatchAnyValueQuery(K key, String... values); - @JacocoGenerated - abstract Stream> buildMatchAllValuesQuery(K key, String... values); + @JacocoGenerated + abstract Stream> buildMatchAllValuesQuery(K key, String... values); - public Stream> buildQuery(K key, String value) { - final var values = splitAndFixMissingRangeValue(key, value); - return buildQuery(key, values); - } + public Stream> buildQuery(K key, String value) { + final var values = splitAndFixMissingRangeValue(key, value); + return buildQuery(key, values); + } - public Stream> buildQuery(K key, String... values) { - return isSearchAny(key) - ? buildMatchAnyValueQuery(key, values) - : buildMatchAllValuesQuery(key, values); - } + public Stream> buildQuery(K key, String... values) { + return isSearchAny(key) + ? buildMatchAnyValueQuery(key, values) + : buildMatchAllValuesQuery(key, values); + } - protected QueryBuilder getSubQuery(K key, String... values) { - return switch (key.fieldType()) { - case KEYWORD -> - new KeywordQuery() - .buildQuery(key, values) - .findFirst() - .orElseThrow() - .getValue(); - case FUZZY_KEYWORD -> - new FuzzyKeywordQuery() - .buildQuery(key, values) - .findFirst() - .orElseThrow() - .getValue(); - case TEXT -> - new TextQuery().buildQuery(key, values).findFirst().orElseThrow().getValue(); - case ACROSS_FIELDS -> - new AcrossFieldsQuery() - .buildQuery(key, values) - .findFirst() - .orElseThrow() - .getValue(); - case FREE_TEXT -> QueryBuilders.matchAllQuery(); - default -> throw new IllegalStateException("Unexpected value: " + key.fieldType()); - }; - } + protected QueryBuilder getSubQuery(K key, String... values) { + return switch (key.fieldType()) { + case KEYWORD -> + new KeywordQuery().buildQuery(key, values).findFirst().orElseThrow().getValue(); + case FUZZY_KEYWORD -> + new FuzzyKeywordQuery().buildQuery(key, values).findFirst().orElseThrow().getValue(); + case TEXT -> new TextQuery().buildQuery(key, values).findFirst().orElseThrow().getValue(); + case ACROSS_FIELDS -> + new AcrossFieldsQuery().buildQuery(key, values).findFirst().orElseThrow().getValue(); + case FREE_TEXT -> QueryBuilders.matchAllQuery(); + default -> throw new IllegalStateException("Unexpected value: " + key.fieldType()); + }; + } - private boolean isSearchAny(K key) { - return key.searchOperator().equals(ANY_OF) || key.searchOperator().equals(NOT_ANY_OF); - } + private boolean isSearchAny(K key) { + return key.searchOperator().equals(ANY_OF) || key.searchOperator().equals(NOT_ANY_OF); + } - private boolean isRangeMissingComma(K key, String value) { - return key.searchOperator().equals(BETWEEN) && !value.contains(COMMA); - } + private boolean isRangeMissingComma(K key, String value) { + return key.searchOperator().equals(BETWEEN) && !value.contains(COMMA); + } - private String[] splitAndFixMissingRangeValue(K key, String value) { - return isRangeMissingComma(key, value) ? new String[] {value, value} : value.split(COMMA); - } + private String[] splitAndFixMissingRangeValue(K key, String value) { + return isRangeMissingComma(key, value) ? new String[] {value, value} : value.split(COMMA); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/AcrossFieldsQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/AcrossFieldsQuery.java index 24a579284..682fab2fe 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/AcrossFieldsQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/AcrossFieldsQuery.java @@ -2,19 +2,17 @@ import static no.unit.nva.constants.Words.KEYWORD_FALSE; +import java.util.Arrays; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import java.util.Arrays; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search across multiple fields. * diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/ExistsQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/ExistsQuery.java index bee0c9ec7..8a20b671e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/ExistsQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/ExistsQuery.java @@ -2,21 +2,17 @@ import static no.unit.nva.constants.Words.KEYWORD_FALSE; import static no.unit.nva.search.common.constant.Functions.queryToEntry; - import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.existsQuery; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.enums.ParameterKey; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.ExistsQueryBuilder; import org.opensearch.index.query.QueryBuilder; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that check if a field exists. * @@ -26,38 +22,38 @@ */ public class ExistsQuery & ParameterKey> extends AbstractBuilder { - public static final String EXISTS_ANY = "ExistsAny-"; - - @Override - protected Stream> buildMatchAnyValueQuery(K key, String... values) { - return buildExistsQuery(key, Boolean.valueOf(values[0])); - } - - @Override - @JacocoGenerated // not currently in use... - protected Stream> buildMatchAllValuesQuery(K key, String... values) { - return buildExistsQuery(key, Boolean.valueOf(values[0])); - } - - private Stream> buildExistsQuery(K key, Boolean exists) { - var builder = boolQuery().queryName(EXISTS_ANY); - key.searchFields(KEYWORD_FALSE) - .map(fieldName -> createExistsQuery(key, fieldName)) - .forEach(existsQueryBuilder -> mustOrNot(exists, builder, existsQueryBuilder)); - return queryToEntry(key, builder); - } - - private ExistsQueryBuilder createExistsQuery(K key, String fieldName) { - return existsQuery(fieldName).boost(key.fieldBoost()); - } - - private void mustOrNot( - Boolean exists, BoolQueryBuilder boolQueryBuilder, QueryBuilder queryBuilder) { - if (exists) { - boolQueryBuilder.should(queryBuilder); - boolQueryBuilder.minimumShouldMatch(1); - } else { - boolQueryBuilder.mustNot(queryBuilder); - } + public static final String EXISTS_ANY = "ExistsAny-"; + + @Override + protected Stream> buildMatchAnyValueQuery(K key, String... values) { + return buildExistsQuery(key, Boolean.valueOf(values[0])); + } + + @Override + @JacocoGenerated // not currently in use... + protected Stream> buildMatchAllValuesQuery(K key, String... values) { + return buildExistsQuery(key, Boolean.valueOf(values[0])); + } + + private Stream> buildExistsQuery(K key, Boolean exists) { + var builder = boolQuery().queryName(EXISTS_ANY); + key.searchFields(KEYWORD_FALSE) + .map(fieldName -> createExistsQuery(key, fieldName)) + .forEach(existsQueryBuilder -> mustOrNot(exists, builder, existsQueryBuilder)); + return queryToEntry(key, builder); + } + + private ExistsQueryBuilder createExistsQuery(K key, String fieldName) { + return existsQuery(fieldName).boost(key.fieldBoost()); + } + + private void mustOrNot( + Boolean exists, BoolQueryBuilder boolQueryBuilder, QueryBuilder queryBuilder) { + if (exists) { + boolQueryBuilder.should(queryBuilder); + boolQueryBuilder.minimumShouldMatch(1); + } else { + boolQueryBuilder.mustNot(queryBuilder); } + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/FuzzyKeywordQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/FuzzyKeywordQuery.java index 0568767f1..7952e4c04 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/FuzzyKeywordQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/FuzzyKeywordQuery.java @@ -3,9 +3,11 @@ import static no.unit.nva.constants.Words.KEYWORD_FALSE; import static no.unit.nva.constants.Words.KEYWORD_TRUE; +import java.util.Arrays; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import org.opensearch.common.unit.Fuzziness; import org.opensearch.index.query.DisMaxQueryBuilder; import org.opensearch.index.query.Operator; @@ -14,10 +16,6 @@ import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; -import java.util.Arrays; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for keywords with fuzzy matching. * diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/HasPartsQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/HasPartsQuery.java index 830e0bf45..23d7f032b 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/HasPartsQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/HasPartsQuery.java @@ -2,18 +2,15 @@ import static no.unit.nva.constants.Words.PART_OF; +import java.util.Map; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import nva.commons.core.JacocoGenerated; - import org.apache.lucene.search.join.ScoreMode; import org.opensearch.index.query.QueryBuilder; import org.opensearch.join.query.HasChildQueryBuilder; -import java.util.Map; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for parts of a document. * @@ -22,23 +19,20 @@ */ public class HasPartsQuery & ParameterKey> extends AbstractBuilder { - @Override - @JacocoGenerated - protected Stream> buildMatchAnyValueQuery(K key, String... values) { - return buildHasChildQuery(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } - - @Override - protected Stream> buildMatchAllValuesQuery(K key, String... values) { - return buildHasChildQuery(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } - - private Stream buildHasChildQuery(K key, String... values) { - var builder = - new HasChildQueryBuilder( - PART_OF, getSubQuery(key.subQuery(), values), ScoreMode.None); - return Stream.of(builder); - } + @Override + @JacocoGenerated + protected Stream> buildMatchAnyValueQuery(K key, String... values) { + return buildHasChildQuery(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); + } + + @Override + protected Stream> buildMatchAllValuesQuery(K key, String... values) { + return buildHasChildQuery(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); + } + + private Stream buildHasChildQuery(K key, String... values) { + var builder = + new HasChildQueryBuilder(PART_OF, getSubQuery(key.subQuery(), values), ScoreMode.None); + return Stream.of(builder); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/KeywordQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/KeywordQuery.java index 98b1e8dc9..0252d921d 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/KeywordQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/KeywordQuery.java @@ -2,19 +2,17 @@ import static no.unit.nva.constants.Words.KEYWORD_TRUE; +import java.util.Arrays; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import org.opensearch.index.query.DisMaxQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermQueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; -import java.util.Arrays; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for keywords. * @@ -24,45 +22,38 @@ */ public class KeywordQuery & ParameterKey> extends AbstractBuilder { - public static final String KEYWORD_ALL = "KeywordAll-"; - public static final String KEYWORD_ANY = "KeywordAny-"; - - private static & ParameterKey> TermQueryBuilder getTermQueryBuilder( - K key, String value, String searchField) { - return new TermQueryBuilder(searchField, value).queryName(KEYWORD_ALL + key.asCamelCase()); - } - - @Override - protected Stream> buildMatchAnyValueQuery(K key, String... values) { - return buildMatchAnyKeywordStream(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } - - @Override - protected Stream> buildMatchAllValuesQuery(K key, String... values) { - return buildMatchAllKeywordStream(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } - - private Stream buildMatchAllKeywordStream(K key, String... values) { - return Arrays.stream(values) - .flatMap( - value -> - key.searchFields(KEYWORD_TRUE) - .map( - searchField -> - getTermQueryBuilder( - key, value, searchField))); - } - - private Stream buildMatchAnyKeywordStream(K key, String... values) { - var disMax = QueryBuilders.disMaxQuery().queryName(KEYWORD_ANY + key.asCamelCase()); - key.searchFields(KEYWORD_TRUE) - .forEach( - field -> - disMax.add( - new TermsQueryBuilder(field, values) - .boost(key.fieldBoost()))); - return Stream.of(disMax); - } + public static final String KEYWORD_ALL = "KeywordAll-"; + public static final String KEYWORD_ANY = "KeywordAny-"; + + private static & ParameterKey> TermQueryBuilder getTermQueryBuilder( + K key, String value, String searchField) { + return new TermQueryBuilder(searchField, value).queryName(KEYWORD_ALL + key.asCamelCase()); + } + + @Override + protected Stream> buildMatchAnyValueQuery(K key, String... values) { + return buildMatchAnyKeywordStream(key, values) + .flatMap(builder -> Functions.queryToEntry(key, builder)); + } + + @Override + protected Stream> buildMatchAllValuesQuery(K key, String... values) { + return buildMatchAllKeywordStream(key, values) + .flatMap(builder -> Functions.queryToEntry(key, builder)); + } + + private Stream buildMatchAllKeywordStream(K key, String... values) { + return Arrays.stream(values) + .flatMap( + value -> + key.searchFields(KEYWORD_TRUE) + .map(searchField -> getTermQueryBuilder(key, value, searchField))); + } + + private Stream buildMatchAnyKeywordStream(K key, String... values) { + var disMax = QueryBuilders.disMaxQuery().queryName(KEYWORD_ANY + key.asCamelCase()); + key.searchFields(KEYWORD_TRUE) + .forEach(field -> disMax.add(new TermsQueryBuilder(field, values).boost(key.fieldBoost()))); + return Stream.of(disMax); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/PartOfQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/PartOfQuery.java index f1330444f..bb2344a23 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/PartOfQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/PartOfQuery.java @@ -2,15 +2,13 @@ import static no.unit.nva.constants.Words.HAS_PARTS; +import java.util.Map; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import org.opensearch.index.query.QueryBuilder; import org.opensearch.join.query.HasParentQueryBuilder; -import java.util.Map; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for parts of a document. * @@ -19,19 +17,18 @@ */ public class PartOfQuery & ParameterKey> extends AbstractBuilder { - @Override - Stream> buildMatchAnyValueQuery(K key, String... values) { - return buildHasParent(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); - } + @Override + Stream> buildMatchAnyValueQuery(K key, String... values) { + return buildHasParent(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); + } - @Override - Stream> buildMatchAllValuesQuery(K key, String... values) { - return buildHasParent(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); - } + @Override + Stream> buildMatchAllValuesQuery(K key, String... values) { + return buildHasParent(key, values).flatMap(builder -> Functions.queryToEntry(key, builder)); + } - private Stream buildHasParent(K key, String... values) { - var builder = - new HasParentQueryBuilder(HAS_PARTS, getSubQuery(key.subQuery(), values), true); - return Stream.of(builder); - } + private Stream buildHasParent(K key, String... values) { + var builder = new HasParentQueryBuilder(HAS_PARTS, getSubQuery(key.subQuery(), values), true); + return Stream.of(builder); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/RangeQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/RangeQuery.java index f01ac71ea..f89eb104e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/RangeQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/RangeQuery.java @@ -2,20 +2,17 @@ import static no.unit.nva.constants.ErrorMessages.OPERATOR_NOT_SUPPORTED; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; import no.unit.nva.search.common.enums.ParameterKind; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for a range of values. * @@ -25,100 +22,94 @@ */ public class RangeQuery & ParameterKey> extends AbstractBuilder { - public static final String GREATER_OR_EQUAL = "GreaterOrEqual-"; - public static final String LESS_THAN = "LessThan-"; - public static final String BETWEEN = "Between-"; - public static final String DASH = " - "; - public static final int WORD_LENGTH_YEAR = 4; - public static final int WORD_LENGTH_YEAR_MONTH = 7; - public static final String STR_01 = "-01"; - public static final String STR_12 = "-12"; - public static final String STR_31 = "-31"; - private static final Logger logger = LoggerFactory.getLogger(RangeQuery.class); - - @JacocoGenerated // never used - @Override - protected Stream> buildMatchAnyValueQuery(K key, String... values) { - return queryAsEntryStream(key, values); + public static final String GREATER_OR_EQUAL = "GreaterOrEqual-"; + public static final String LESS_THAN = "LessThan-"; + public static final String BETWEEN = "Between-"; + public static final String DASH = " - "; + public static final int WORD_LENGTH_YEAR = 4; + public static final int WORD_LENGTH_YEAR_MONTH = 7; + public static final String STR_01 = "-01"; + public static final String STR_12 = "-12"; + public static final String STR_31 = "-31"; + private static final Logger logger = LoggerFactory.getLogger(RangeQuery.class); + + @JacocoGenerated // never used + @Override + protected Stream> buildMatchAnyValueQuery(K key, String... values) { + return queryAsEntryStream(key, values); + } + + @Override + protected Stream> buildMatchAllValuesQuery(K key, String... values) { + return queryAsEntryStream(key, values); + } + + protected Stream> queryAsEntryStream(K key, String... values) { + final var searchField = key.searchFields().findFirst().orElseThrow(); + var firstParam = getFirstParam(values, key); + var secondParam = getSecondParam(values, key); + + logger.debug(firstParam + DASH + secondParam); + return Functions.queryToEntry( + key, + switch (key.searchOperator()) { + case GREATER_THAN_OR_EQUAL_TO -> + QueryBuilders.rangeQuery(searchField) + .gte(firstParam) + .queryName(GREATER_OR_EQUAL + key.asCamelCase()); + case LESS_THAN -> + QueryBuilders.rangeQuery(searchField) + .lt(firstParam) + .queryName(LESS_THAN + key.asCamelCase()); + case BETWEEN -> + QueryBuilders.rangeQuery(searchField) + .from(firstParam, true) + .to(secondParam, true) + .queryName(BETWEEN + key.asCamelCase()); + default -> throw new IllegalArgumentException(OPERATOR_NOT_SUPPORTED); + }); + } + + private String getSecondParam(String[] values, K key) { + return values.length == 1 + ? null + : valueOrNull(values[1]).map(date -> expandDateLast(date, key)).findFirst().orElse(null); + } + + private String getFirstParam(String[] values, K key) { + return valueOrNull(values[0]).map(date -> expandDateFirst(date, key)).findFirst().orElse(null); + } + + @SuppressWarnings("PMD.NullAssignment") + private Stream valueOrNull(String value) { + return Stream.ofNullable(value.isBlank() ? null : value); + } + + private String expandDateFirst(String date, K key) { + if (key.fieldType() != ParameterKind.DATE) { + return date; } - - @Override - protected Stream> buildMatchAllValuesQuery(K key, String... values) { - return queryAsEntryStream(key, values); + var expandedDate = new StringBuilder(date); + if (WORD_LENGTH_YEAR == expandedDate.length()) { + expandedDate.append(STR_01); } - - protected Stream> queryAsEntryStream(K key, String... values) { - final var searchField = key.searchFields().findFirst().orElseThrow(); - var firstParam = getFirstParam(values, key); - var secondParam = getSecondParam(values, key); - - logger.debug(firstParam + DASH + secondParam); - return Functions.queryToEntry( - key, - switch (key.searchOperator()) { - case GREATER_THAN_OR_EQUAL_TO -> - QueryBuilders.rangeQuery(searchField) - .gte(firstParam) - .queryName(GREATER_OR_EQUAL + key.asCamelCase()); - case LESS_THAN -> - QueryBuilders.rangeQuery(searchField) - .lt(firstParam) - .queryName(LESS_THAN + key.asCamelCase()); - case BETWEEN -> - QueryBuilders.rangeQuery(searchField) - .from(firstParam, true) - .to(secondParam, true) - .queryName(BETWEEN + key.asCamelCase()); - default -> throw new IllegalArgumentException(OPERATOR_NOT_SUPPORTED); - }); - } - - private String getSecondParam(String[] values, K key) { - return values.length == 1 - ? null - : valueOrNull(values[1]) - .map(date -> expandDateLast(date, key)) - .findFirst() - .orElse(null); + if (WORD_LENGTH_YEAR_MONTH == expandedDate.length()) { + expandedDate.append(STR_01); } + return expandedDate.toString(); + } - private String getFirstParam(String[] values, K key) { - return valueOrNull(values[0]) - .map(date -> expandDateFirst(date, key)) - .findFirst() - .orElse(null); + private String expandDateLast(String date, K key) { + if (key.fieldType() != ParameterKind.DATE) { + return date; } - - @SuppressWarnings("PMD.NullAssignment") - private Stream valueOrNull(String value) { - return Stream.ofNullable(value.isBlank() ? null : value); - } - - private String expandDateFirst(String date, K key) { - if (key.fieldType() != ParameterKind.DATE) { - return date; - } - var expandedDate = new StringBuilder(date); - if (WORD_LENGTH_YEAR == expandedDate.length()) { - expandedDate.append(STR_01); - } - if (WORD_LENGTH_YEAR_MONTH == expandedDate.length()) { - expandedDate.append(STR_01); - } - return expandedDate.toString(); + var expandedDate = new StringBuilder(date); + if (WORD_LENGTH_YEAR == expandedDate.length()) { + expandedDate.append(STR_12); } - - private String expandDateLast(String date, K key) { - if (key.fieldType() != ParameterKind.DATE) { - return date; - } - var expandedDate = new StringBuilder(date); - if (WORD_LENGTH_YEAR == expandedDate.length()) { - expandedDate.append(STR_12); - } - if (WORD_LENGTH_YEAR_MONTH == expandedDate.length()) { - expandedDate.append(STR_31); - } - return expandedDate.toString(); + if (WORD_LENGTH_YEAR_MONTH == expandedDate.length()) { + expandedDate.append(STR_31); } + return expandedDate.toString(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/builder/TextQuery.java b/search-commons/src/main/java/no/unit/nva/search/common/builder/TextQuery.java index 8cb360c8a..2a9a20739 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/builder/TextQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/builder/TextQuery.java @@ -1,13 +1,14 @@ package no.unit.nva.search.common.builder; import static no.unit.nva.constants.Words.KEYWORD_FALSE; - import static org.opensearch.index.query.QueryBuilders.matchPhrasePrefixQuery; import static org.opensearch.index.query.QueryBuilders.matchQuery; +import java.util.Arrays; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.ParameterKey; - import org.opensearch.index.query.DisMaxQueryBuilder; import org.opensearch.index.query.MatchPhrasePrefixQueryBuilder; import org.opensearch.index.query.MatchQueryBuilder; @@ -15,10 +16,6 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; -import java.util.Arrays; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search for text. * @@ -28,55 +25,53 @@ */ public class TextQuery & ParameterKey> extends AbstractBuilder { - private static final String TEXT_ALL = "TextAll-"; - private static final String TEXT_ANY = "TextAny-"; + private static final String TEXT_ALL = "TextAll-"; + private static final String TEXT_ANY = "TextAny-"; - @Override - protected Stream> buildMatchAnyValueQuery(K key, String... values) { - return buildAnyComboMustHitQuery(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } + @Override + protected Stream> buildMatchAnyValueQuery(K key, String... values) { + return buildAnyComboMustHitQuery(key, values) + .flatMap(builder -> Functions.queryToEntry(key, builder)); + } - @Override - protected Stream> buildMatchAllValuesQuery(K key, String... values) { - return buildAllMustHitQuery(key, values) - .flatMap(builder -> Functions.queryToEntry(key, builder)); - } + @Override + protected Stream> buildMatchAllValuesQuery(K key, String... values) { + return buildAllMustHitQuery(key, values) + .flatMap(builder -> Functions.queryToEntry(key, builder)); + } - private Stream buildAllMustHitQuery(K key, String... values) { - return Arrays.stream(values) - .map( - singleValue -> - phrasePrefixBuilder(singleValue, key) - .collect( - DisMaxQueryBuilder::new, - DisMaxQueryBuilder::add, - DisMaxQueryBuilder::add) - .queryName(TEXT_ALL + key.asCamelCase())); - } + private Stream buildAllMustHitQuery(K key, String... values) { + return Arrays.stream(values) + .map( + singleValue -> + phrasePrefixBuilder(singleValue, key) + .collect( + DisMaxQueryBuilder::new, DisMaxQueryBuilder::add, DisMaxQueryBuilder::add) + .queryName(TEXT_ALL + key.asCamelCase())); + } - private Stream buildAnyComboMustHitQuery(K key, String... values) { - var disMax = QueryBuilders.disMaxQuery().queryName(TEXT_ANY + key.asCamelCase()); - Arrays.stream(values) - .flatMap(singleValue -> phrasePrefixBuilder(singleValue, key)) - .forEach(disMax::add); - return Stream.of(disMax); - } + private Stream buildAnyComboMustHitQuery(K key, String... values) { + var disMax = QueryBuilders.disMaxQuery().queryName(TEXT_ANY + key.asCamelCase()); + Arrays.stream(values) + .flatMap(singleValue -> phrasePrefixBuilder(singleValue, key)) + .forEach(disMax::add); + return Stream.of(disMax); + } - private Stream phrasePrefixBuilder(String singleValue, K key) { - return Stream.concat( - key.searchFields(KEYWORD_FALSE) - .map(fieldName -> matchPhrasePrefixBuilder(singleValue, key, fieldName)), - key.searchFields(KEYWORD_FALSE) - .map(fieldName -> matchQueryBuilder(singleValue, key, fieldName))); - } + private Stream phrasePrefixBuilder(String singleValue, K key) { + return Stream.concat( + key.searchFields(KEYWORD_FALSE) + .map(fieldName -> matchPhrasePrefixBuilder(singleValue, key, fieldName)), + key.searchFields(KEYWORD_FALSE) + .map(fieldName -> matchQueryBuilder(singleValue, key, fieldName))); + } - private MatchQueryBuilder matchQueryBuilder(String singleValue, K key, String fieldName) { - return matchQuery(fieldName, singleValue).operator(Operator.AND).boost(key.fieldBoost()); - } + private MatchQueryBuilder matchQueryBuilder(String singleValue, K key, String fieldName) { + return matchQuery(fieldName, singleValue).operator(Operator.AND).boost(key.fieldBoost()); + } - private MatchPhrasePrefixQueryBuilder matchPhrasePrefixBuilder( - String singleValue, K key, String fieldName) { - return matchPhrasePrefixQuery(fieldName, singleValue).boost(key.fieldBoost() + 0.1F); - } + private MatchPhrasePrefixQueryBuilder matchPhrasePrefixBuilder( + String singleValue, K key, String fieldName) { + return matchPhrasePrefixQuery(fieldName, singleValue).boost(key.fieldBoost() + 0.1F); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/constant/Functions.java b/search-commons/src/main/java/no/unit/nva/search/common/constant/Functions.java index e2dd71f71..5d2801202 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/constant/Functions.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/constant/Functions.java @@ -1,5 +1,6 @@ package no.unit.nva.search.common.constant; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Defaults.ENVIRONMENT; import static no.unit.nva.constants.Words.BOKMAAL_CODE; import static no.unit.nva.constants.Words.COLON; @@ -14,16 +15,19 @@ import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_ASC_DESC_VALUE; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_ASC_OR_DESC_GROUP; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_SELECTED_GROUP; - import static nva.commons.core.StringUtils.SPACE; -import static java.util.Objects.nonNull; - +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collection; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.constants.Defaults; import no.unit.nva.search.common.enums.ParameterKey; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.AggregationBuilders; @@ -31,15 +35,6 @@ import org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Class for building OpenSearch queries that search across multiple fields. * diff --git a/search-commons/src/main/java/no/unit/nva/search/common/csv/ExportCsv.java b/search-commons/src/main/java/no/unit/nva/search/common/csv/ExportCsv.java index aaef32c7a..3329d5151 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/csv/ExportCsv.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/csv/ExportCsv.java @@ -1,12 +1,10 @@ package no.unit.nva.search.common.csv; import com.opencsv.bean.CsvBindByName; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.StringUtils; - import java.util.List; import java.util.Objects; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.StringUtils; /** * Class for exporting data to CSV. diff --git a/search-commons/src/main/java/no/unit/nva/search/common/csv/HeaderColumnNameAndOrderMappingStrategy.java b/search-commons/src/main/java/no/unit/nva/search/common/csv/HeaderColumnNameAndOrderMappingStrategy.java index 3aa4b9ef0..62d25d10e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/csv/HeaderColumnNameAndOrderMappingStrategy.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/csv/HeaderColumnNameAndOrderMappingStrategy.java @@ -5,16 +5,14 @@ import com.opencsv.bean.CsvCustomBindByName; import com.opencsv.bean.HeaderColumnNameMappingStrategy; import com.opencsv.exceptions.CsvRequiredFieldEmptyException; - +import java.util.Arrays; +import java.util.stream.Collectors; import org.apache.commons.collections4.comparators.ComparableComparator; import org.apache.commons.collections4.comparators.ComparatorChain; import org.apache.commons.collections4.comparators.FixedOrderComparator; import org.apache.commons.collections4.comparators.NullComparator; import org.apache.commons.lang3.StringUtils; -import java.util.Arrays; -import java.util.stream.Collectors; - /** * Class for exporting data to CSV. * @@ -22,92 +20,85 @@ * @param The type of the bean to be exported. */ public class HeaderColumnNameAndOrderMappingStrategy extends HeaderColumnNameMappingStrategy { - public HeaderColumnNameAndOrderMappingStrategy() { - super(); - } + public HeaderColumnNameAndOrderMappingStrategy() { + super(); + } - public static HeaderColumnNameAndOrderMappingStrategy - headerColumnNameAndOrderMappingStrategyWithType(Class type) { - var strategy = new HeaderColumnNameAndOrderMappingStrategy<>(); - strategy.setType(type); - return strategy; - } + public static HeaderColumnNameAndOrderMappingStrategy + headerColumnNameAndOrderMappingStrategyWithType(Class type) { + var strategy = new HeaderColumnNameAndOrderMappingStrategy<>(); + strategy.setType(type); + return strategy; + } - /** - * This maintains case of header strings. - * - * @param bean One fully populated bean from which the header can be derived. This is important - * in the face of joining and splitting. If we have a MultiValuedMap as a field that is the - * target for a join on reading, that same field must be split into multiple columns on - * writing. Since the joining is done via regular expressions, it is impossible for opencsv - * to know what the column names are supposed to be on writing unless this bean includes a - * fully populated map. - * @return Array of header Strings maintaining case. - * @throws CsvRequiredFieldEmptyException in case any field that is marked "required" is empty. - */ - @Override - public String[] generateHeader(T bean) throws CsvRequiredFieldEmptyException { - String[] header = super.generateHeader(bean); - final int numColumns = headerIndex.findMaxIndex(); - if (numColumns == -1) { - return header; - } + /** + * This maintains case of header strings. + * + * @param bean One fully populated bean from which the header can be derived. This is important in + * the face of joining and splitting. If we have a MultiValuedMap as a field that is the + * target for a join on reading, that same field must be split into multiple columns on + * writing. Since the joining is done via regular expressions, it is impossible for opencsv to + * know what the column names are supposed to be on writing unless this bean includes a fully + * populated map. + * @return Array of header Strings maintaining case. + * @throws CsvRequiredFieldEmptyException in case any field that is marked "required" is empty. + */ + @Override + public String[] generateHeader(T bean) throws CsvRequiredFieldEmptyException { + String[] header = super.generateHeader(bean); + final int numColumns = headerIndex.findMaxIndex(); + if (numColumns == -1) { + return header; + } - header = new String[numColumns + 1]; + header = new String[numColumns + 1]; - BeanField beanField; - for (int i = 0; i <= numColumns; i++) { - beanField = findField(i); - String columnHeaderName = extractHeaderName(beanField); - header[i] = columnHeaderName; - } - return header; + BeanField beanField; + for (int i = 0; i <= numColumns; i++) { + beanField = findField(i); + String columnHeaderName = extractHeaderName(beanField); + header[i] = columnHeaderName; } + return header; + } - /** - * This override allows setting of the column order based on annotation {@link - * CsvBindByNameOrder} . - */ - @Override - protected void loadFieldMap() { - if (writeOrder == null && type.isAnnotationPresent(CsvBindByNameOrder.class)) { - var predefinedList = - Arrays.stream(type.getAnnotation(CsvBindByNameOrder.class).value()) - .map(String::toUpperCase) - .collect(Collectors.toList()); - var fixedComparator = new FixedOrderComparator<>(predefinedList); - fixedComparator.setUnknownObjectBehavior( - FixedOrderComparator.UnknownObjectBehavior.AFTER); - var comparator = - new ComparatorChain<>( - Arrays.asList( - fixedComparator, - new NullComparator<>(false), - new ComparableComparator<>())); - setColumnOrderOnWrite(comparator); - } - super.loadFieldMap(); + /** + * This override allows setting of the column order based on annotation {@link CsvBindByNameOrder} + * . + */ + @Override + protected void loadFieldMap() { + if (writeOrder == null && type.isAnnotationPresent(CsvBindByNameOrder.class)) { + var predefinedList = + Arrays.stream(type.getAnnotation(CsvBindByNameOrder.class).value()) + .map(String::toUpperCase) + .collect(Collectors.toList()); + var fixedComparator = new FixedOrderComparator<>(predefinedList); + fixedComparator.setUnknownObjectBehavior(FixedOrderComparator.UnknownObjectBehavior.AFTER); + var comparator = + new ComparatorChain<>( + Arrays.asList( + fixedComparator, new NullComparator<>(false), new ComparableComparator<>())); + setColumnOrderOnWrite(comparator); } + super.loadFieldMap(); + } - private String extractHeaderName(final BeanField beanField) { - if (beanField == null - || beanField.getField() == null - || beanField.getField().getDeclaredAnnotationsByType(CsvBindByName.class).length - == 0) { - return StringUtils.EMPTY; - } + private String extractHeaderName(final BeanField beanField) { + if (beanField == null + || beanField.getField() == null + || beanField.getField().getDeclaredAnnotationsByType(CsvBindByName.class).length == 0) { + return StringUtils.EMPTY; + } - if (beanField.getField().isAnnotationPresent(CsvBindByName.class)) { - return beanField - .getField() - .getDeclaredAnnotationsByType(CsvBindByName.class)[0] - .column(); - } else if (beanField.getField().isAnnotationPresent(CsvCustomBindByName.class)) { - return beanField - .getField() - .getDeclaredAnnotationsByType(CsvCustomBindByName.class)[0] - .column(); - } - return StringUtils.EMPTY; + if (beanField.getField().isAnnotationPresent(CsvBindByName.class)) { + return beanField.getField().getDeclaredAnnotationsByType(CsvBindByName.class)[0].column(); + } else if (beanField.getField().isAnnotationPresent(CsvCustomBindByName.class)) { + return beanField + .getField() + .getDeclaredAnnotationsByType(CsvCustomBindByName.class)[0] + .column(); } + return StringUtils.EMPTY; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/csv/ResourceCsvTransformer.java b/search-commons/src/main/java/no/unit/nva/search/common/csv/ResourceCsvTransformer.java index a5103d976..2385a799b 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/csv/ResourceCsvTransformer.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/csv/ResourceCsvTransformer.java @@ -6,7 +6,6 @@ import com.opencsv.bean.StatefulBeanToCsvBuilder; import com.opencsv.exceptions.CsvDataTypeMismatchException; import com.opencsv.exceptions.CsvRequiredFieldEmptyException; - import java.io.StringWriter; import java.util.List; import java.util.stream.Collectors; @@ -22,130 +21,123 @@ */ public final class ResourceCsvTransformer { - private static final String IDENTITY_NAME_JSON_POINTER = "/identity/name"; - private static final String IDENTITY_NAME_SOURCE_POINTER = - "entityDescription.contributors.identity.name"; - private static final String CONTRIBUTORS_JSON_POINTER = "/entityDescription/contributors"; - private static final String ID_JSON_POINTER = "/id"; - private static final String ID_SOURCE_POINTER = "id"; - private static final String MAIN_TITLE_JSON_POINTER = "/entityDescription/mainTitle"; - private static final String MAIN_TITLE_SOURCE_POINTER = "entityDescription.mainTitle"; - private static final String PUBLICATION_DATE_YEAR_JSON_POINTER = - "/entityDescription/publicationDate/year"; - private static final String PUBLICATION_DATE_YEAR_SOURCE_POINTER = - "entityDescription.publicationDate.year"; - private static final String PUBLICATION_DATE_MONTH_JSON_POINTER = - "/entityDescription/publicationDate/month"; - private static final String PUBLICATION_DATE_MONTH_SOURCE_POINTER = - "entityDescription.publicationDate.month"; - private static final String PUBLICATION_DATE_DAY_JSON_POINTER = - "/entityDescription/publicationDate/day"; - private static final String PUBLICATION_DATE_DAY_SOURCE_POINTER = - "entityDescription.publicationDate.day"; - private static final String PUBLICATION_INSTANCE_TYPE_JSON_POINTER = - "/entityDescription/reference/publicationInstance/type"; - private static final String PUBLICATION_INSTANCE_TYPE_SOURCE_POINTER = - "entityDescription.reference.publicationInstance.type"; - private static final String EMPTY_STRING = ""; - private static final char UTF8_BOM = '\ufeff'; - private static final char QUOTE_CHAR = '"'; - private static final char SEPARATOR = ';'; - private static final String LINE_END = "\r\n"; - - private ResourceCsvTransformer() {} - - public static String transform(List hits) { - var stringWriter = new StringWriter(); - stringWriter.append(UTF8_BOM); - var lines = extractedJsonSearchResults(hits); - var csvWriter = - new StatefulBeanToCsvBuilder(stringWriter) - .withApplyQuotesToAll(true) - .withQuotechar(QUOTE_CHAR) - .withSeparator(SEPARATOR) - .withLineEnd(LINE_END) - .withMappingStrategy( - headerColumnNameAndOrderMappingStrategyWithType(ExportCsv.class)) - .build(); - try { - csvWriter.write(lines); - } catch (CsvDataTypeMismatchException | CsvRequiredFieldEmptyException e) { - throw new RuntimeException(e); - } - return stringWriter.toString(); - } - - public static List getJsonFields() { - return List.of( - ID_SOURCE_POINTER, - MAIN_TITLE_SOURCE_POINTER, - PUBLICATION_DATE_YEAR_SOURCE_POINTER, - PUBLICATION_DATE_MONTH_SOURCE_POINTER, - PUBLICATION_DATE_DAY_SOURCE_POINTER, - PUBLICATION_INSTANCE_TYPE_SOURCE_POINTER, - IDENTITY_NAME_SOURCE_POINTER); - } - - private static List extractedJsonSearchResults(List searchResults) { - return searchResults.stream() - .map(ResourceCsvTransformer::createLine) - .collect(Collectors.toList()); - } - - private static ExportCsv createLine(JsonNode searchResult) { - var mainTitle = getMainTitle(searchResult); - var year = extractYear(searchResult); - var month = extractMonth(searchResult); - var day = extractDay(searchResult); - var publicationInstance = extractPublicationInstance(searchResult); - var contributors = getContributorsName(searchResult); - return new ExportCsv( - extractId(searchResult), - mainTitle, - year, - month, - day, - publicationInstance, - contributors); - } - - private static String extractPublicationInstance(JsonNode searchResult) { - return extractText(searchResult, PUBLICATION_INSTANCE_TYPE_JSON_POINTER, EMPTY_STRING); - } - - private static String extractDay(JsonNode searchResult) { - return extractText(searchResult, PUBLICATION_DATE_DAY_JSON_POINTER, EMPTY_STRING); - } - - private static String extractMonth(JsonNode searchResult) { - return extractText(searchResult, PUBLICATION_DATE_MONTH_JSON_POINTER, EMPTY_STRING); - } - - private static String extractYear(JsonNode searchResult) { - return extractText(searchResult, PUBLICATION_DATE_YEAR_JSON_POINTER, EMPTY_STRING); - } - - private static String getMainTitle(JsonNode searchResult) { - return extractText(searchResult, MAIN_TITLE_JSON_POINTER, EMPTY_STRING); - } - - private static String extractId(JsonNode searchResult) { - return extractText(searchResult, ID_JSON_POINTER, EMPTY_STRING); - } - - private static List getContributorsName(JsonNode document) { - var contributors = document.at(CONTRIBUTORS_JSON_POINTER); - return StreamSupport.stream(contributors.spliterator(), false) - .map(ResourceCsvTransformer::extractName) - .collect(Collectors.toList()); - } - - private static String extractName(JsonNode contributor) { - return extractText(contributor, IDENTITY_NAME_JSON_POINTER, null); - } - - private static String extractText(JsonNode node, String pointer, String defaultValue) { - var value = node.at(pointer); - return value.isMissingNode() ? defaultValue : value.asText(); + private static final String IDENTITY_NAME_JSON_POINTER = "/identity/name"; + private static final String IDENTITY_NAME_SOURCE_POINTER = + "entityDescription.contributors.identity.name"; + private static final String CONTRIBUTORS_JSON_POINTER = "/entityDescription/contributors"; + private static final String ID_JSON_POINTER = "/id"; + private static final String ID_SOURCE_POINTER = "id"; + private static final String MAIN_TITLE_JSON_POINTER = "/entityDescription/mainTitle"; + private static final String MAIN_TITLE_SOURCE_POINTER = "entityDescription.mainTitle"; + private static final String PUBLICATION_DATE_YEAR_JSON_POINTER = + "/entityDescription/publicationDate/year"; + private static final String PUBLICATION_DATE_YEAR_SOURCE_POINTER = + "entityDescription.publicationDate.year"; + private static final String PUBLICATION_DATE_MONTH_JSON_POINTER = + "/entityDescription/publicationDate/month"; + private static final String PUBLICATION_DATE_MONTH_SOURCE_POINTER = + "entityDescription.publicationDate.month"; + private static final String PUBLICATION_DATE_DAY_JSON_POINTER = + "/entityDescription/publicationDate/day"; + private static final String PUBLICATION_DATE_DAY_SOURCE_POINTER = + "entityDescription.publicationDate.day"; + private static final String PUBLICATION_INSTANCE_TYPE_JSON_POINTER = + "/entityDescription/reference/publicationInstance/type"; + private static final String PUBLICATION_INSTANCE_TYPE_SOURCE_POINTER = + "entityDescription.reference.publicationInstance.type"; + private static final String EMPTY_STRING = ""; + private static final char UTF8_BOM = '\ufeff'; + private static final char QUOTE_CHAR = '"'; + private static final char SEPARATOR = ';'; + private static final String LINE_END = "\r\n"; + + private ResourceCsvTransformer() {} + + public static String transform(List hits) { + var stringWriter = new StringWriter(); + stringWriter.append(UTF8_BOM); + var lines = extractedJsonSearchResults(hits); + var csvWriter = + new StatefulBeanToCsvBuilder(stringWriter) + .withApplyQuotesToAll(true) + .withQuotechar(QUOTE_CHAR) + .withSeparator(SEPARATOR) + .withLineEnd(LINE_END) + .withMappingStrategy(headerColumnNameAndOrderMappingStrategyWithType(ExportCsv.class)) + .build(); + try { + csvWriter.write(lines); + } catch (CsvDataTypeMismatchException | CsvRequiredFieldEmptyException e) { + throw new RuntimeException(e); } + return stringWriter.toString(); + } + + public static List getJsonFields() { + return List.of( + ID_SOURCE_POINTER, + MAIN_TITLE_SOURCE_POINTER, + PUBLICATION_DATE_YEAR_SOURCE_POINTER, + PUBLICATION_DATE_MONTH_SOURCE_POINTER, + PUBLICATION_DATE_DAY_SOURCE_POINTER, + PUBLICATION_INSTANCE_TYPE_SOURCE_POINTER, + IDENTITY_NAME_SOURCE_POINTER); + } + + private static List extractedJsonSearchResults(List searchResults) { + return searchResults.stream() + .map(ResourceCsvTransformer::createLine) + .collect(Collectors.toList()); + } + + private static ExportCsv createLine(JsonNode searchResult) { + var mainTitle = getMainTitle(searchResult); + var year = extractYear(searchResult); + var month = extractMonth(searchResult); + var day = extractDay(searchResult); + var publicationInstance = extractPublicationInstance(searchResult); + var contributors = getContributorsName(searchResult); + return new ExportCsv( + extractId(searchResult), mainTitle, year, month, day, publicationInstance, contributors); + } + + private static String extractPublicationInstance(JsonNode searchResult) { + return extractText(searchResult, PUBLICATION_INSTANCE_TYPE_JSON_POINTER, EMPTY_STRING); + } + + private static String extractDay(JsonNode searchResult) { + return extractText(searchResult, PUBLICATION_DATE_DAY_JSON_POINTER, EMPTY_STRING); + } + + private static String extractMonth(JsonNode searchResult) { + return extractText(searchResult, PUBLICATION_DATE_MONTH_JSON_POINTER, EMPTY_STRING); + } + + private static String extractYear(JsonNode searchResult) { + return extractText(searchResult, PUBLICATION_DATE_YEAR_JSON_POINTER, EMPTY_STRING); + } + + private static String getMainTitle(JsonNode searchResult) { + return extractText(searchResult, MAIN_TITLE_JSON_POINTER, EMPTY_STRING); + } + + private static String extractId(JsonNode searchResult) { + return extractText(searchResult, ID_JSON_POINTER, EMPTY_STRING); + } + + private static List getContributorsName(JsonNode document) { + var contributors = document.at(CONTRIBUTORS_JSON_POINTER); + return StreamSupport.stream(contributors.spliterator(), false) + .map(ResourceCsvTransformer::extractName) + .collect(Collectors.toList()); + } + + private static String extractName(JsonNode contributor) { + return extractText(contributor, IDENTITY_NAME_JSON_POINTER, null); + } + + private static String extractText(JsonNode node, String pointer, String defaultValue) { + var value = node.at(pointer); + return value.isMissingNode() ? defaultValue : value.asText(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/enums/ParameterKey.java b/search-commons/src/main/java/no/unit/nva/search/common/enums/ParameterKey.java index 468ed90e3..e6c087502 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/enums/ParameterKey.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/enums/ParameterKey.java @@ -1,5 +1,6 @@ package no.unit.nva.search.common.enums; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.ErrorMessages.INVALID_BOOLEAN; import static no.unit.nva.constants.ErrorMessages.INVALID_DATE; import static no.unit.nva.constants.ErrorMessages.INVALID_NUMBER; @@ -13,16 +14,12 @@ import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_NUMBER; import static no.unit.nva.search.common.enums.ParameterKind.CUSTOM; import static no.unit.nva.search.common.enums.ParameterKind.KEYWORD; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import static java.util.Objects.nonNull; - -import no.unit.nva.constants.Words; - import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; +import no.unit.nva.constants.Words; /** * Interface for defining the keys used in the search service. diff --git a/search-commons/src/main/java/no/unit/nva/search/common/jwt/CachedJwtProvider.java b/search-commons/src/main/java/no/unit/nva/search/common/jwt/CachedJwtProvider.java index 683c5c2e9..29518cad3 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/jwt/CachedJwtProvider.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/jwt/CachedJwtProvider.java @@ -1,7 +1,6 @@ package no.unit.nva.search.common.jwt; import com.auth0.jwt.interfaces.DecodedJWT; - import java.time.Clock; import java.util.Date; diff --git a/search-commons/src/main/java/no/unit/nva/search/common/jwt/CognitoAuthenticator.java b/search-commons/src/main/java/no/unit/nva/search/common/jwt/CognitoAuthenticator.java index ae4bf5ac9..b8cb43c78 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/jwt/CognitoAuthenticator.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/jwt/CognitoAuthenticator.java @@ -2,20 +2,12 @@ import static no.unit.nva.auth.AuthorizedBackendClient.APPLICATION_X_WWW_FORM_URLENCODED; import static no.unit.nva.constants.Words.AUTHORIZATION; - import static nva.commons.core.attempt.Try.attempt; - import static org.apache.http.protocol.HTTP.CONTENT_TYPE; import com.auth0.jwt.JWT; import com.auth0.jwt.interfaces.DecodedJWT; import com.fasterxml.jackson.jr.ob.JSON; - -import no.unit.nva.auth.CognitoCredentials; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.paths.UriWrapper; - import java.net.HttpURLConnection; import java.net.URI; import java.net.http.HttpClient; @@ -24,6 +16,9 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Objects; +import no.unit.nva.auth.CognitoCredentials; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.paths.UriWrapper; /** * Class for authenticating with Cognito. diff --git a/search-commons/src/main/java/no/unit/nva/search/common/jwt/Tools.java b/search-commons/src/main/java/no/unit/nva/search/common/jwt/Tools.java index 979ce7ebc..b3101e20c 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/jwt/Tools.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/jwt/Tools.java @@ -3,45 +3,43 @@ import static no.unit.nva.constants.Words.SEARCH_INFRASTRUCTURE_CREDENTIALS; import static no.unit.nva.search.common.constant.Functions.readSearchInfrastructureAuthUri; +import java.net.URI; +import java.util.stream.Stream; import no.unit.nva.auth.CognitoCredentials; import no.unit.nva.search.common.records.UsernamePasswordWrapper; - import nva.commons.core.JacocoGenerated; import nva.commons.secrets.SecretsReader; -import java.net.URI; -import java.util.stream.Stream; - /** * Tools for handling JWT. * * @author Sondre Vestad */ public final class Tools { - @JacocoGenerated - public Tools() {} - - @JacocoGenerated - public static CachedJwtProvider getCachedJwtProvider(SecretsReader reader) { - return getUsernamePasswordStream(reader) - .map(Tools::getCognitoCredentials) - .map(CognitoAuthenticator::prepareWithCognitoCredentials) - .map(CachedJwtProvider::prepareWithAuthenticator) - .findFirst() - .orElseThrow(); - } - - @JacocoGenerated - public static Stream getUsernamePasswordStream( - SecretsReader secretsReader) { - return Stream.of( - secretsReader.fetchClassSecret( - SEARCH_INFRASTRUCTURE_CREDENTIALS, UsernamePasswordWrapper.class)); - } - - @JacocoGenerated - public static CognitoCredentials getCognitoCredentials(UsernamePasswordWrapper wrapper) { - var uri = URI.create(readSearchInfrastructureAuthUri()); - return new CognitoCredentials(wrapper::getUsername, wrapper::getPassword, uri); - } + @JacocoGenerated + public Tools() {} + + @JacocoGenerated + public static CachedJwtProvider getCachedJwtProvider(SecretsReader reader) { + return getUsernamePasswordStream(reader) + .map(Tools::getCognitoCredentials) + .map(CognitoAuthenticator::prepareWithCognitoCredentials) + .map(CachedJwtProvider::prepareWithAuthenticator) + .findFirst() + .orElseThrow(); + } + + @JacocoGenerated + public static Stream getUsernamePasswordStream( + SecretsReader secretsReader) { + return Stream.of( + secretsReader.fetchClassSecret( + SEARCH_INFRASTRUCTURE_CREDENTIALS, UsernamePasswordWrapper.class)); + } + + @JacocoGenerated + public static CognitoCredentials getCognitoCredentials(UsernamePasswordWrapper wrapper) { + var uri = URI.create(readSearchInfrastructureAuthUri()); + return new CognitoCredentials(wrapper::getUsername, wrapper::getPassword, uri); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/Facet.java b/search-commons/src/main/java/no/unit/nva/search/common/records/Facet.java index dcd7b5c48..4febb923b 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/Facet.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/Facet.java @@ -2,12 +2,10 @@ import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -import no.unit.nva.commons.json.JsonSerializable; - import java.net.URI; import java.util.Map; import java.util.Objects; +import no.unit.nva.commons.json.JsonSerializable; /** * Facet is a class that represents a facet in a search result. @@ -20,10 +18,10 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public record Facet( - URI id, String key, @JsonAlias("doc_count") Integer count, Map labels) - implements JsonSerializable { + URI id, String key, @JsonAlias("doc_count") Integer count, Map labels) + implements JsonSerializable { - public Facet { - Objects.requireNonNull(count); - } + public Facet { + Objects.requireNonNull(count); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/FacetsBuilder.java b/search-commons/src/main/java/no/unit/nva/search/common/records/FacetsBuilder.java index 88d0761db..e3b983cde 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/FacetsBuilder.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/FacetsBuilder.java @@ -3,52 +3,48 @@ import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.core.type.TypeReference; - -import no.unit.nva.commons.json.JsonUtils; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.paths.UriWrapper; - import java.net.URI; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import no.unit.nva.commons.json.JsonUtils; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.paths.UriWrapper; /** * @author Stig Norland */ final class FacetsBuilder { - @JacocoGenerated - public FacetsBuilder() {} - - public static Map> build(String aggregations, URI id) { - return toMapOfFacets(aggregations).entrySet().stream() - .map((entry) -> addIdToFacets(entry, id)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } - - private static Map> toMapOfFacets(String aggregations) { - final var typeReference = new TypeReference>>() {}; - return attempt(() -> JsonUtils.dtoObjectMapper.readValue(aggregations, typeReference)) - .orElseThrow(); - } - - private static Map.Entry> addIdToFacets( - Map.Entry> entry, URI id) { - var facets = - entry.getValue().stream() - .map( - facet -> - new Facet( - UriWrapper.fromUri(id) - .addQueryParameter( - entry.getKey(), facet.key()) - .getUri(), - facet.key(), - facet.count(), - facet.labels())) - .toList(); - return Map.entry(entry.getKey(), facets); - } + @JacocoGenerated + public FacetsBuilder() {} + + public static Map> build(String aggregations, URI id) { + return toMapOfFacets(aggregations).entrySet().stream() + .map((entry) -> addIdToFacets(entry, id)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private static Map> toMapOfFacets(String aggregations) { + final var typeReference = new TypeReference>>() {}; + return attempt(() -> JsonUtils.dtoObjectMapper.readValue(aggregations, typeReference)) + .orElseThrow(); + } + + private static Map.Entry> addIdToFacets( + Map.Entry> entry, URI id) { + var facets = + entry.getValue().stream() + .map( + facet -> + new Facet( + UriWrapper.fromUri(id) + .addQueryParameter(entry.getKey(), facet.key()) + .getUri(), + facet.key(), + facet.count(), + facet.labels())) + .toList(); + return Map.entry(entry.getKey(), facets); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/FilterBuilder.java b/search-commons/src/main/java/no/unit/nva/search/common/records/FilterBuilder.java index 7ab0aeb3d..356d4ed01 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/FilterBuilder.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/FilterBuilder.java @@ -1,7 +1,6 @@ package no.unit.nva.search.common.records; import no.unit.nva.search.common.Query; - import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.UnauthorizedException; diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/HttpResponseFormatter.java b/search-commons/src/main/java/no/unit/nva/search/common/records/HttpResponseFormatter.java index 62909868c..3a84e8cb7 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/HttpResponseFormatter.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/HttpResponseFormatter.java @@ -1,28 +1,23 @@ package no.unit.nva.search.common.records; import static com.google.common.net.MediaType.CSV_UTF_8; - +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.COMMA; import static no.unit.nva.search.common.constant.Functions.hasContent; - import static nva.commons.core.paths.UriWrapper.fromUri; -import static java.util.Objects.nonNull; - import com.google.common.net.MediaType; - -import no.unit.nva.constants.Words; -import no.unit.nva.search.common.AggregationFormat; -import no.unit.nva.search.common.QueryKeys; -import no.unit.nva.search.common.csv.ResourceCsvTransformer; -import no.unit.nva.search.common.enums.ParameterKey; - import java.net.URI; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.constants.Words; +import no.unit.nva.search.common.AggregationFormat; +import no.unit.nva.search.common.QueryKeys; +import no.unit.nva.search.common.csv.ResourceCsvTransformer; +import no.unit.nva.search.common.enums.ParameterKey; /** * HttpResponseFormatter is a class that formats a search response. @@ -32,101 +27,101 @@ * define the parameters that can be used in the query. */ public final class HttpResponseFormatter & ParameterKey> { - private final SwsResponse response; - private final MediaType mediaType; - private final URI source; - private final Integer offset; - private final Integer size; - private final Map facetPaths; - private final QueryKeys queryKeys; - private List mutators; - - public HttpResponseFormatter( - SwsResponse response, - MediaType mediaType, - URI source, - Integer offset, - Integer size, - Map facetPaths, - QueryKeys requestParameter) { - this.response = response; - this.mediaType = mediaType; - this.source = source; - this.offset = offset; - this.size = size; - this.facetPaths = facetPaths; - this.queryKeys = requestParameter; - } - - public HttpResponseFormatter(SwsResponse response, MediaType mediaType) { - this(response, mediaType, null, 0, 0, Map.of(), null); + private final SwsResponse response; + private final MediaType mediaType; + private final URI source; + private final Integer offset; + private final Integer size; + private final Map facetPaths; + private final QueryKeys queryKeys; + private List mutators; + + public HttpResponseFormatter( + SwsResponse response, + MediaType mediaType, + URI source, + Integer offset, + Integer size, + Map facetPaths, + QueryKeys requestParameter) { + this.response = response; + this.mediaType = mediaType; + this.source = source; + this.offset = offset; + this.size = size; + this.facetPaths = facetPaths; + this.queryKeys = requestParameter; + } + + public HttpResponseFormatter(SwsResponse response, MediaType mediaType) { + this(response, mediaType, null, 0, 0, Map.of(), null); + } + + public HttpResponseFormatter withMutators(JsonNodeMutator... mutators) { + this.mutators = List.of(mutators); + return this; + } + + public QueryKeys parameters() { + return queryKeys; + } + + public SwsResponse swsResponse() { + return response; + } + + public PagedSearch toPagedResponse() { + final var aggregationFormatted = + AggregationFormat.apply(response.aggregations(), facetPaths).toString(); + final var hits = + response.getSearchHits().stream() + .flatMap(hit -> getMutators().map(mutator -> mutator.transform(hit))) + .toList(); + + return new PagedSearchBuilder() + .withTotalHits(response.getTotalSize()) + .withHits(hits) + .withIds(source, getRequestParameter(), offset, size) + .withNextResultsBySortKey(nextResultsBySortKey(getRequestParameter(), source)) + .withAggregations(aggregationFormatted) + .build(); + } + + private Stream getMutators() { + if (this.mutators == null) { + return Stream.of(defaultMutator()); } - - public HttpResponseFormatter withMutators(JsonNodeMutator... mutators) { - this.mutators = List.of(mutators); - return this; - } - - public QueryKeys parameters() { - return queryKeys; - } - - public SwsResponse swsResponse() { - return response; - } - - public PagedSearch toPagedResponse() { - final var aggregationFormatted = - AggregationFormat.apply(response.aggregations(), facetPaths).toString(); - final var hits = - response.getSearchHits().stream() - .flatMap(hit -> getMutators().map(mutator -> mutator.transform(hit))) - .toList(); - - return new PagedSearchBuilder() - .withTotalHits(response.getTotalSize()) - .withHits(hits) - .withIds(source, getRequestParameter(), offset, size) - .withNextResultsBySortKey(nextResultsBySortKey(getRequestParameter(), source)) - .withAggregations(aggregationFormatted) - .build(); - } - - private Stream getMutators() { - if (this.mutators == null) { - return Stream.of(defaultMutator()); - } - return this.mutators.stream(); - } - - private JsonNodeMutator defaultMutator() { - return source -> source; - } - - public String toCsvText() { - return ResourceCsvTransformer.transform(response.getSearchHits()); - } - - private Map getRequestParameter() { - return queryKeys.asMap(); - } - - private URI nextResultsBySortKey(Map requestParameter, URI gatewayUri) { - requestParameter.remove(Words.FROM); - var sortParameter = - response.getSort().stream() - .map(value -> nonNull(value) ? value : "null") - .collect(Collectors.joining(COMMA)); - if (!hasContent(sortParameter)) { - return null; - } - var searchAfter = Words.SEARCH_AFTER.toLowerCase(Locale.getDefault()); - requestParameter.put(searchAfter, sortParameter); - return fromUri(gatewayUri).addQueryParameters(requestParameter).getUri(); - } - - @Override - public String toString() { - return CSV_UTF_8.is(this.mediaType) ? toCsvText() : toPagedResponse().toJsonString(); + return this.mutators.stream(); + } + + private JsonNodeMutator defaultMutator() { + return source -> source; + } + + public String toCsvText() { + return ResourceCsvTransformer.transform(response.getSearchHits()); + } + + private Map getRequestParameter() { + return queryKeys.asMap(); + } + + private URI nextResultsBySortKey(Map requestParameter, URI gatewayUri) { + requestParameter.remove(Words.FROM); + var sortParameter = + response.getSort().stream() + .map(value -> nonNull(value) ? value : "null") + .collect(Collectors.joining(COMMA)); + if (!hasContent(sortParameter)) { + return null; } + var searchAfter = Words.SEARCH_AFTER.toLowerCase(Locale.getDefault()); + requestParameter.put(searchAfter, sortParameter); + return fromUri(gatewayUri).addQueryParameters(requestParameter).getUri(); + } + + @Override + public String toString() { + return CSV_UTF_8.is(this.mediaType) ? toCsvText() : toPagedResponse().toJsonString(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearch.java b/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearch.java index 4019c24fa..5049ca12c 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearch.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearch.java @@ -1,18 +1,15 @@ package no.unit.nva.search.common.records; -import static no.unit.nva.constants.Defaults.PAGINATED_SEARCH_RESULT_CONTEXT; - import static java.util.Objects.nonNull; +import static no.unit.nva.constants.Defaults.PAGINATED_SEARCH_RESULT_CONTEXT; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.JsonNode; - -import no.unit.nva.commons.json.JsonSerializable; - import java.net.URI; import java.util.List; import java.util.Map; +import no.unit.nva.commons.json.JsonSerializable; /** * PagedSearch is a class that represents a paged search result. diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearchBuilder.java b/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearchBuilder.java index 8f5e163a9..a9c1e1dd0 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearchBuilder.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/PagedSearchBuilder.java @@ -3,15 +3,12 @@ import static java.util.Objects.isNull; import com.fasterxml.jackson.databind.JsonNode; - -import no.unit.nva.constants.Words; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.paths.UriWrapper; - import java.net.URI; import java.util.List; import java.util.Map; +import no.unit.nva.constants.Words; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.paths.UriWrapper; /** * PagedSearchBuilder is a class that builds @@ -20,78 +17,68 @@ */ public class PagedSearchBuilder { - private URI id; - private int totalHits; - private List hits; - private URI nextResults; - private URI nextSearchAfterResults; - private URI previousResults; - private Map> aggregations; - - @SuppressWarnings("PMD.NullAssignment") - public PagedSearch build() { - if (hasNoNextNorSort()) { - this.nextSearchAfterResults = null; // null values are not serialized - } - return new PagedSearch( - id, - totalHits, - hits, - nextResults, - nextSearchAfterResults, - previousResults, - aggregations); + private URI id; + private int totalHits; + private List hits; + private URI nextResults; + private URI nextSearchAfterResults; + private URI previousResults; + private Map> aggregations; + + @SuppressWarnings("PMD.NullAssignment") + public PagedSearch build() { + if (hasNoNextNorSort()) { + this.nextSearchAfterResults = null; // null values are not serialized } - - public PagedSearchBuilder withIds( - URI gatewayUri, Map requestParameter, Integer offset, Integer size) { - requestParameter.remove(Words.PAGE); - requestParameter.remove(Words.FROM); - this.id = createUriOffsetRef(requestParameter, offset, gatewayUri); - this.previousResults = createUriOffsetRef(requestParameter, offset - size, gatewayUri); - this.nextResults = - createNextResults(requestParameter, offset + size, totalHits, gatewayUri); - return this; - } - - public PagedSearchBuilder withNextResultsBySortKey(URI nextSearchAfterResults) { - this.nextSearchAfterResults = nextSearchAfterResults; - return this; - } - - public PagedSearchBuilder withTotalHits(int totalHits) { - this.totalHits = totalHits; - return this; - } - - public PagedSearchBuilder withHits(List hits) { - this.hits = hits; - return this; - } - - public PagedSearchBuilder withAggregations(String aggregations) { - this.aggregations = FacetsBuilder.build(aggregations, this.id); - return this; - } - - private URI createNextResults( - Map requestParameter, - Integer offset, - Integer totalSize, - URI gatewayUri) { - return offset < totalSize ? createUriOffsetRef(requestParameter, offset, gatewayUri) : null; - } - - private URI createUriOffsetRef(Map params, Integer offset, URI gatewayUri) { - if (offset < 0) { - return null; - } - params.put(Words.FROM, String.valueOf(offset)); - return UriWrapper.fromUri(gatewayUri).addQueryParameters(params).getUri(); - } - - @JacocoGenerated - private boolean hasNoNextNorSort() { - return isNull(this.nextResults) || !nextResults.getQuery().contains("sort"); + return new PagedSearch( + id, totalHits, hits, nextResults, nextSearchAfterResults, previousResults, aggregations); + } + + public PagedSearchBuilder withIds( + URI gatewayUri, Map requestParameter, Integer offset, Integer size) { + requestParameter.remove(Words.PAGE); + requestParameter.remove(Words.FROM); + this.id = createUriOffsetRef(requestParameter, offset, gatewayUri); + this.previousResults = createUriOffsetRef(requestParameter, offset - size, gatewayUri); + this.nextResults = createNextResults(requestParameter, offset + size, totalHits, gatewayUri); + return this; + } + + public PagedSearchBuilder withNextResultsBySortKey(URI nextSearchAfterResults) { + this.nextSearchAfterResults = nextSearchAfterResults; + return this; + } + + public PagedSearchBuilder withTotalHits(int totalHits) { + this.totalHits = totalHits; + return this; + } + + public PagedSearchBuilder withHits(List hits) { + this.hits = hits; + return this; + } + + public PagedSearchBuilder withAggregations(String aggregations) { + this.aggregations = FacetsBuilder.build(aggregations, this.id); + return this; + } + + private URI createNextResults( + Map requestParameter, Integer offset, Integer totalSize, URI gatewayUri) { + return offset < totalSize ? createUriOffsetRef(requestParameter, offset, gatewayUri) : null; + } + + private URI createUriOffsetRef(Map params, Integer offset, URI gatewayUri) { + if (offset < 0) { + return null; } + params.put(Words.FROM, String.valueOf(offset)); + return UriWrapper.fromUri(gatewayUri).addQueryParameters(params).getUri(); + } + + @JacocoGenerated + private boolean hasNoNextNorSort() { + return isNull(this.nextResults) || !nextResults.getQuery().contains("sort"); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/SwsResponse.java b/search-commons/src/main/java/no/unit/nva/search/common/records/SwsResponse.java index f3d2035f0..6ba459c90 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/SwsResponse.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/SwsResponse.java @@ -4,14 +4,11 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.JsonNode; - -import no.unit.nva.search.common.records.SwsResponse.HitsInfo.Hit; - -import nva.commons.core.JacocoGenerated; - import java.beans.Transient; import java.util.List; import java.util.Optional; +import no.unit.nva.search.common.records.SwsResponse.HitsInfo.Hit; +import nva.commons.core.JacocoGenerated; /** * Response from SWS, almost identical to Opensearch's response. @@ -19,106 +16,106 @@ * @author Stig Norland */ public record SwsResponse( - int took, - boolean timed_out, - ShardsInfo _shards, - HitsInfo hits, - JsonNode aggregations, - String _scroll_id) { - - @Transient - public Integer getTotalSize() { - return hits.total.value; + int took, + boolean timed_out, + ShardsInfo _shards, + HitsInfo hits, + JsonNode aggregations, + String _scroll_id) { + + @Transient + public Integer getTotalSize() { + return hits.total.value; + } + + @Transient + public List getSearchHits() { + return hits.hits().stream().map(Hit::_source).toList(); + } + + @JacocoGenerated + @Transient + public List getSort() { + return nonNull(hits) && nonNull(hits.hits) && !hits.hits.isEmpty() + ? Optional.ofNullable(hits.hits.getLast().sort()).orElse(List.of()) + : List.of(); + } + + public record ShardsInfo(Long total, Long successful, Long skipped, Long failed) {} + + public record HitsInfo(TotalInfo total, double max_score, List hits) { + public record TotalInfo(Integer value, String relation) {} + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + public record Hit( + String _index, + String _type, + String _id, + double _score, + JsonNode _source, + JsonNode inner_hits, + List sort) {} + } + + public static final class SwsResponseBuilder { + private String scrollId; + private int took; + private boolean timed_out; + private ShardsInfo shards; + private HitsInfo hits; + private JsonNode aggregations; + + private SwsResponseBuilder() {} + + public static SwsResponseBuilder swsResponseBuilder() { + return new SwsResponseBuilder(); } - @Transient - public List getSearchHits() { - return hits.hits().stream().map(Hit::_source).toList(); + public SwsResponseBuilder withScrollId(String scrollId) { + this.scrollId = scrollId; + return this; } - @JacocoGenerated - @Transient - public List getSort() { - return nonNull(hits) && nonNull(hits.hits) && !hits.hits.isEmpty() - ? Optional.ofNullable(hits.hits.getLast().sort()).orElse(List.of()) - : List.of(); + public SwsResponseBuilder withTook(int took) { + this.took = took; + return this; } - public record ShardsInfo(Long total, Long successful, Long skipped, Long failed) {} + public SwsResponseBuilder withTimedOut(boolean timedOut) { + this.timed_out = timedOut; + return this; + } - public record HitsInfo(TotalInfo total, double max_score, List hits) { - public record TotalInfo(Integer value, String relation) {} + public SwsResponseBuilder withShards(ShardsInfo shards) { + this.shards = shards; + return this; + } + + public SwsResponseBuilder withHits(HitsInfo hits) { + if (nonNull(hits) && hits.total().value() >= 0) { + this.hits = hits; + } + return this; + } + + public SwsResponseBuilder withAggregations(JsonNode aggregations) { + if (nonNull(aggregations) && !aggregations.isEmpty()) { + this.aggregations = aggregations; + } + return this; + } - @JsonInclude(JsonInclude.Include.NON_EMPTY) - public record Hit( - String _index, - String _type, - String _id, - double _score, - JsonNode _source, - JsonNode inner_hits, - List sort) {} + public SwsResponseBuilder merge(SwsResponse response) { + return withHits(response.hits()) + .withAggregations(response.aggregations()) + .withShards(response._shards()) + .withScrollId(response._scroll_id()) + .withTimedOut(response.timed_out()) + .withTook(response.took()); } - public static final class SwsResponseBuilder { - private String scrollId; - private int took; - private boolean timed_out; - private ShardsInfo shards; - private HitsInfo hits; - private JsonNode aggregations; - - private SwsResponseBuilder() {} - - public static SwsResponseBuilder swsResponseBuilder() { - return new SwsResponseBuilder(); - } - - public SwsResponseBuilder withScrollId(String scrollId) { - this.scrollId = scrollId; - return this; - } - - public SwsResponseBuilder withTook(int took) { - this.took = took; - return this; - } - - public SwsResponseBuilder withTimedOut(boolean timedOut) { - this.timed_out = timedOut; - return this; - } - - public SwsResponseBuilder withShards(ShardsInfo shards) { - this.shards = shards; - return this; - } - - public SwsResponseBuilder withHits(HitsInfo hits) { - if (nonNull(hits) && hits.total().value() >= 0) { - this.hits = hits; - } - return this; - } - - public SwsResponseBuilder withAggregations(JsonNode aggregations) { - if (nonNull(aggregations) && !aggregations.isEmpty()) { - this.aggregations = aggregations; - } - return this; - } - - public SwsResponseBuilder merge(SwsResponse response) { - return withHits(response.hits()) - .withAggregations(response.aggregations()) - .withShards(response._shards()) - .withScrollId(response._scroll_id()) - .withTimedOut(response.timed_out()) - .withTook(response.took()); - } - - public SwsResponse build() { - return new SwsResponse(took, timed_out, shards, hits, aggregations, scrollId); - } + public SwsResponse build() { + return new SwsResponse(took, timed_out, shards, hits, aggregations, scrollId); } + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/UserSettings.java b/search-commons/src/main/java/no/unit/nva/search/common/records/UserSettings.java index 695768f56..cedbd0bbe 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/UserSettings.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/UserSettings.java @@ -3,7 +3,6 @@ import static nva.commons.core.paths.UriWrapper.fromUri; import com.fasterxml.jackson.annotation.JsonInclude; - import java.util.List; /** @@ -13,8 +12,8 @@ */ @JsonInclude public record UserSettings(List promotedPublications) { - @Override - public List promotedPublications() { - return promotedPublications.stream().map(id -> fromUri(id).getLastPathElement()).toList(); - } + @Override + public List promotedPublications() { + return promotedPublications.stream().map(id -> fromUri(id).getLastPathElement()).toList(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/common/records/UsernamePasswordWrapper.java b/search-commons/src/main/java/no/unit/nva/search/common/records/UsernamePasswordWrapper.java index 04e99ddf0..e311cba8a 100644 --- a/search-commons/src/main/java/no/unit/nva/search/common/records/UsernamePasswordWrapper.java +++ b/search-commons/src/main/java/no/unit/nva/search/common/records/UsernamePasswordWrapper.java @@ -1,7 +1,6 @@ package no.unit.nva.search.common.records; import com.fasterxml.jackson.annotation.JsonProperty; - import nva.commons.core.JacocoGenerated; /** diff --git a/search-commons/src/main/java/no/unit/nva/search/importcandidate/Constants.java b/search-commons/src/main/java/no/unit/nva/search/importcandidate/Constants.java index e367e6f9a..b1c2d89be 100644 --- a/search-commons/src/main/java/no/unit/nva/search/importcandidate/Constants.java +++ b/search-commons/src/main/java/no/unit/nva/search/importcandidate/Constants.java @@ -20,17 +20,15 @@ import static no.unit.nva.search.common.constant.Functions.multipleFields; import static no.unit.nva.search.common.constant.Functions.nestedBranchBuilder; +import java.util.List; +import java.util.Locale; +import java.util.Map; import nva.commons.core.JacocoGenerated; - import org.opensearch.script.Script; import org.opensearch.script.ScriptType; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder; -import java.util.List; -import java.util.Locale; -import java.util.Map; - /** * Constants for the import candidates search. * @@ -38,99 +36,95 @@ */ public final class Constants { - public static final String ADDITIONAL_IDENTIFIERS_KEYWORD = - "additionalIdentifiers.value.keyword"; - public static final String CANDIDATE_STATUS = "candidateStatus"; - public static final String COLLABORATION_TYPE = "collaborationType"; - public static final String IMPORT_STATUS = "importStatus"; - public static final String COLLABORATION_TYPE_KEYWORD = COLLABORATION_TYPE + DOT + KEYWORD; - public static final String CONTRIBUTORS_IDENTITY_ID = "contributors.identity.id.keyword"; - public static final String CONTRIBUTORS_IDENTITY_NAME = "contributors.identity.name.keyword"; - public static final String CONTRIBUTOR_IDENTITY_KEYWORDS = - CONTRIBUTORS_IDENTITY_ID + PIPE + CONTRIBUTORS_IDENTITY_NAME; - public static final String DOI_KEYWORD = DOI + DOT + KEYWORD; - public static final String IMPORT_CANDIDATES_INDEX_NAME = "import-candidates"; - public static final String ID_KEYWORD = ID + DOT + KEYWORD; - public static final String MODIFIED_DATE_PATH = - IMPORT_STATUS + DOT + MODIFIED_DATE + DOT + KEYWORD; - public static final String MAIN_TITLE_KEYWORD = "mainTitle.keyword"; - public static final String ORGANIZATIONS = "organizations"; - public static final String ORGANIZATIONS_PATH = ORGANIZATIONS + DOT + ID_KEYWORD; - public static final String PUBLICATION_INSTANCE_TYPE = "publicationInstance.type.keyword"; - public static final String PUBLICATION_YEAR = "publicationYear"; - public static final String PUBLICATION_YEAR_KEYWORD = PUBLICATION_YEAR + DOT + KEYWORD; - public static final List IMPORT_CANDIDATES_AGGREGATIONS = - List.of( - branchBuilder(COLLABORATION_TYPE, COLLABORATION_TYPE_KEYWORD), - branchBuilder(TYPE, PUBLICATION_INSTANCE_TYPE), - branchBuilder(PUBLICATION_YEAR, PUBLICATION_YEAR_KEYWORD), - branchBuilder(IMPORT_STATUS, IMPORT_STATUS, CANDIDATE_STATUS, KEYWORD), - branchBuilder(FILES, FILES_STATUS, KEYWORD), - contributor(), - topLevelOrganisationsHierarchy()); - public static final String PUBLISHER_ID_KEYWORD = "publisher.id.keyword"; - public static final String IMPORT_STATUS_PATH = - multipleFields( - jsonPath(IMPORT_STATUS, CANDIDATE_STATUS, KEYWORD), - jsonPath(IMPORT_STATUS, "setBy", KEYWORD)); - public static final String TYPE_KEYWORD = "type.keyword"; - public static final String FILES_STATUS_PATH = FILES_STATUS + DOT + KEYWORD; - public static final Map FACET_IMPORT_CANDIDATE_PATHS = - Map.of( - CONTRIBUTOR, "/withAppliedFilter/contributor/id", - COLLABORATION_TYPE, "/withAppliedFilter/collaborationType", - TYPE, "/withAppliedFilter/type", - IMPORT_STATUS, "/withAppliedFilter/importStatus", - FILES, "/withAppliedFilter/files", - PUBLICATION_YEAR, "/withAppliedFilter/publicationYear", - TOP_LEVEL_ORGANIZATION, "/withAppliedFilter/organizations/id" - // LICENSE, "/withAppliedFilter/associatedArtifacts/license" - ); + public static final String ADDITIONAL_IDENTIFIERS_KEYWORD = "additionalIdentifiers.value.keyword"; + public static final String CANDIDATE_STATUS = "candidateStatus"; + public static final String COLLABORATION_TYPE = "collaborationType"; + public static final String IMPORT_STATUS = "importStatus"; + public static final String COLLABORATION_TYPE_KEYWORD = COLLABORATION_TYPE + DOT + KEYWORD; + public static final String CONTRIBUTORS_IDENTITY_ID = "contributors.identity.id.keyword"; + public static final String CONTRIBUTORS_IDENTITY_NAME = "contributors.identity.name.keyword"; + public static final String CONTRIBUTOR_IDENTITY_KEYWORDS = + CONTRIBUTORS_IDENTITY_ID + PIPE + CONTRIBUTORS_IDENTITY_NAME; + public static final String DOI_KEYWORD = DOI + DOT + KEYWORD; + public static final String IMPORT_CANDIDATES_INDEX_NAME = "import-candidates"; + public static final String ID_KEYWORD = ID + DOT + KEYWORD; + public static final String MODIFIED_DATE_PATH = + IMPORT_STATUS + DOT + MODIFIED_DATE + DOT + KEYWORD; + public static final String MAIN_TITLE_KEYWORD = "mainTitle.keyword"; + public static final String ORGANIZATIONS = "organizations"; + public static final String ORGANIZATIONS_PATH = ORGANIZATIONS + DOT + ID_KEYWORD; + public static final String PUBLICATION_INSTANCE_TYPE = "publicationInstance.type.keyword"; + public static final String PUBLICATION_YEAR = "publicationYear"; + public static final String PUBLICATION_YEAR_KEYWORD = PUBLICATION_YEAR + DOT + KEYWORD; + public static final List IMPORT_CANDIDATES_AGGREGATIONS = + List.of( + branchBuilder(COLLABORATION_TYPE, COLLABORATION_TYPE_KEYWORD), + branchBuilder(TYPE, PUBLICATION_INSTANCE_TYPE), + branchBuilder(PUBLICATION_YEAR, PUBLICATION_YEAR_KEYWORD), + branchBuilder(IMPORT_STATUS, IMPORT_STATUS, CANDIDATE_STATUS, KEYWORD), + branchBuilder(FILES, FILES_STATUS, KEYWORD), + contributor(), + topLevelOrganisationsHierarchy()); + public static final String PUBLISHER_ID_KEYWORD = "publisher.id.keyword"; + public static final String IMPORT_STATUS_PATH = + multipleFields( + jsonPath(IMPORT_STATUS, CANDIDATE_STATUS, KEYWORD), + jsonPath(IMPORT_STATUS, "setBy", KEYWORD)); + public static final String TYPE_KEYWORD = "type.keyword"; + public static final String FILES_STATUS_PATH = FILES_STATUS + DOT + KEYWORD; + public static final Map FACET_IMPORT_CANDIDATE_PATHS = + Map.of( + CONTRIBUTOR, "/withAppliedFilter/contributor/id", + COLLABORATION_TYPE, "/withAppliedFilter/collaborationType", + TYPE, "/withAppliedFilter/type", + IMPORT_STATUS, "/withAppliedFilter/importStatus", + FILES, "/withAppliedFilter/files", + PUBLICATION_YEAR, "/withAppliedFilter/publicationYear", + TOP_LEVEL_ORGANIZATION, "/withAppliedFilter/organizations/id" + // LICENSE, "/withAppliedFilter/associatedArtifacts/license" + ); - @JacocoGenerated - public Constants() {} + @JacocoGenerated + public Constants() {} - private static NestedAggregationBuilder contributor() { - return nestedBranchBuilder(CONTRIBUTOR, CONTRIBUTORS) - .subAggregation( - branchBuilder(ID, CONTRIBUTORS, IDENTITY, ID, KEYWORD) - .subAggregation( - branchBuilder( - NAME, CONTRIBUTORS, IDENTITY, NAME, KEYWORD))); - } + private static NestedAggregationBuilder contributor() { + return nestedBranchBuilder(CONTRIBUTOR, CONTRIBUTORS) + .subAggregation( + branchBuilder(ID, CONTRIBUTORS, IDENTITY, ID, KEYWORD) + .subAggregation(branchBuilder(NAME, CONTRIBUTORS, IDENTITY, NAME, KEYWORD))); + } - public static NestedAggregationBuilder topLevelOrganisationsHierarchy() { - return nestedBranchBuilder(ORGANIZATIONS, ORGANIZATIONS) - .subAggregation( - branchBuilder(ID, ORGANIZATIONS, ID, KEYWORD) - .subAggregation(labels(ORGANIZATIONS))); - } + public static NestedAggregationBuilder topLevelOrganisationsHierarchy() { + return nestedBranchBuilder(ORGANIZATIONS, ORGANIZATIONS) + .subAggregation( + branchBuilder(ID, ORGANIZATIONS, ID, KEYWORD).subAggregation(labels(ORGANIZATIONS))); + } - public static Script selectByLicense(String license) { - var script = - """ - if (doc['associatedArtifacts.license.keyword'].size()==0) { return false;} - def url = doc['associatedArtifacts.license.keyword'].value; - if (url.contains("/by-nc-nd")) { - return "CC-NC-ND".equals(params.license); - } else if (url.contains("/by-nc-sa")) { - return "CC-NC-SA".equals(params.license); - } else if (url.contains("/by-nc")) { - return "CC-NC".equals(params.license); - } else if (url.contains("/by-nd")) { - return "CC-ND".equals(params.license); - } else if (url.contains("/by-sa")) { - return "CC-SA".equals(params.license); - } else if (url.contains("/by")) { - return "CC-BY".equals(params.license); - } else { - return "Other".equals(params.license); - } - """; - return new Script( - ScriptType.INLINE, - no.unit.nva.search.resource.Constants.PAINLESS, - script, - Map.of("license", license.toUpperCase(Locale.getDefault()))); - } + public static Script selectByLicense(String license) { + var script = + """ + if (doc['associatedArtifacts.license.keyword'].size()==0) { return false;} + def url = doc['associatedArtifacts.license.keyword'].value; + if (url.contains("/by-nc-nd")) { + return "CC-NC-ND".equals(params.license); + } else if (url.contains("/by-nc-sa")) { + return "CC-NC-SA".equals(params.license); + } else if (url.contains("/by-nc")) { + return "CC-NC".equals(params.license); + } else if (url.contains("/by-nd")) { + return "CC-ND".equals(params.license); + } else if (url.contains("/by-sa")) { + return "CC-SA".equals(params.license); + } else if (url.contains("/by")) { + return "CC-BY".equals(params.license); + } else { + return "Other".equals(params.license); + } + """; + return new Script( + ScriptType.INLINE, + no.unit.nva.search.resource.Constants.PAINLESS, + script, + Map.of("license", license.toUpperCase(Locale.getDefault()))); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateClient.java b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateClient.java index 98442c91a..01c1e605d 100644 --- a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateClient.java @@ -4,55 +4,50 @@ import static no.unit.nva.search.common.jwt.Tools.getCachedJwtProvider; import com.fasterxml.jackson.core.JsonProcessingException; - +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.function.BinaryOperator; import no.unit.nva.search.common.OpenSearchClient; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.records.SwsResponse; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.FunctionWithException; import nva.commons.secrets.SecretsReader; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.function.BinaryOperator; - /** * ImportCandidateClient is a class that sends a request to the search index. * * @author Stig Norland */ public class ImportCandidateClient - extends OpenSearchClient { - - public ImportCandidateClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { - super(client, cachedJwtProvider); - } - - @JacocoGenerated - public static ImportCandidateClient defaultClient() { - var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); - return new ImportCandidateClient(HttpClient.newHttpClient(), cachedJwtProvider); - } - - @Override - protected SwsResponse jsonToResponse(HttpResponse response) - throws JsonProcessingException { - return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); - } - - @Override - protected BinaryOperator responseAccumulator() { - return (a, b) -> - SwsResponse.SwsResponseBuilder.swsResponseBuilder().merge(a).merge(b).build(); - } - - @Override - protected FunctionWithException - logAndReturnResult() { - return result -> { - logger.info(buildLogInfo(result)); - return result; - }; - } + extends OpenSearchClient { + + public ImportCandidateClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { + super(client, cachedJwtProvider); + } + + @JacocoGenerated + public static ImportCandidateClient defaultClient() { + var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); + return new ImportCandidateClient(HttpClient.newHttpClient(), cachedJwtProvider); + } + + @Override + protected SwsResponse jsonToResponse(HttpResponse response) + throws JsonProcessingException { + return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); + } + + @Override + protected BinaryOperator responseAccumulator() { + return (a, b) -> SwsResponse.SwsResponseBuilder.swsResponseBuilder().merge(a).merge(b).build(); + } + + @Override + protected FunctionWithException logAndReturnResult() { + return result -> { + logger.info(buildLogInfo(result)); + return result; + }; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateParameter.java b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateParameter.java index a7ac33799..ecd59a5c7 100644 --- a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateParameter.java +++ b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateParameter.java @@ -1,5 +1,6 @@ package no.unit.nva.search.importcandidate; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.ErrorMessages.NOT_IMPLEMENTED_FOR; import static no.unit.nva.constants.Words.CHAR_UNDERSCORE; import static no.unit.nva.constants.Words.COLON; @@ -41,27 +42,22 @@ import static no.unit.nva.search.importcandidate.Constants.PUBLISHER_ID_KEYWORD; import static no.unit.nva.search.resource.Constants.ASSOCIATED_ARTIFACTS_LICENSE; -import static java.util.Objects.nonNull; - +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.constants.Words; import no.unit.nva.search.common.enums.FieldOperator; import no.unit.nva.search.common.enums.ParameterKey; import no.unit.nva.search.common.enums.ParameterKind; import no.unit.nva.search.common.enums.ValueEncoding; - import nva.commons.core.JacocoGenerated; - import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.text.CaseUtils; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.Locale; -import java.util.Set; -import java.util.StringJoiner; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Enum for all the parameters that can be used to query the search index. This enum needs to * implement these parameters cristin @@ -71,184 +67,180 @@ */ @SuppressWarnings({"PMD.ExcessivePublicCount"}) public enum ImportCandidateParameter implements ParameterKey { - INVALID(ParameterKind.INVALID), - ADDITIONAL_IDENTIFIERS(FUZZY_KEYWORD, ANY_OF, ADDITIONAL_IDENTIFIERS_KEYWORD), - ADDITIONAL_IDENTIFIERS_NOT(KEYWORD, NOT_ALL_OF, ADDITIONAL_IDENTIFIERS_KEYWORD), - CATEGORY(FUZZY_KEYWORD, ANY_OF, PUBLICATION_INSTANCE_TYPE), - CATEGORY_NOT(KEYWORD, NOT_ALL_OF, PUBLICATION_INSTANCE_TYPE), - CREATED_DATE(DATE, BETWEEN, Words.CREATED_DATE), - CONTRIBUTOR(FUZZY_KEYWORD, ALL_OF, Constants.CONTRIBUTOR_IDENTITY_KEYWORDS), - CONTRIBUTOR_NOT(FUZZY_KEYWORD, NOT_ALL_OF, Constants.CONTRIBUTOR_IDENTITY_KEYWORDS), - COLLABORATION_TYPE(FUZZY_KEYWORD, ANY_OF, COLLABORATION_TYPE_KEYWORD), - COLLABORATION_TYPE_NOT(KEYWORD, NOT_ALL_OF, COLLABORATION_TYPE_KEYWORD), - CRISTIN_IDENTIFIER(CUSTOM), - DOI(FUZZY_KEYWORD, ANY_OF, Constants.DOI_KEYWORD), - DOI_NOT(TEXT, NOT_ALL_OF, Constants.DOI_KEYWORD), - FILES(KEYWORD, ALL_OF, FILES_STATUS_PATH), - ID(FUZZY_KEYWORD, ANY_OF, ID_KEYWORD), - ID_NOT(FUZZY_KEYWORD, NOT_ALL_OF, ID_KEYWORD), - IMPORT_STATUS(FUZZY_KEYWORD, ANY_OF, IMPORT_STATUS_PATH), - IMPORT_STATUS_NOT(FUZZY_KEYWORD, NOT_ALL_OF, IMPORT_STATUS_PATH), - LICENSE(CUSTOM, ALL_OF, ASSOCIATED_ARTIFACTS_LICENSE), - MODIFIED_DATE(DATE, BETWEEN, MODIFIED_DATE_PATH), - LICENSE_NOT(CUSTOM, NOT_ALL_OF, ASSOCIATED_ARTIFACTS_LICENSE), - PUBLICATION_YEAR(NUMBER, BETWEEN, PUBLICATION_YEAR_KEYWORD), - PUBLICATION_YEAR_BEFORE(NUMBER, FieldOperator.LESS_THAN, PUBLICATION_YEAR_KEYWORD), - PUBLICATION_YEAR_SINCE( - NUMBER, FieldOperator.GREATER_THAN_OR_EQUAL_TO, PUBLICATION_YEAR_KEYWORD), - PUBLISHER(KEYWORD, ALL_OF, PUBLISHER_ID_KEYWORD), - PUBLISHER_NOT(KEYWORD, NOT_ALL_OF, PUBLISHER_ID_KEYWORD), - SCOPUS_IDENTIFIER(CUSTOM), - TOP_LEVEL_ORGANIZATION(KEYWORD, ANY_OF, ORGANIZATIONS_PATH), - TOP_LEVEL_ORGANIZATION_NOT(KEYWORD, NOT_ALL_OF, ORGANIZATIONS_PATH), - TITLE(TEXT, ANY_OF, Constants.MAIN_TITLE_KEYWORD, null, null, 2F), - TITLE_NOT(TEXT, NOT_ALL_OF, Constants.MAIN_TITLE_KEYWORD), - TYPE(FUZZY_KEYWORD, ANY_OF, PUBLICATION_INSTANCE_TYPE), - TYPE_NOT(KEYWORD, NOT_ALL_OF, PUBLICATION_INSTANCE_TYPE), - // Query parameters passed to SWS/Opensearch - SEARCH_ALL(FREE_TEXT, ALL_OF, Q, PATTERN_IS_SEARCH_ALL_KEY, null, null), - // Pagination parameters - NODES_SEARCHED(FLAG, null, null, PATTERN_IS_FIELDS_SEARCHED, null, null), - NODES_INCLUDED(FLAG), - NODES_EXCLUDED(FLAG), - AGGREGATION(FLAG), - PAGE(NUMBER), - FROM(NUMBER, null, null, PATTERN_IS_FROM_KEY, null, null), - SIZE(NUMBER, null, null, PATTERN_IS_SIZE_KEY, null, null), - SEARCH_AFTER(FLAG), - SORT(SORT_KEY, null, null, PATTERN_IS_SORT_KEY, null, null), - SORT_ORDER(FLAG, ALL_OF, null, PATTERN_IS_SORT_ORDER_KEY, PATTERN_IS_ASC_DESC_VALUE, null), - ; - - public static final int IGNORE_PARAMETER_INDEX = 0; - - public static final Set IMPORT_CANDIDATE_PARAMETER_SET = - Arrays.stream(values()) - .filter(ImportCandidateParameter::isSearchField) - .sorted(ParameterKey::compareAscending) - .collect(Collectors.toCollection(LinkedHashSet::new)); - - private final ValueEncoding encoding; - private final String keyPattern; - private final String validValuePattern; - private final String[] fieldsToSearch; - private final FieldOperator fieldOperator; - private final String errorMsg; - private final ParameterKind paramkind; - private final Float boost; - - ImportCandidateParameter(ParameterKind kind) { - this(kind, ALL_OF, null, null, null, null); - } - - ImportCandidateParameter(ParameterKind kind, FieldOperator operator, String fieldsToSearch) { - this(kind, operator, fieldsToSearch, null, null, null); - } - - ImportCandidateParameter( - ParameterKind kind, - FieldOperator operator, - String fieldsToSearch, - String keyPattern, - String valuePattern, - Float boost) { - - this.fieldOperator = nonNull(operator) ? operator : NA; - this.boost = nonNull(boost) ? boost : 1F; - this.fieldsToSearch = - nonNull(fieldsToSearch) - ? fieldsToSearch.split(PATTERN_IS_PIPE) - : new String[] {name()}; - this.validValuePattern = ParameterKey.getValuePattern(kind, valuePattern); - this.errorMsg = ParameterKey.getErrorMessage(kind); - this.encoding = ParameterKey.getEncoding(kind); - this.keyPattern = - nonNull(keyPattern) - ? keyPattern - : PATTERN_IS_IGNORE_CASE - + name().replace(UNDERSCORE, PATTERN_IS_NONE_OR_ONE); - this.paramkind = kind; - } - - public static ImportCandidateParameter keyFromString(String paramName) { - var result = - Arrays.stream(values()) - .filter(ImportCandidateParameter::ignoreInvalidKey) - .filter(ParameterKey.equalTo(paramName)) - .collect(Collectors.toSet()); - return result.size() == 1 ? result.stream().findFirst().get() : INVALID; - } - - private static boolean ignoreInvalidKey(ImportCandidateParameter f) { - return f.ordinal() > IGNORE_PARAMETER_INDEX; - } - - private static boolean isSearchField(ImportCandidateParameter f) { - return f.ordinal() > IGNORE_PARAMETER_INDEX && f.ordinal() < SEARCH_ALL.ordinal(); - } - - @Override - public String asCamelCase() { - return CaseUtils.toCamelCase(this.name(), false, CHAR_UNDERSCORE); - } - - @Override - public String asLowerCase() { - return this.name().toLowerCase(Locale.getDefault()); - } - - @Override - public Float fieldBoost() { - return boost; - } - - @Override - public ParameterKind fieldType() { - return paramkind; - } - - @Override - public String fieldPattern() { - return keyPattern; - } - - @Override - public String valuePattern() { - return validValuePattern; - } - - @Override - public ValueEncoding valueEncoding() { - return encoding; - } - - @Override - public Stream searchFields(boolean... isKeyWord) { - return Arrays.stream(fieldsToSearch).map(ParameterKey.trimKeyword(fieldType(), isKeyWord)); - } - - @Override - public FieldOperator searchOperator() { - return fieldOperator; - } - - @Override - public String errorMessage() { - return errorMsg; - } - - @Override - @JacocoGenerated - public ImportCandidateParameter subQuery() { - throw new NotImplementedException(NOT_IMPLEMENTED_FOR + this.getClass().getName()); - } - - @Override - @JacocoGenerated - public String toString() { - return new StringJoiner(COLON, "Key[", "]") - .add(String.valueOf(ordinal())) - .add(asCamelCase()) - .toString(); - } + INVALID(ParameterKind.INVALID), + ADDITIONAL_IDENTIFIERS(FUZZY_KEYWORD, ANY_OF, ADDITIONAL_IDENTIFIERS_KEYWORD), + ADDITIONAL_IDENTIFIERS_NOT(KEYWORD, NOT_ALL_OF, ADDITIONAL_IDENTIFIERS_KEYWORD), + CATEGORY(FUZZY_KEYWORD, ANY_OF, PUBLICATION_INSTANCE_TYPE), + CATEGORY_NOT(KEYWORD, NOT_ALL_OF, PUBLICATION_INSTANCE_TYPE), + CREATED_DATE(DATE, BETWEEN, Words.CREATED_DATE), + CONTRIBUTOR(FUZZY_KEYWORD, ALL_OF, Constants.CONTRIBUTOR_IDENTITY_KEYWORDS), + CONTRIBUTOR_NOT(FUZZY_KEYWORD, NOT_ALL_OF, Constants.CONTRIBUTOR_IDENTITY_KEYWORDS), + COLLABORATION_TYPE(FUZZY_KEYWORD, ANY_OF, COLLABORATION_TYPE_KEYWORD), + COLLABORATION_TYPE_NOT(KEYWORD, NOT_ALL_OF, COLLABORATION_TYPE_KEYWORD), + CRISTIN_IDENTIFIER(CUSTOM), + DOI(FUZZY_KEYWORD, ANY_OF, Constants.DOI_KEYWORD), + DOI_NOT(TEXT, NOT_ALL_OF, Constants.DOI_KEYWORD), + FILES(KEYWORD, ALL_OF, FILES_STATUS_PATH), + ID(FUZZY_KEYWORD, ANY_OF, ID_KEYWORD), + ID_NOT(FUZZY_KEYWORD, NOT_ALL_OF, ID_KEYWORD), + IMPORT_STATUS(FUZZY_KEYWORD, ANY_OF, IMPORT_STATUS_PATH), + IMPORT_STATUS_NOT(FUZZY_KEYWORD, NOT_ALL_OF, IMPORT_STATUS_PATH), + LICENSE(CUSTOM, ALL_OF, ASSOCIATED_ARTIFACTS_LICENSE), + MODIFIED_DATE(DATE, BETWEEN, MODIFIED_DATE_PATH), + LICENSE_NOT(CUSTOM, NOT_ALL_OF, ASSOCIATED_ARTIFACTS_LICENSE), + PUBLICATION_YEAR(NUMBER, BETWEEN, PUBLICATION_YEAR_KEYWORD), + PUBLICATION_YEAR_BEFORE(NUMBER, FieldOperator.LESS_THAN, PUBLICATION_YEAR_KEYWORD), + PUBLICATION_YEAR_SINCE(NUMBER, FieldOperator.GREATER_THAN_OR_EQUAL_TO, PUBLICATION_YEAR_KEYWORD), + PUBLISHER(KEYWORD, ALL_OF, PUBLISHER_ID_KEYWORD), + PUBLISHER_NOT(KEYWORD, NOT_ALL_OF, PUBLISHER_ID_KEYWORD), + SCOPUS_IDENTIFIER(CUSTOM), + TOP_LEVEL_ORGANIZATION(KEYWORD, ANY_OF, ORGANIZATIONS_PATH), + TOP_LEVEL_ORGANIZATION_NOT(KEYWORD, NOT_ALL_OF, ORGANIZATIONS_PATH), + TITLE(TEXT, ANY_OF, Constants.MAIN_TITLE_KEYWORD, null, null, 2F), + TITLE_NOT(TEXT, NOT_ALL_OF, Constants.MAIN_TITLE_KEYWORD), + TYPE(FUZZY_KEYWORD, ANY_OF, PUBLICATION_INSTANCE_TYPE), + TYPE_NOT(KEYWORD, NOT_ALL_OF, PUBLICATION_INSTANCE_TYPE), + // Query parameters passed to SWS/Opensearch + SEARCH_ALL(FREE_TEXT, ALL_OF, Q, PATTERN_IS_SEARCH_ALL_KEY, null, null), + // Pagination parameters + NODES_SEARCHED(FLAG, null, null, PATTERN_IS_FIELDS_SEARCHED, null, null), + NODES_INCLUDED(FLAG), + NODES_EXCLUDED(FLAG), + AGGREGATION(FLAG), + PAGE(NUMBER), + FROM(NUMBER, null, null, PATTERN_IS_FROM_KEY, null, null), + SIZE(NUMBER, null, null, PATTERN_IS_SIZE_KEY, null, null), + SEARCH_AFTER(FLAG), + SORT(SORT_KEY, null, null, PATTERN_IS_SORT_KEY, null, null), + SORT_ORDER(FLAG, ALL_OF, null, PATTERN_IS_SORT_ORDER_KEY, PATTERN_IS_ASC_DESC_VALUE, null), + ; + + public static final int IGNORE_PARAMETER_INDEX = 0; + + public static final Set IMPORT_CANDIDATE_PARAMETER_SET = + Arrays.stream(values()) + .filter(ImportCandidateParameter::isSearchField) + .sorted(ParameterKey::compareAscending) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + private final ValueEncoding encoding; + private final String keyPattern; + private final String validValuePattern; + private final String[] fieldsToSearch; + private final FieldOperator fieldOperator; + private final String errorMsg; + private final ParameterKind paramkind; + private final Float boost; + + ImportCandidateParameter(ParameterKind kind) { + this(kind, ALL_OF, null, null, null, null); + } + + ImportCandidateParameter(ParameterKind kind, FieldOperator operator, String fieldsToSearch) { + this(kind, operator, fieldsToSearch, null, null, null); + } + + ImportCandidateParameter( + ParameterKind kind, + FieldOperator operator, + String fieldsToSearch, + String keyPattern, + String valuePattern, + Float boost) { + + this.fieldOperator = nonNull(operator) ? operator : NA; + this.boost = nonNull(boost) ? boost : 1F; + this.fieldsToSearch = + nonNull(fieldsToSearch) ? fieldsToSearch.split(PATTERN_IS_PIPE) : new String[] {name()}; + this.validValuePattern = ParameterKey.getValuePattern(kind, valuePattern); + this.errorMsg = ParameterKey.getErrorMessage(kind); + this.encoding = ParameterKey.getEncoding(kind); + this.keyPattern = + nonNull(keyPattern) + ? keyPattern + : PATTERN_IS_IGNORE_CASE + name().replace(UNDERSCORE, PATTERN_IS_NONE_OR_ONE); + this.paramkind = kind; + } + + public static ImportCandidateParameter keyFromString(String paramName) { + var result = + Arrays.stream(values()) + .filter(ImportCandidateParameter::ignoreInvalidKey) + .filter(ParameterKey.equalTo(paramName)) + .collect(Collectors.toSet()); + return result.size() == 1 ? result.stream().findFirst().get() : INVALID; + } + + private static boolean ignoreInvalidKey(ImportCandidateParameter f) { + return f.ordinal() > IGNORE_PARAMETER_INDEX; + } + + private static boolean isSearchField(ImportCandidateParameter f) { + return f.ordinal() > IGNORE_PARAMETER_INDEX && f.ordinal() < SEARCH_ALL.ordinal(); + } + + @Override + public String asCamelCase() { + return CaseUtils.toCamelCase(this.name(), false, CHAR_UNDERSCORE); + } + + @Override + public String asLowerCase() { + return this.name().toLowerCase(Locale.getDefault()); + } + + @Override + public Float fieldBoost() { + return boost; + } + + @Override + public ParameterKind fieldType() { + return paramkind; + } + + @Override + public String fieldPattern() { + return keyPattern; + } + + @Override + public String valuePattern() { + return validValuePattern; + } + + @Override + public ValueEncoding valueEncoding() { + return encoding; + } + + @Override + public Stream searchFields(boolean... isKeyWord) { + return Arrays.stream(fieldsToSearch).map(ParameterKey.trimKeyword(fieldType(), isKeyWord)); + } + + @Override + public FieldOperator searchOperator() { + return fieldOperator; + } + + @Override + public String errorMessage() { + return errorMsg; + } + + @Override + @JacocoGenerated + public ImportCandidateParameter subQuery() { + throw new NotImplementedException(NOT_IMPLEMENTED_FOR + this.getClass().getName()); + } + + @Override + @JacocoGenerated + public String toString() { + return new StringJoiner(COLON, "Key[", "]") + .add(String.valueOf(ordinal())) + .add(asCamelCase()) + .toString(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSearchQuery.java b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSearchQuery.java index 203b3d2c4..1306dc2f5 100644 --- a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSearchQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSearchQuery.java @@ -32,33 +32,28 @@ import static no.unit.nva.search.importcandidate.ImportCandidateParameter.SEARCH_AFTER; import static no.unit.nva.search.importcandidate.ImportCandidateParameter.SIZE; import static no.unit.nva.search.importcandidate.ImportCandidateParameter.SORT; - import static nva.commons.core.paths.UriWrapper.fromUri; - import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.termQuery; +import java.net.URI; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Stream; import no.unit.nva.search.common.AsType; import no.unit.nva.search.common.ParameterValidator; import no.unit.nva.search.common.SearchQuery; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.SortKey; - import nva.commons.core.JacocoGenerated; - import org.apache.lucene.search.join.ScoreMode; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.sort.SortOrder; -import java.net.URI; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.Stream; - /** * ImportCandidateSearchQuery is a class that represents a search query for import candidates. * @@ -66,204 +61,189 @@ */ public final class ImportCandidateSearchQuery extends SearchQuery { - ImportCandidateSearchQuery() { - super(); - } - - public static ImportCandidateValidator builder() { - return new ImportCandidateValidator(); - } - - @Override - protected ImportCandidateParameter keyAggregation() { - return AGGREGATION; - } - - @Override - protected ImportCandidateParameter keyFields() { - return NODES_SEARCHED; - } - - @Override - protected ImportCandidateParameter keySearchAfter() { - return SEARCH_AFTER; - } - - @Override - protected ImportCandidateParameter toKey(String keyName) { - return ImportCandidateParameter.keyFromString(keyName); - } - - @Override - protected SortKey toSortKey(String sortName) { - return ImportCandidateSort.fromSortKey(sortName); + ImportCandidateSearchQuery() { + super(); + } + + public static ImportCandidateValidator builder() { + return new ImportCandidateValidator(); + } + + @Override + protected ImportCandidateParameter keyAggregation() { + return AGGREGATION; + } + + @Override + protected ImportCandidateParameter keyFields() { + return NODES_SEARCHED; + } + + @Override + protected ImportCandidateParameter keySearchAfter() { + return SEARCH_AFTER; + } + + @Override + protected ImportCandidateParameter toKey(String keyName) { + return ImportCandidateParameter.keyFromString(keyName); + } + + @Override + protected SortKey toSortKey(String sortName) { + return ImportCandidateSort.fromSortKey(sortName); + } + + @Override + protected AsType from() { + return parameters().get(FROM); + } + + @Override + protected AsType size() { + return parameters().get(SIZE); + } + + @Override + public AsType sort() { + return parameters().get(SORT); + } + + @Override + protected String[] exclude() { + return parameters().get(NODES_EXCLUDED).split(COMMA); + } + + @Override + protected String[] include() { + return parameters().get(NODES_INCLUDED).split(COMMA); + } + + @Override + public URI openSearchUri() { + return fromUri(infrastructureApiUri).addChild(IMPORT_CANDIDATES_INDEX_NAME, SEARCH).getUri(); + } + + @Override + protected Map facetPaths() { + return FACET_IMPORT_CANDIDATE_PATHS; + } + + @Override + protected List builderAggregations() { + return IMPORT_CANDIDATES_AGGREGATIONS; + } + + @JacocoGenerated // default value shouldn't happen, (developer have forgotten to handle a key) + @Override + protected Stream> builderCustomQueryStream( + ImportCandidateParameter key) { + return switch (key) { + case CRISTIN_IDENTIFIER -> builderStreamAdditionalIdentifier(key, CRISTIN_AS_TYPE); + case SCOPUS_IDENTIFIER -> builderStreamAdditionalIdentifier(key, SCOPUS_AS_TYPE); + case LICENSE, LICENSE_NOT -> licenseQuery(key); + default -> throw new IllegalArgumentException("unhandled key -> " + key.name()); + }; + } + + private Stream> builderStreamAdditionalIdentifier( + ImportCandidateParameter key, String source) { + var value = parameters().get(key).as(); + var query = + QueryBuilders.nestedQuery( + ADDITIONAL_IDENTIFIERS, + boolQuery() + .must(termQuery(jsonPath(ADDITIONAL_IDENTIFIERS, VALUE, KEYWORD), value)) + .must(termQuery(jsonPath(ADDITIONAL_IDENTIFIERS, SOURCE_NAME, KEYWORD), source)), + ScoreMode.None); + + return Functions.queryToEntry(key, query); + } + + public Stream> licenseQuery( + ImportCandidateParameter key) { + var query = QueryBuilders.scriptQuery(selectByLicense(parameters().get(key).as())); + return Functions.queryToEntry(key, query); + } + + public static class ImportCandidateValidator + extends ParameterValidator { + + ImportCandidateValidator() { + super(new ImportCandidateSearchQuery()); } @Override - protected AsType from() { - return parameters().get(FROM); + protected void assignDefaultValues() { + requiredMissing() + .forEach( + key -> { + switch (key) { + case FROM -> setValue(key.name(), DEFAULT_OFFSET); + case SIZE -> setValue(key.name(), DEFAULT_VALUE_PER_PAGE); + case SORT -> setValue(key.name(), RELEVANCE_KEY_NAME); + case AGGREGATION -> setValue(key.name(), NONE); + default -> { + /* do nothing */ + } + } + }); } + @JacocoGenerated @Override - protected AsType size() { - return parameters().get(SIZE); + protected void applyRulesAfterValidation() { + // convert page to offset if offset is not set + if (query.parameters().isPresent(PAGE)) { + if (query.parameters().isPresent(FROM)) { + var page = query.parameters().get(PAGE).as(); + var perPage = query.parameters().get(SIZE).as(); + query.parameters().set(FROM, String.valueOf(page.longValue() * perPage.longValue())); + } + query.parameters().remove(PAGE); + } } @Override - public AsType sort() { - return parameters().get(SORT); + protected Collection validKeys() { + return IMPORT_CANDIDATE_PARAMETER_SET.stream() + .map(ImportCandidateParameter::asLowerCase) + .toList(); } @Override - protected String[] exclude() { - return parameters().get(NODES_EXCLUDED).split(COMMA); + protected void validateSortKeyName(String name) { + var nameSort = name.split(COLON_OR_SPACE); + if (nameSort.length == NAME_AND_SORT_LENGTH) { + SortOrder.fromString(nameSort[1]); + } else if (nameSort.length > NAME_AND_SORT_LENGTH) { + throw new IllegalArgumentException(TOO_MANY_ARGUMENTS + name); + } + if (ImportCandidateSort.fromSortKey(nameSort[0]) == ImportCandidateSort.INVALID) { + throw new IllegalArgumentException( + INVALID_VALUE_WITH_SORT.formatted(name, ImportCandidateSort.validSortKeys())); + } } @Override - protected String[] include() { - return parameters().get(NODES_INCLUDED).split(COMMA); - } + protected void setValue(String key, String value) { + var qpKey = ImportCandidateParameter.keyFromString(key); + var decodedValue = getDecodedValue(qpKey, value); - @Override - public URI openSearchUri() { - return fromUri(infrastructureApiUri) - .addChild(IMPORT_CANDIDATES_INDEX_NAME, SEARCH) - .getUri(); + switch (qpKey) { + case SEARCH_AFTER, FROM, SIZE, PAGE, AGGREGATION -> + query.parameters().set(qpKey, decodedValue); + case NODES_SEARCHED -> query.parameters().set(qpKey, ignoreInvalidFields(decodedValue)); + case SORT -> mergeToKey(SORT, trimSpace(decodedValue)); + case SORT_ORDER -> mergeToKey(SORT, decodedValue); + case INVALID -> invalidKeys.add(key); + default -> mergeToKey(qpKey, decodedValue); + } } @Override - protected Map facetPaths() { - return FACET_IMPORT_CANDIDATE_PATHS; - } - - @Override - protected List builderAggregations() { - return IMPORT_CANDIDATES_AGGREGATIONS; - } - - @JacocoGenerated // default value shouldn't happen, (developer have forgotten to handle a key) - @Override - protected Stream> builderCustomQueryStream( - ImportCandidateParameter key) { - return switch (key) { - case CRISTIN_IDENTIFIER -> builderStreamAdditionalIdentifier(key, CRISTIN_AS_TYPE); - case SCOPUS_IDENTIFIER -> builderStreamAdditionalIdentifier(key, SCOPUS_AS_TYPE); - case LICENSE, LICENSE_NOT -> licenseQuery(key); - default -> throw new IllegalArgumentException("unhandled key -> " + key.name()); - }; - } - - private Stream> builderStreamAdditionalIdentifier( - ImportCandidateParameter key, String source) { - var value = parameters().get(key).as(); - var query = - QueryBuilders.nestedQuery( - ADDITIONAL_IDENTIFIERS, - boolQuery() - .must( - termQuery( - jsonPath(ADDITIONAL_IDENTIFIERS, VALUE, KEYWORD), - value)) - .must( - termQuery( - jsonPath( - ADDITIONAL_IDENTIFIERS, - SOURCE_NAME, - KEYWORD), - source)), - ScoreMode.None); - - return Functions.queryToEntry(key, query); - } - - public Stream> licenseQuery( - ImportCandidateParameter key) { - var query = QueryBuilders.scriptQuery(selectByLicense(parameters().get(key).as())); - return Functions.queryToEntry(key, query); - } - - public static class ImportCandidateValidator - extends ParameterValidator { - - ImportCandidateValidator() { - super(new ImportCandidateSearchQuery()); - } - - @Override - protected void assignDefaultValues() { - requiredMissing() - .forEach( - key -> { - switch (key) { - case FROM -> setValue(key.name(), DEFAULT_OFFSET); - case SIZE -> setValue(key.name(), DEFAULT_VALUE_PER_PAGE); - case SORT -> setValue(key.name(), RELEVANCE_KEY_NAME); - case AGGREGATION -> setValue(key.name(), NONE); - default -> { - /* do nothing */ - } - } - }); - } - - @JacocoGenerated - @Override - protected void applyRulesAfterValidation() { - // convert page to offset if offset is not set - if (query.parameters().isPresent(PAGE)) { - if (query.parameters().isPresent(FROM)) { - var page = query.parameters().get(PAGE).as(); - var perPage = query.parameters().get(SIZE).as(); - query.parameters() - .set(FROM, String.valueOf(page.longValue() * perPage.longValue())); - } - query.parameters().remove(PAGE); - } - } - - @Override - protected Collection validKeys() { - return IMPORT_CANDIDATE_PARAMETER_SET.stream() - .map(ImportCandidateParameter::asLowerCase) - .toList(); - } - - @Override - protected void validateSortKeyName(String name) { - var nameSort = name.split(COLON_OR_SPACE); - if (nameSort.length == NAME_AND_SORT_LENGTH) { - SortOrder.fromString(nameSort[1]); - } else if (nameSort.length > NAME_AND_SORT_LENGTH) { - throw new IllegalArgumentException(TOO_MANY_ARGUMENTS + name); - } - if (ImportCandidateSort.fromSortKey(nameSort[0]) == ImportCandidateSort.INVALID) { - throw new IllegalArgumentException( - INVALID_VALUE_WITH_SORT.formatted( - name, ImportCandidateSort.validSortKeys())); - } - } - - @Override - protected void setValue(String key, String value) { - var qpKey = ImportCandidateParameter.keyFromString(key); - var decodedValue = getDecodedValue(qpKey, value); - - switch (qpKey) { - case SEARCH_AFTER, FROM, SIZE, PAGE, AGGREGATION -> - query.parameters().set(qpKey, decodedValue); - case NODES_SEARCHED -> - query.parameters().set(qpKey, ignoreInvalidFields(decodedValue)); - case SORT -> mergeToKey(SORT, trimSpace(decodedValue)); - case SORT_ORDER -> mergeToKey(SORT, decodedValue); - case INVALID -> invalidKeys.add(key); - default -> mergeToKey(qpKey, decodedValue); - } - } - - @Override - protected boolean isKeyValid(String keyName) { - return ImportCandidateParameter.keyFromString(keyName) - != ImportCandidateParameter.INVALID; - } + protected boolean isKeyValid(String keyName) { + return ImportCandidateParameter.keyFromString(keyName) != ImportCandidateParameter.INVALID; } + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSort.java b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSort.java index e31074c4f..85ee57c48 100644 --- a/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSort.java +++ b/search-commons/src/main/java/no/unit/nva/search/importcandidate/ImportCandidateSort.java @@ -2,19 +2,16 @@ import static no.unit.nva.constants.Words.CHAR_UNDERSCORE; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_PIPE; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import no.unit.nva.constants.Words; -import no.unit.nva.search.common.enums.SortKey; - -import org.apache.commons.text.CaseUtils; - import java.util.Arrays; import java.util.Collection; import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.constants.Words; +import no.unit.nva.search.common.enums.SortKey; +import org.apache.commons.text.CaseUtils; /** * ImportCandidateSort is an enum for sorting import candidates. @@ -22,56 +19,54 @@ * @author Stig Norland */ public enum ImportCandidateSort implements SortKey { - INVALID(EMPTY_STRING), - RELEVANCE(Words.SCORE), - COLLABORATION_TYPE(Constants.COLLABORATION_TYPE_KEYWORD), - CREATED_DATE(Words.CREATED_DATE), - INSTANCE_TYPE(Constants.TYPE_KEYWORD), - PUBLICATION_YEAR(Constants.PUBLICATION_YEAR_KEYWORD), - TITLE(Constants.MAIN_TITLE_KEYWORD), - TYPE(Constants.TYPE_KEYWORD); + INVALID(EMPTY_STRING), + RELEVANCE(Words.SCORE), + COLLABORATION_TYPE(Constants.COLLABORATION_TYPE_KEYWORD), + CREATED_DATE(Words.CREATED_DATE), + INSTANCE_TYPE(Constants.TYPE_KEYWORD), + PUBLICATION_YEAR(Constants.PUBLICATION_YEAR_KEYWORD), + TITLE(Constants.MAIN_TITLE_KEYWORD), + TYPE(Constants.TYPE_KEYWORD); - private final String keyValidationRegEx; - private final String path; + private final String keyValidationRegEx; + private final String path; - ImportCandidateSort(String jsonPath) { - this.keyValidationRegEx = SortKey.getIgnoreCaseAndUnderscoreKeyExpression(this.name()); - this.path = jsonPath; - } + ImportCandidateSort(String jsonPath) { + this.keyValidationRegEx = SortKey.getIgnoreCaseAndUnderscoreKeyExpression(this.name()); + this.path = jsonPath; + } - public static ImportCandidateSort fromSortKey(String keyName) { - var result = - Arrays.stream(values()) - .filter(SortKey.equalTo(keyName)) - .collect(Collectors.toSet()); - return result.size() == 1 ? result.stream().findFirst().get() : INVALID; - } + public static ImportCandidateSort fromSortKey(String keyName) { + var result = + Arrays.stream(values()).filter(SortKey.equalTo(keyName)).collect(Collectors.toSet()); + return result.size() == 1 ? result.stream().findFirst().get() : INVALID; + } - public static Collection validSortKeys() { - return Arrays.stream(values()) - .sorted(SortKey::compareAscending) - .skip(1) // skip INVALID - .map(SortKey::asCamelCase) - .toList(); - } + public static Collection validSortKeys() { + return Arrays.stream(values()) + .sorted(SortKey::compareAscending) + .skip(1) // skip INVALID + .map(SortKey::asCamelCase) + .toList(); + } - @Override - public String asCamelCase() { - return CaseUtils.toCamelCase(this.name(), false, CHAR_UNDERSCORE); - } + @Override + public String asCamelCase() { + return CaseUtils.toCamelCase(this.name(), false, CHAR_UNDERSCORE); + } - @Override - public String asLowerCase() { - return this.name().toLowerCase(Locale.getDefault()); - } + @Override + public String asLowerCase() { + return this.name().toLowerCase(Locale.getDefault()); + } - @Override - public String keyPattern() { - return keyValidationRegEx; - } + @Override + public String keyPattern() { + return keyValidationRegEx; + } - @Override - public Stream jsonPaths() { - return Arrays.stream(path.split(PATTERN_IS_PIPE)); - } + @Override + public Stream jsonPaths() { + return Arrays.stream(path.split(PATTERN_IS_PIPE)); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/Constants.java b/search-commons/src/main/java/no/unit/nva/search/resource/Constants.java index e66320768..0db545c65 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/Constants.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/Constants.java @@ -69,8 +69,11 @@ import static no.unit.nva.search.common.constant.Functions.multipleFields; import static no.unit.nva.search.common.constant.Functions.nestedBranchBuilder; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import nva.commons.core.JacocoGenerated; - import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder; @@ -79,11 +82,6 @@ import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - /** * Constants for the Resource Search. * @@ -101,370 +99,343 @@ public final class Constants { public static final String CRISTIN_IDENTIFIER = "cristinIdentifier"; public static final String V_2024_12_01_SIMPLER_MODEL = "2024-12-01"; - public static final String PERSON_PREFERENCES = "/person-preferences/"; - public static final String UNIQUE_PUBLICATIONS = "unique_publications"; - public static final String CRISTIN_ORGANIZATION_PATH = "/cristin/organization/"; - public static final String CRISTIN_PERSON_PATH = "/cristin/person/"; - public static final List GLOBAL_EXCLUDED_FIELDS = List.of("joinField"); - public static final String DEFAULT_RESOURCE_SORT_FIELDS = - RELEVANCE_KEY_NAME + COMMA + IDENTIFIER; - public static final String IDENTIFIER_KEYWORD = jsonPath(IDENTIFIER, KEYWORD); - public static final String FILES_STATUS_KEYWORD = jsonPath(FILES_STATUS, KEYWORD); - public static final String ENTITY_CONTRIBUTORS = jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS); - public static final String CONTRIBUTOR_COUNT_NO_KEYWORD = - jsonPath(ENTITY_DESCRIPTION, "contributorsCount"); - public static final String ENTITY_PUBLICATION_CONTEXT = - jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); - - public static final String REFERENCE_PUBLICATION_CONTEXT_ID_KEYWORD = - jsonPath(ENTITY_PUBLICATION_CONTEXT, ID, KEYWORD); - - public static final String ENTITY_PUBLICATION_INSTANCE = - jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE); - - public static final String CONTRIBUTOR_ORG_KEYWORD = - jsonPath(CONTRIBUTOR_ORGANIZATIONS, KEYWORD); - - public static final String CONTRIBUTORS_AFFILIATION_ID_KEYWORD = - jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, ID, KEYWORD); - - public static final String CONTRIBUTORS_AFFILIATION_LABELS = - jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, LABELS); - public static final String ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION_LABELS_KEYWORD = - multipleFields( - jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, ENGLISH_CODE, KEYWORD), - jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, NYNORSK_CODE, KEYWORD), - jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, BOKMAAL_CODE, KEYWORD), - jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, SAMI_CODE, KEYWORD)); - public static final String ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION = - multipleFields( - CONTRIBUTORS_AFFILIATION_ID_KEYWORD, - ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION_LABELS_KEYWORD); - public static final String CONTRIBUTORS_FIELDS = - multipleFields( - jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ASTERISK), - jsonPath(ENTITY_CONTRIBUTORS, ROLE, ASTERISK), - jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, ASTERISK)); - public static final String CONTRIBUTORS_IDENTITY_ID = - jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ID, KEYWORD); - public static final String CONTRIBUTORS_IDENTITY_NAME_KEYWORD = - multipleFields( - jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, NAME, KEYWORD), - CONTRIBUTORS_IDENTITY_ID); - public static final String CONTRIBUTORS_IDENTITY_ORC_ID_KEYWORD = - jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ORC_ID, KEYWORD); - public static final String SCIENTIFIC_SERIES = - jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, SCIENTIFIC_VALUE, KEYWORD); - public static final String SCIENTIFIC_PUBLISHER = - jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, SCIENTIFIC_VALUE, KEYWORD); - public static final String SCIENTIFIC_OTHER = - jsonPath(ENTITY_PUBLICATION_CONTEXT, SCIENTIFIC_VALUE, KEYWORD); - public static final String SCIENTIFIC_LEVEL_SEARCH_FIELD = - multipleFields(SCIENTIFIC_SERIES, SCIENTIFIC_PUBLISHER, SCIENTIFIC_OTHER); - public static final String COURSE_CODE_KEYWORD = - jsonPath(ENTITY_PUBLICATION_CONTEXT, COURSE, CODE, KEYWORD); - public static final String ENTITY_DESCRIPTION_PUBLICATION_PAGES = - jsonPath(ENTITY_PUBLICATION_INSTANCE, PAGES, PAGES, KEYWORD); - public static final String SUBJECTS = "subjects"; - public static final String ENTITY_DESCRIPTION_PUBLICATION_DATE_YEAR = - jsonPath(ENTITY_DESCRIPTION, PUBLICATION_DATE, YEAR); - public static final String REFERENCE_DOI_KEYWORD = - multipleFields( - jsonPath(ENTITY_DESCRIPTION, REFERENCE, DOI, KEYWORD), jsonPath(DOI, KEYWORD)); - public static final String ASSOCIATED_ARTIFACTS_LABELS = - jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, LABELS); - public static final String ASSOCIATED_ARTIFACTS_LICENSE = - multipleFields( - jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, NAME, KEYWORD), - jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, VALUE, KEYWORD), - jsonPath(ASSOCIATED_ARTIFACTS_LABELS, ENGLISH_CODE, KEYWORD), - jsonPath(ASSOCIATED_ARTIFACTS_LABELS, NYNORSK_CODE, KEYWORD), - jsonPath(ASSOCIATED_ARTIFACTS_LABELS, BOKMAAL_CODE, KEYWORD), - jsonPath(ASSOCIATED_ARTIFACTS_LABELS, SAMI_CODE, KEYWORD)); - public static final String PUBLISHER_ID_KEYWORD = jsonPath(PUBLISHER, ID, KEYWORD); - public static final String STATUS_KEYWORD = jsonPath(STATUS, KEYWORD); - public static final String PUBLICATION_CONTEXT_ISBN_LIST = - jsonPath(ENTITY_PUBLICATION_CONTEXT, ISBN_LIST); - public static final String PUBLICATION_CONTEXT_TYPE_KEYWORD = - jsonPath(ENTITY_PUBLICATION_CONTEXT, TYPE, KEYWORD); - public static final String PUBLICATION_INSTANCE_TYPE = - jsonPath(ENTITY_PUBLICATION_INSTANCE, TYPE, KEYWORD); - public static final String PUBLICATION_CONTEXT_PUBLISHER = - multipleFields( - jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, NAME, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, ID, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, ISBN_PREFIX, KEYWORD)); - - public static final String ENTITY_DESCRIPTION_REFERENCE_CONTEXT_REFERENCE = - jsonPath(ENTITY_PUBLICATION_CONTEXT, ENTITY_DESCRIPTION, REFERENCE); - public static final String ENTITY_DESCRIPTION_REFERENCE_SERIES = - multipleFields( - jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, "issn", KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, NAME, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, TITLE, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, ID, KEYWORD)); - public static final String ENTITY_DESCRIPTION_REFERENCE_JOURNAL = - multipleFields( - jsonPath(ENTITY_PUBLICATION_CONTEXT, NAME, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, ID, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, PRINT_ISSN, KEYWORD), - jsonPath(ENTITY_PUBLICATION_CONTEXT, ONLINE_ISSN, KEYWORD)); - public static final String ENTITY_DESCRIPTION_MAIN_TITLE = - jsonPath(ENTITY_DESCRIPTION, MAIN_TITLE); - public static final String ENTITY_DESCRIPTION_MAIN_TITLE_KEYWORD = - jsonPath(ENTITY_DESCRIPTION_MAIN_TITLE, KEYWORD); - public static final String FUNDINGS_SOURCE_LABELS = jsonPath(FUNDINGS, SOURCE, LABELS); - public static final String HANDLE_KEYWORD = - multipleFields( - jsonPath(HANDLE, KEYWORD), jsonPath("additionalIdentifiers", VALUE, KEYWORD)); - public static final String RESOURCE_OWNER_OWNER_AFFILIATION_KEYWORD = - jsonPath(RESOURCE_OWNER, OWNER_AFFILIATION, KEYWORD); - public static final String RESOURCE_OWNER_OWNER_KEYWORD = - jsonPath(RESOURCE_OWNER, OWNER, KEYWORD); - public static final String ENTITY_TAGS = jsonPath(ENTITY_DESCRIPTION, TAGS, KEYWORD); - // ----------------------------------- - public static final String TOP_LEVEL_ORG_ID = jsonPath(TOP_LEVEL_ORGANIZATIONS, ID, KEYWORD); - public static final String ENTITY_ABSTRACT = jsonPath(ENTITY_DESCRIPTION, ABSTRACT); - public static final String ENTITY_DESCRIPTION_LANGUAGE = - jsonPath(ENTITY_DESCRIPTION, LANGUAGE, KEYWORD); - public static final String SCIENTIFIC_INDEX_YEAR = jsonPath(SCIENTIFIC_INDEX, YEAR); - public static final String SCIENTIFIC_INDEX_STATUS_KEYWORD = - jsonPath(SCIENTIFIC_INDEX, STATUS_KEYWORD); - public static final String PUBLICATION_CONTEXT_PATH = - jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); - public static final String ENTITY_DESCRIPTION_REFERENCE_PUBLICATION_CONTEXT_ISSN = - multipleFields( - jsonPath(PUBLICATION_CONTEXT_PATH, ONLINE_ISSN, KEYWORD), - jsonPath(PUBLICATION_CONTEXT_PATH, PRINT_ISSN, KEYWORD), - jsonPath(PUBLICATION_CONTEXT_PATH, SERIES, ONLINE_ISSN, KEYWORD), - jsonPath(PUBLICATION_CONTEXT_PATH, SERIES, PRINT_ISSN, KEYWORD)); - - public static final String FUNDING_IDENTIFIER_KEYWORD = jsonPath(FUNDINGS, IDENTIFIER_KEYWORD); - - public static final String FUNDINGS_IDENTIFIER_FUNDINGS_SOURCE_IDENTIFIER = - multipleFields( - jsonPath(FUNDINGS, IDENTIFIER_KEYWORD), - jsonPath(FUNDINGS, SOURCE, IDENTIFIER, KEYWORD)); - - public static final String FUNDINGS_SOURCE_IDENTIFIER_FUNDINGS_SOURCE_LABELS = - multipleFields( - FUNDINGS_IDENTIFIER_FUNDINGS_SOURCE_IDENTIFIER, - jsonPath(FUNDINGS_SOURCE_LABELS, ENGLISH_CODE, KEYWORD), - jsonPath(FUNDINGS_SOURCE_LABELS, NYNORSK_CODE, KEYWORD), - jsonPath(FUNDINGS_SOURCE_LABELS, BOKMAAL_CODE, KEYWORD), - jsonPath(FUNDINGS_SOURCE_LABELS, SAMI_CODE, KEYWORD)); - - public static final String PARENT_PUBLICATION_ID = - multipleFields( - jsonPath(ENTITY_PUBLICATION_INSTANCE, "compliesWith", KEYWORD), - jsonPath(ENTITY_PUBLICATION_INSTANCE, "referencedBy", KEYWORD), - jsonPath(ENTITY_PUBLICATION_INSTANCE, "corrigendumFor", KEYWORD), - jsonPath(ENTITY_PUBLICATION_INSTANCE, "manifestations", ID, KEYWORD), - jsonPath(ENTITY_PUBLICATION_INSTANCE, ID, KEYWORD)); - public static final String PAINLESS = "painless"; - public static final List RESOURCES_AGGREGATIONS = - List.of( - filesHierarchy(), - associatedArtifactsHierarchy(), - entityDescriptionHierarchy(), - fundingSourceHierarchy(), - scientificIndexHierarchy(), - topLevelOrganisationsHierarchy()); - public static final String SEQUENCE = "sequence"; - private static final Map facetResourcePaths1 = - Map.of( - TYPE, "/withAppliedFilter/entityDescription/reference/publicationInstance/type", - SERIES, - "/withAppliedFilter/entityDescription/reference/publicationContext/series/id", - LICENSE, "/withAppliedFilter/associatedArtifacts/license"); - private static final Map facetResourcePaths2 = - Map.of( - FILES, "/withAppliedFilter/files", - PUBLISHER, - "/withAppliedFilter/entityDescription/reference/publicationContext/publisher", - JOURNAL, - "/withAppliedFilter/entityDescription/reference/publicationContext/journal/id", - CONTRIBUTOR, "/withAppliedFilter/entityDescription/contributor/id", - FUNDING_SOURCE, "/withAppliedFilter/fundings/id", - TOP_LEVEL_ORGANIZATION, "/withAppliedFilter/topLevelOrganization/id", - SCIENTIFIC_INDEX, "/withAppliedFilter/scientificIndex/year"); - public static final Map facetResourcePaths = - Stream.of(facetResourcePaths1, facetResourcePaths2) - .flatMap(map -> map.entrySet().stream()) - .sorted(Map.Entry.comparingByValue()) - .collect( - LinkedHashMap::new, - (map, entry) -> map.put(entry.getKey(), entry.getValue()), - LinkedHashMap::putAll); - - @JacocoGenerated - public Constants() {} - - public static NestedAggregationBuilder topLevelOrganisationsHierarchy() { - return nestedBranchBuilder(TOP_LEVEL_ORGANIZATION, TOP_LEVEL_ORGANIZATIONS) + public static final String PERSON_PREFERENCES = "/person-preferences/"; + public static final String UNIQUE_PUBLICATIONS = "unique_publications"; + public static final String CRISTIN_ORGANIZATION_PATH = "/cristin/organization/"; + public static final String CRISTIN_PERSON_PATH = "/cristin/person/"; + public static final List GLOBAL_EXCLUDED_FIELDS = List.of("joinField"); + public static final String DEFAULT_RESOURCE_SORT_FIELDS = RELEVANCE_KEY_NAME + COMMA + IDENTIFIER; + public static final String IDENTIFIER_KEYWORD = jsonPath(IDENTIFIER, KEYWORD); + public static final String FILES_STATUS_KEYWORD = jsonPath(FILES_STATUS, KEYWORD); + public static final String ENTITY_CONTRIBUTORS = jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS); + public static final String CONTRIBUTOR_COUNT_NO_KEYWORD = + jsonPath(ENTITY_DESCRIPTION, "contributorsCount"); + public static final String ENTITY_PUBLICATION_CONTEXT = + jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); + + public static final String REFERENCE_PUBLICATION_CONTEXT_ID_KEYWORD = + jsonPath(ENTITY_PUBLICATION_CONTEXT, ID, KEYWORD); + + public static final String ENTITY_PUBLICATION_INSTANCE = + jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE); + + public static final String CONTRIBUTOR_ORG_KEYWORD = jsonPath(CONTRIBUTOR_ORGANIZATIONS, KEYWORD); + + public static final String CONTRIBUTORS_AFFILIATION_ID_KEYWORD = + jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, ID, KEYWORD); + + public static final String CONTRIBUTORS_AFFILIATION_LABELS = + jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, LABELS); + public static final String ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION_LABELS_KEYWORD = + multipleFields( + jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, ENGLISH_CODE, KEYWORD), + jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, NYNORSK_CODE, KEYWORD), + jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, BOKMAAL_CODE, KEYWORD), + jsonPath(CONTRIBUTORS_AFFILIATION_LABELS, SAMI_CODE, KEYWORD)); + public static final String ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION = + multipleFields( + CONTRIBUTORS_AFFILIATION_ID_KEYWORD, + ENTITY_DESCRIPTION_CONTRIBUTORS_AFFILIATION_LABELS_KEYWORD); + public static final String CONTRIBUTORS_FIELDS = + multipleFields( + jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ASTERISK), + jsonPath(ENTITY_CONTRIBUTORS, ROLE, ASTERISK), + jsonPath(ENTITY_CONTRIBUTORS, AFFILIATIONS, ASTERISK)); + public static final String CONTRIBUTORS_IDENTITY_ID = + jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ID, KEYWORD); + public static final String CONTRIBUTORS_IDENTITY_NAME_KEYWORD = + multipleFields( + jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, NAME, KEYWORD), CONTRIBUTORS_IDENTITY_ID); + public static final String CONTRIBUTORS_IDENTITY_ORC_ID_KEYWORD = + jsonPath(ENTITY_CONTRIBUTORS, IDENTITY, ORC_ID, KEYWORD); + public static final String SCIENTIFIC_SERIES = + jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, SCIENTIFIC_VALUE, KEYWORD); + public static final String SCIENTIFIC_PUBLISHER = + jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, SCIENTIFIC_VALUE, KEYWORD); + public static final String SCIENTIFIC_OTHER = + jsonPath(ENTITY_PUBLICATION_CONTEXT, SCIENTIFIC_VALUE, KEYWORD); + public static final String SCIENTIFIC_LEVEL_SEARCH_FIELD = + multipleFields(SCIENTIFIC_SERIES, SCIENTIFIC_PUBLISHER, SCIENTIFIC_OTHER); + public static final String COURSE_CODE_KEYWORD = + jsonPath(ENTITY_PUBLICATION_CONTEXT, COURSE, CODE, KEYWORD); + public static final String ENTITY_DESCRIPTION_PUBLICATION_PAGES = + jsonPath(ENTITY_PUBLICATION_INSTANCE, PAGES, PAGES, KEYWORD); + public static final String SUBJECTS = "subjects"; + public static final String ENTITY_DESCRIPTION_PUBLICATION_DATE_YEAR = + jsonPath(ENTITY_DESCRIPTION, PUBLICATION_DATE, YEAR); + public static final String REFERENCE_DOI_KEYWORD = + multipleFields(jsonPath(ENTITY_DESCRIPTION, REFERENCE, DOI, KEYWORD), jsonPath(DOI, KEYWORD)); + public static final String ASSOCIATED_ARTIFACTS_LABELS = + jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, LABELS); + public static final String ASSOCIATED_ARTIFACTS_LICENSE = + multipleFields( + jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, NAME, KEYWORD), + jsonPath(ASSOCIATED_ARTIFACTS, LICENSE, VALUE, KEYWORD), + jsonPath(ASSOCIATED_ARTIFACTS_LABELS, ENGLISH_CODE, KEYWORD), + jsonPath(ASSOCIATED_ARTIFACTS_LABELS, NYNORSK_CODE, KEYWORD), + jsonPath(ASSOCIATED_ARTIFACTS_LABELS, BOKMAAL_CODE, KEYWORD), + jsonPath(ASSOCIATED_ARTIFACTS_LABELS, SAMI_CODE, KEYWORD)); + public static final String PUBLISHER_ID_KEYWORD = jsonPath(PUBLISHER, ID, KEYWORD); + public static final String STATUS_KEYWORD = jsonPath(STATUS, KEYWORD); + public static final String PUBLICATION_CONTEXT_ISBN_LIST = + jsonPath(ENTITY_PUBLICATION_CONTEXT, ISBN_LIST); + public static final String PUBLICATION_CONTEXT_TYPE_KEYWORD = + jsonPath(ENTITY_PUBLICATION_CONTEXT, TYPE, KEYWORD); + public static final String PUBLICATION_INSTANCE_TYPE = + jsonPath(ENTITY_PUBLICATION_INSTANCE, TYPE, KEYWORD); + public static final String PUBLICATION_CONTEXT_PUBLISHER = + multipleFields( + jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, NAME, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, ID, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, PUBLISHER, ISBN_PREFIX, KEYWORD)); + + public static final String ENTITY_DESCRIPTION_REFERENCE_CONTEXT_REFERENCE = + jsonPath(ENTITY_PUBLICATION_CONTEXT, ENTITY_DESCRIPTION, REFERENCE); + public static final String ENTITY_DESCRIPTION_REFERENCE_SERIES = + multipleFields( + jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, "issn", KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, NAME, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, TITLE, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, SERIES, ID, KEYWORD)); + public static final String ENTITY_DESCRIPTION_REFERENCE_JOURNAL = + multipleFields( + jsonPath(ENTITY_PUBLICATION_CONTEXT, NAME, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, ID, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, PRINT_ISSN, KEYWORD), + jsonPath(ENTITY_PUBLICATION_CONTEXT, ONLINE_ISSN, KEYWORD)); + public static final String ENTITY_DESCRIPTION_MAIN_TITLE = + jsonPath(ENTITY_DESCRIPTION, MAIN_TITLE); + public static final String ENTITY_DESCRIPTION_MAIN_TITLE_KEYWORD = + jsonPath(ENTITY_DESCRIPTION_MAIN_TITLE, KEYWORD); + public static final String FUNDINGS_SOURCE_LABELS = jsonPath(FUNDINGS, SOURCE, LABELS); + public static final String HANDLE_KEYWORD = + multipleFields(jsonPath(HANDLE, KEYWORD), jsonPath("additionalIdentifiers", VALUE, KEYWORD)); + public static final String RESOURCE_OWNER_OWNER_AFFILIATION_KEYWORD = + jsonPath(RESOURCE_OWNER, OWNER_AFFILIATION, KEYWORD); + public static final String RESOURCE_OWNER_OWNER_KEYWORD = + jsonPath(RESOURCE_OWNER, OWNER, KEYWORD); + public static final String ENTITY_TAGS = jsonPath(ENTITY_DESCRIPTION, TAGS, KEYWORD); + // ----------------------------------- + public static final String TOP_LEVEL_ORG_ID = jsonPath(TOP_LEVEL_ORGANIZATIONS, ID, KEYWORD); + public static final String ENTITY_ABSTRACT = jsonPath(ENTITY_DESCRIPTION, ABSTRACT); + public static final String ENTITY_DESCRIPTION_LANGUAGE = + jsonPath(ENTITY_DESCRIPTION, LANGUAGE, KEYWORD); + public static final String SCIENTIFIC_INDEX_YEAR = jsonPath(SCIENTIFIC_INDEX, YEAR); + public static final String SCIENTIFIC_INDEX_STATUS_KEYWORD = + jsonPath(SCIENTIFIC_INDEX, STATUS_KEYWORD); + public static final String PUBLICATION_CONTEXT_PATH = + jsonPath(ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); + public static final String ENTITY_DESCRIPTION_REFERENCE_PUBLICATION_CONTEXT_ISSN = + multipleFields( + jsonPath(PUBLICATION_CONTEXT_PATH, ONLINE_ISSN, KEYWORD), + jsonPath(PUBLICATION_CONTEXT_PATH, PRINT_ISSN, KEYWORD), + jsonPath(PUBLICATION_CONTEXT_PATH, SERIES, ONLINE_ISSN, KEYWORD), + jsonPath(PUBLICATION_CONTEXT_PATH, SERIES, PRINT_ISSN, KEYWORD)); + + public static final String FUNDING_IDENTIFIER_KEYWORD = jsonPath(FUNDINGS, IDENTIFIER_KEYWORD); + + public static final String FUNDINGS_IDENTIFIER_FUNDINGS_SOURCE_IDENTIFIER = + multipleFields( + jsonPath(FUNDINGS, IDENTIFIER_KEYWORD), jsonPath(FUNDINGS, SOURCE, IDENTIFIER, KEYWORD)); + + public static final String FUNDINGS_SOURCE_IDENTIFIER_FUNDINGS_SOURCE_LABELS = + multipleFields( + FUNDINGS_IDENTIFIER_FUNDINGS_SOURCE_IDENTIFIER, + jsonPath(FUNDINGS_SOURCE_LABELS, ENGLISH_CODE, KEYWORD), + jsonPath(FUNDINGS_SOURCE_LABELS, NYNORSK_CODE, KEYWORD), + jsonPath(FUNDINGS_SOURCE_LABELS, BOKMAAL_CODE, KEYWORD), + jsonPath(FUNDINGS_SOURCE_LABELS, SAMI_CODE, KEYWORD)); + + public static final String PARENT_PUBLICATION_ID = + multipleFields( + jsonPath(ENTITY_PUBLICATION_INSTANCE, "compliesWith", KEYWORD), + jsonPath(ENTITY_PUBLICATION_INSTANCE, "referencedBy", KEYWORD), + jsonPath(ENTITY_PUBLICATION_INSTANCE, "corrigendumFor", KEYWORD), + jsonPath(ENTITY_PUBLICATION_INSTANCE, "manifestations", ID, KEYWORD), + jsonPath(ENTITY_PUBLICATION_INSTANCE, ID, KEYWORD)); + public static final String PAINLESS = "painless"; + public static final List RESOURCES_AGGREGATIONS = + List.of( + filesHierarchy(), + associatedArtifactsHierarchy(), + entityDescriptionHierarchy(), + fundingSourceHierarchy(), + scientificIndexHierarchy(), + topLevelOrganisationsHierarchy()); + public static final String SEQUENCE = "sequence"; + private static final Map facetResourcePaths1 = + Map.of( + TYPE, "/withAppliedFilter/entityDescription/reference/publicationInstance/type", + SERIES, "/withAppliedFilter/entityDescription/reference/publicationContext/series/id", + LICENSE, "/withAppliedFilter/associatedArtifacts/license"); + private static final Map facetResourcePaths2 = + Map.of( + FILES, "/withAppliedFilter/files", + PUBLISHER, "/withAppliedFilter/entityDescription/reference/publicationContext/publisher", + JOURNAL, "/withAppliedFilter/entityDescription/reference/publicationContext/journal/id", + CONTRIBUTOR, "/withAppliedFilter/entityDescription/contributor/id", + FUNDING_SOURCE, "/withAppliedFilter/fundings/id", + TOP_LEVEL_ORGANIZATION, "/withAppliedFilter/topLevelOrganization/id", + SCIENTIFIC_INDEX, "/withAppliedFilter/scientificIndex/year"); + public static final Map facetResourcePaths = + Stream.of(facetResourcePaths1, facetResourcePaths2) + .flatMap(map -> map.entrySet().stream()) + .sorted(Map.Entry.comparingByValue()) + .collect( + LinkedHashMap::new, + (map, entry) -> map.put(entry.getKey(), entry.getValue()), + LinkedHashMap::putAll); + + @JacocoGenerated + public Constants() {} + + public static NestedAggregationBuilder topLevelOrganisationsHierarchy() { + return nestedBranchBuilder(TOP_LEVEL_ORGANIZATION, TOP_LEVEL_ORGANIZATIONS) + .subAggregation( + branchBuilder(ID, TOP_LEVEL_ORGANIZATIONS, ID, KEYWORD) + .subAggregation(labels(TOP_LEVEL_ORGANIZATIONS))); + } + + public static TermsAggregationBuilder filesHierarchy() { + return branchBuilder(FILES, jsonPath(FILES_STATUS, KEYWORD)); + } + + public static NestedAggregationBuilder associatedArtifactsHierarchy() { + return nestedBranchBuilder(ASSOCIATED_ARTIFACTS, ASSOCIATED_ARTIFACTS) + .subAggregation(license()); + } + + private static TermsAggregationBuilder license() { + return branchBuilder(LICENSE, ASSOCIATED_ARTIFACTS, LICENSE, NAME, KEYWORD) + .subAggregation(labels(jsonPath(ASSOCIATED_ARTIFACTS, LICENSE))) + .subAggregation(getReverseNestedAggregationBuilder()); + } + + private static ReverseNestedAggregationBuilder getReverseNestedAggregationBuilder() { + return AggregationBuilders.reverseNested(ROOT).subAggregation(uniquePublications()); + } + + private static CardinalityAggregationBuilder uniquePublications() { + return AggregationBuilders.cardinality(UNIQUE_PUBLICATIONS).field(jsonPath(ID, KEYWORD)); + } + + private static NestedAggregationBuilder scientificIndexHierarchy() { + return nestedBranchBuilder(SCIENTIFIC_INDEX, SCIENTIFIC_INDEX) + .subAggregation( + branchBuilder(YEAR, SCIENTIFIC_INDEX, YEAR, KEYWORD) + .subAggregation(branchBuilder(NAME, SCIENTIFIC_INDEX, STATUS, KEYWORD))); + } + + public static NestedAggregationBuilder fundingSourceHierarchy() { + return nestedBranchBuilder(FUNDINGS, FUNDINGS) + .subAggregation( + branchBuilder(ID, FUNDINGS, SOURCE, IDENTIFIER, KEYWORD) + .subAggregation(labels(jsonPath(FUNDINGS, SOURCE))) + .subAggregation(getReverseNestedAggregationBuilder())); + } + + public static NestedAggregationBuilder entityDescriptionHierarchy() { + return nestedBranchBuilder(ENTITY_DESCRIPTION, ENTITY_DESCRIPTION) + .subAggregation(contributor()) + .subAggregation(reference()); + } + + private static NestedAggregationBuilder contributor() { + return nestedBranchBuilder(CONTRIBUTOR, ENTITY_DESCRIPTION, CONTRIBUTORS) + .subAggregation( + branchBuilder(ID, ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, ID, KEYWORD) .subAggregation( - branchBuilder(ID, TOP_LEVEL_ORGANIZATIONS, ID, KEYWORD) - .subAggregation(labels(TOP_LEVEL_ORGANIZATIONS))); - } - - public static TermsAggregationBuilder filesHierarchy() { - return branchBuilder(FILES, jsonPath(FILES_STATUS, KEYWORD)); - } - - public static NestedAggregationBuilder associatedArtifactsHierarchy() { - return nestedBranchBuilder(ASSOCIATED_ARTIFACTS, ASSOCIATED_ARTIFACTS) - .subAggregation(license()); - } - - private static TermsAggregationBuilder license() { - return branchBuilder(LICENSE, ASSOCIATED_ARTIFACTS, LICENSE, NAME, KEYWORD) - .subAggregation(labels(jsonPath(ASSOCIATED_ARTIFACTS, LICENSE))) - .subAggregation(getReverseNestedAggregationBuilder()); - } - - private static ReverseNestedAggregationBuilder getReverseNestedAggregationBuilder() { - return AggregationBuilders.reverseNested(ROOT).subAggregation(uniquePublications()); - } - - private static CardinalityAggregationBuilder uniquePublications() { - return AggregationBuilders.cardinality(UNIQUE_PUBLICATIONS).field(jsonPath(ID, KEYWORD)); - } - - private static NestedAggregationBuilder scientificIndexHierarchy() { - return nestedBranchBuilder(SCIENTIFIC_INDEX, SCIENTIFIC_INDEX) - .subAggregation( - branchBuilder(YEAR, SCIENTIFIC_INDEX, YEAR, KEYWORD) - .subAggregation( - branchBuilder(NAME, SCIENTIFIC_INDEX, STATUS, KEYWORD))); - } - - public static NestedAggregationBuilder fundingSourceHierarchy() { - return nestedBranchBuilder(FUNDINGS, FUNDINGS) - .subAggregation( - branchBuilder(ID, FUNDINGS, SOURCE, IDENTIFIER, KEYWORD) - .subAggregation(labels(jsonPath(FUNDINGS, SOURCE))) - .subAggregation(getReverseNestedAggregationBuilder())); - } - - public static NestedAggregationBuilder entityDescriptionHierarchy() { - return nestedBranchBuilder(ENTITY_DESCRIPTION, ENTITY_DESCRIPTION) - .subAggregation(contributor()) - .subAggregation(reference()); - } - - private static NestedAggregationBuilder contributor() { - return nestedBranchBuilder(CONTRIBUTOR, ENTITY_DESCRIPTION, CONTRIBUTORS) - .subAggregation( - branchBuilder(ID, ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, ID, KEYWORD) - .subAggregation( - branchBuilder( - NAME, - ENTITY_DESCRIPTION, - CONTRIBUTORS, - IDENTITY, - NAME, - KEYWORD))); - } - - private static NestedAggregationBuilder reference() { - return nestedBranchBuilder(REFERENCE, ENTITY_DESCRIPTION, REFERENCE) - .subAggregation( - publicationContext() - .subAggregation(publisher()) - .subAggregation(series()) - .subAggregation(journal())) - .subAggregation( - publicationInstance() // Split or just a branch? - .subAggregation(instanceType())); - } - - private static NestedAggregationBuilder publicationContext() { - return nestedBranchBuilder( - PUBLICATION_CONTEXT, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); - } - - private static NestedAggregationBuilder publicationInstance() { - return nestedBranchBuilder( - PUBLICATION_INSTANCE, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE); - } - - private static TermsAggregationBuilder instanceType() { - return branchBuilder( - TYPE, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE, TYPE, KEYWORD); - } - - private static FilterAggregationBuilder series() { - var filterBySeriesType = - filterBranchBuilder( - SERIES, - SERIES_AS_TYPE, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - SERIES, - TYPE, - KEYWORD); - - return filterBySeriesType.subAggregation( + branchBuilder( + NAME, ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, NAME, KEYWORD))); + } + + private static NestedAggregationBuilder reference() { + return nestedBranchBuilder(REFERENCE, ENTITY_DESCRIPTION, REFERENCE) + .subAggregation( + publicationContext() + .subAggregation(publisher()) + .subAggregation(series()) + .subAggregation(journal())) + .subAggregation( + publicationInstance() // Split or just a branch? + .subAggregation(instanceType())); + } + + private static NestedAggregationBuilder publicationContext() { + return nestedBranchBuilder( + PUBLICATION_CONTEXT, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT); + } + + private static NestedAggregationBuilder publicationInstance() { + return nestedBranchBuilder( + PUBLICATION_INSTANCE, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE); + } + + private static TermsAggregationBuilder instanceType() { + return branchBuilder(TYPE, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_INSTANCE, TYPE, KEYWORD); + } + + private static FilterAggregationBuilder series() { + var filterBySeriesType = + filterBranchBuilder( + SERIES, + SERIES_AS_TYPE, + ENTITY_DESCRIPTION, + REFERENCE, + PUBLICATION_CONTEXT, + SERIES, + TYPE, + KEYWORD); + + return filterBySeriesType.subAggregation( + branchBuilder( + ID, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT, SERIES, IDENTIFIER_KEYWORD) + .subAggregation( branchBuilder( - ID, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - SERIES, - IDENTIFIER_KEYWORD) - .subAggregation( - branchBuilder( - NAME, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - SERIES, - NAME, - KEYWORD))); - } - - private static AggregationBuilder journal() { - var filterByJournalType = - filterBranchBuilder( - JOURNAL, - JOURNAL_AS_TYPE, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - TYPE, - KEYWORD); - - var seriesName = - branchBuilder( - NAME, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT, NAME, KEYWORD); - - return filterByJournalType.subAggregation( - branchBuilder( - ID, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - IDENTIFIER_KEYWORD) - .subAggregation(seriesName)); - } - - private static TermsAggregationBuilder publisher() { - return branchBuilder( - PUBLISHER, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - PUBLISHER, - IDENTIFIER_KEYWORD) - .subAggregation( - branchBuilder( - NAME, - ENTITY_DESCRIPTION, - REFERENCE, - PUBLICATION_CONTEXT, - PUBLISHER, - NAME, - KEYWORD)); - } + NAME, + ENTITY_DESCRIPTION, + REFERENCE, + PUBLICATION_CONTEXT, + SERIES, + NAME, + KEYWORD))); + } + + private static AggregationBuilder journal() { + var filterByJournalType = + filterBranchBuilder( + JOURNAL, + JOURNAL_AS_TYPE, + ENTITY_DESCRIPTION, + REFERENCE, + PUBLICATION_CONTEXT, + TYPE, + KEYWORD); + + var seriesName = + branchBuilder(NAME, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT, NAME, KEYWORD); + + return filterByJournalType.subAggregation( + branchBuilder(ID, ENTITY_DESCRIPTION, REFERENCE, PUBLICATION_CONTEXT, IDENTIFIER_KEYWORD) + .subAggregation(seriesName)); + } + + private static TermsAggregationBuilder publisher() { + return branchBuilder( + PUBLISHER, + ENTITY_DESCRIPTION, + REFERENCE, + PUBLICATION_CONTEXT, + PUBLISHER, + IDENTIFIER_KEYWORD) + .subAggregation( + branchBuilder( + NAME, + ENTITY_DESCRIPTION, + REFERENCE, + PUBLICATION_CONTEXT, + PUBLISHER, + NAME, + KEYWORD)); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/LegacyMutator.java b/search-commons/src/main/java/no/unit/nva/search/resource/LegacyMutator.java index 1d3772504..ef075b45a 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/LegacyMutator.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/LegacyMutator.java @@ -7,10 +7,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; - -import no.unit.nva.search.common.records.JsonNodeMutator; - import java.util.List; +import no.unit.nva.search.common.records.JsonNodeMutator; public class LegacyMutator implements JsonNodeMutator { diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceAccessFilter.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceAccessFilter.java index 8091a6860..ec8377e20 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceAccessFilter.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceAccessFilter.java @@ -10,23 +10,19 @@ import static no.unit.nva.search.resource.Constants.CONTRIBUTOR_ORG_KEYWORD; import static no.unit.nva.search.resource.Constants.STATUS_KEYWORD; import static no.unit.nva.search.resource.ResourceParameter.STATISTICS; - import static nva.commons.apigateway.AccessRight.MANAGE_CUSTOMERS; import static nva.commons.apigateway.AccessRight.MANAGE_RESOURCES_ALL; +import java.net.URI; +import java.util.Arrays; import no.unit.nva.search.common.enums.PublicationStatus; import no.unit.nva.search.common.records.FilterBuilder; - import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.UnauthorizedException; - import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermsQueryBuilder; -import java.net.URI; -import java.util.Arrays; - /** * ResourceAccessFilter is a class that filters tickets based on access rights. * @@ -40,120 +36,117 @@ */ public class ResourceAccessFilter implements FilterBuilder { - private static final String CURATING_INST_KEYWORD = CURATING_INSTITUTIONS + DOT + KEYWORD; - private static final String EDITOR_CURATOR_FILTER = "EditorCuratorFilter"; - private static final String EDITOR_FILTER = "EditorFilter"; - private static final String CURATOR_FILTER = "CuratorFilter"; - - private final ResourceSearchQuery searchQuery; - - public ResourceAccessFilter(ResourceSearchQuery query) { - this.searchQuery = query; - this.searchQuery.filters().set(); - } - - private static URI getCurationInstitutionId(RequestInfo requestInfo) - throws UnauthorizedException { - return requestInfo.getTopLevelOrgCristinId().isPresent() - ? requestInfo.getTopLevelOrgCristinId().get() - : requestInfo.getPersonAffiliation(); + private static final String CURATING_INST_KEYWORD = CURATING_INSTITUTIONS + DOT + KEYWORD; + private static final String EDITOR_CURATOR_FILTER = "EditorCuratorFilter"; + private static final String EDITOR_FILTER = "EditorFilter"; + private static final String CURATOR_FILTER = "CuratorFilter"; + + private final ResourceSearchQuery searchQuery; + + public ResourceAccessFilter(ResourceSearchQuery query) { + this.searchQuery = query; + this.searchQuery.filters().set(); + } + + private static URI getCurationInstitutionId(RequestInfo requestInfo) + throws UnauthorizedException { + return requestInfo.getTopLevelOrgCristinId().isPresent() + ? requestInfo.getTopLevelOrgCristinId().get() + : requestInfo.getPersonAffiliation(); + } + + @Override + public ResourceSearchQuery apply() { + return searchQuery; + } + + @Override + public ResourceSearchQuery fromRequestInfo(RequestInfo requestInfo) throws UnauthorizedException { + + return customerCurationInstitutions(requestInfo) + .requiredStatus(PUBLISHED, PUBLISHED_METADATA) + .apply(); + } + + /** + * Filter on Required Status. + * + *

Only STATUES specified here will be available for the Query. + * + *

This is to avoid the Query to return documents that are not available for the user. + * + *

See {@link PublicationStatus} for available values. + * + * @param publicationStatus the required statues + * @return {@link ResourceAccessFilter} (builder pattern) + */ + public ResourceAccessFilter requiredStatus(PublicationStatus... publicationStatus) { + final var values = + Arrays.stream(publicationStatus) + .filter(this::isStatusAllowed) + .map(PublicationStatus::toString) + .toArray(String[]::new); + final var filter = new TermsQueryBuilder(STATUS_KEYWORD, values).queryName(STATUS); + this.searchQuery.filters().add(filter); + return this; + } + + /** + * Filter on organization and curationInstitutions. + * + *

Only documents belonging to organization specified are searchable (for the user) + * + * @param requestInfo fetches TopLevelOrgCristinId PersonAffiliation + * @return {@link ResourceAccessFilter} (builder pattern) + */ + public ResourceAccessFilter customerCurationInstitutions(RequestInfo requestInfo) + throws UnauthorizedException { + if (isCurator() && isStatisticsQuery()) { + return this; } - - @Override - public ResourceSearchQuery apply() { - return searchQuery; - } - - @Override - public ResourceSearchQuery fromRequestInfo(RequestInfo requestInfo) - throws UnauthorizedException { - - return customerCurationInstitutions(requestInfo) - .requiredStatus(PUBLISHED, PUBLISHED_METADATA) - .apply(); + final var filter = + QueryBuilders.boolQuery().minimumShouldMatch(1).queryName(EDITOR_CURATOR_FILTER); + var curationInstitutionId = getCurationInstitutionId(requestInfo).toString(); + if (isCurator()) { + filter.should(getCuratingInstitutionAccessFilter(curationInstitutionId)); + filter.should(getContributingOrganisationAccessFilter(curationInstitutionId)); + } else if (isEditor()) { + filter.should(getContributingOrganisationAccessFilter(curationInstitutionId)); } - - /** - * Filter on Required Status. - * - *

Only STATUES specified here will be available for the Query. - * - *

This is to avoid the Query to return documents that are not available for the user. - * - *

See {@link PublicationStatus} for available values. - * - * @param publicationStatus the required statues - * @return {@link ResourceAccessFilter} (builder pattern) - */ - public ResourceAccessFilter requiredStatus(PublicationStatus... publicationStatus) { - final var values = - Arrays.stream(publicationStatus) - .filter(this::isStatusAllowed) - .map(PublicationStatus::toString) - .toArray(String[]::new); - final var filter = new TermsQueryBuilder(STATUS_KEYWORD, values).queryName(STATUS); - this.searchQuery.filters().add(filter); - return this; - } - - /** - * Filter on organization and curationInstitutions. - * - *

Only documents belonging to organization specified are searchable (for the user) - * - * @param requestInfo fetches TopLevelOrgCristinId PersonAffiliation - * @return {@link ResourceAccessFilter} (builder pattern) - */ - public ResourceAccessFilter customerCurationInstitutions(RequestInfo requestInfo) - throws UnauthorizedException { - if (isCurator() && isStatisticsQuery()) { - return this; - } - final var filter = - QueryBuilders.boolQuery().minimumShouldMatch(1).queryName(EDITOR_CURATOR_FILTER); - var curationInstitutionId = getCurationInstitutionId(requestInfo).toString(); - if (isCurator()) { - filter.should(getCuratingInstitutionAccessFilter(curationInstitutionId)); - filter.should(getContributingOrganisationAccessFilter(curationInstitutionId)); - } else if (isEditor()) { - filter.should(getContributingOrganisationAccessFilter(curationInstitutionId)); - } - if (!filter.hasClauses()) { - throw new UnauthorizedException(); - } - this.searchQuery.filters().add(filter); - return this; - } - - private QueryBuilder getContributingOrganisationAccessFilter(String institutionId) { - return QueryBuilders.termQuery(CONTRIBUTOR_ORG_KEYWORD, institutionId) - .queryName(EDITOR_FILTER); - } - - private QueryBuilder getCuratingInstitutionAccessFilter(String institutionId) { - return QueryBuilders.termQuery(CURATING_INST_KEYWORD, institutionId) - .queryName(CURATOR_FILTER); - } - - /** - * Only Editors are allowed to see UNPUBLISHED publications - * - * @param publicationStatus status to check - * @return true if allowed - */ - private boolean isStatusAllowed(PublicationStatus publicationStatus) { - return isEditor() || publicationStatus != UNPUBLISHED; - } - - private boolean isStatisticsQuery() { - return searchQuery.parameters().isPresent(STATISTICS); - } - - private boolean isCurator() { - return searchQuery.hasAccessRights(MANAGE_CUSTOMERS); - } - - private boolean isEditor() { - return searchQuery.hasAccessRights(MANAGE_RESOURCES_ALL); + if (!filter.hasClauses()) { + throw new UnauthorizedException(); } + this.searchQuery.filters().add(filter); + return this; + } + + private QueryBuilder getContributingOrganisationAccessFilter(String institutionId) { + return QueryBuilders.termQuery(CONTRIBUTOR_ORG_KEYWORD, institutionId).queryName(EDITOR_FILTER); + } + + private QueryBuilder getCuratingInstitutionAccessFilter(String institutionId) { + return QueryBuilders.termQuery(CURATING_INST_KEYWORD, institutionId).queryName(CURATOR_FILTER); + } + + /** + * Only Editors are allowed to see UNPUBLISHED publications + * + * @param publicationStatus status to check + * @return true if allowed + */ + private boolean isStatusAllowed(PublicationStatus publicationStatus) { + return isEditor() || publicationStatus != UNPUBLISHED; + } + + private boolean isStatisticsQuery() { + return searchQuery.parameters().isPresent(STATISTICS); + } + + private boolean isCurator() { + return searchQuery.hasAccessRights(MANAGE_CUSTOMERS); + } + + private boolean isEditor() { + return searchQuery.hasAccessRights(MANAGE_RESOURCES_ALL); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceClient.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceClient.java index 38fecfd6d..23cb334c7 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceClient.java @@ -5,21 +5,18 @@ import static no.unit.nva.search.common.records.SwsResponse.SwsResponseBuilder.swsResponseBuilder; import com.fasterxml.jackson.core.JsonProcessingException; - -import no.unit.nva.search.common.OpenSearchClient; -import no.unit.nva.search.common.jwt.CachedJwtProvider; -import no.unit.nva.search.common.records.SwsResponse; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.attempt.FunctionWithException; -import nva.commons.secrets.SecretsReader; - import java.net.http.HttpClient; import java.net.http.HttpResponse; import java.time.Duration; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.function.BinaryOperator; +import no.unit.nva.search.common.OpenSearchClient; +import no.unit.nva.search.common.jwt.CachedJwtProvider; +import no.unit.nva.search.common.records.SwsResponse; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.attempt.FunctionWithException; +import nva.commons.secrets.SecretsReader; /** * Client for searching resources. @@ -28,57 +25,54 @@ */ public class ResourceClient extends OpenSearchClient { - @SuppressWarnings("PMD.DoNotUseThreads") - private static final ExecutorService executorService = Executors.newFixedThreadPool(3); + @SuppressWarnings("PMD.DoNotUseThreads") + private static final ExecutorService executorService = Executors.newFixedThreadPool(3); - private final UserSettingsClient userSettingsClient; + private final UserSettingsClient userSettingsClient; - public ResourceClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { - this(client, cachedJwtProvider, new UserSettingsClient(client, cachedJwtProvider)); - } + public ResourceClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { + this(client, cachedJwtProvider, new UserSettingsClient(client, cachedJwtProvider)); + } - public ResourceClient( - HttpClient client, - CachedJwtProvider jwtProvider, - UserSettingsClient userSettingsClient) { - super(client, jwtProvider); - this.userSettingsClient = userSettingsClient; - } + public ResourceClient( + HttpClient client, CachedJwtProvider jwtProvider, UserSettingsClient userSettingsClient) { + super(client, jwtProvider); + this.userSettingsClient = userSettingsClient; + } - @JacocoGenerated - public static ResourceClient defaultClient() { - var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); - var httpClient = - HttpClient.newBuilder() - .executor(executorService) - .version(HttpClient.Version.HTTP_2) - .connectTimeout(Duration.ofSeconds(10)) - .build(); - return new ResourceClient(httpClient, cachedJwtProvider); - } + @JacocoGenerated + public static ResourceClient defaultClient() { + var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); + var httpClient = + HttpClient.newBuilder() + .executor(executorService) + .version(HttpClient.Version.HTTP_2) + .connectTimeout(Duration.ofSeconds(10)) + .build(); + return new ResourceClient(httpClient, cachedJwtProvider); + } - @Override - public SwsResponse doSearch(ResourceSearchQuery query) { - return super.doSearch(query.withUserSettings(userSettingsClient)); - } + @Override + public SwsResponse doSearch(ResourceSearchQuery query) { + return super.doSearch(query.withUserSettings(userSettingsClient)); + } - @Override - protected SwsResponse jsonToResponse(HttpResponse response) - throws JsonProcessingException { - return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); - } + @Override + protected SwsResponse jsonToResponse(HttpResponse response) + throws JsonProcessingException { + return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); + } - @Override - protected BinaryOperator responseAccumulator() { - return (a, b) -> swsResponseBuilder().merge(a).merge(b).build(); - } + @Override + protected BinaryOperator responseAccumulator() { + return (a, b) -> swsResponseBuilder().merge(a).merge(b).build(); + } - @Override - protected FunctionWithException - logAndReturnResult() { - return result -> { - logger.info(buildLogInfo(result)); - return result; - }; - } + @Override + protected FunctionWithException logAndReturnResult() { + return result -> { + logger.info(buildLogInfo(result)); + return result; + }; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceParameter.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceParameter.java index 46d5955eb..dcf7a5abb 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceParameter.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceParameter.java @@ -1,5 +1,6 @@ package no.unit.nva.search.resource; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.CHAR_UNDERSCORE; import static no.unit.nva.constants.Words.COLON; import static no.unit.nva.constants.Words.CREATED_DATE; @@ -88,17 +89,6 @@ import static no.unit.nva.search.resource.Constants.SUBJECTS; import static no.unit.nva.search.resource.Constants.TOP_LEVEL_ORG_ID; -import static java.util.Objects.nonNull; - -import no.unit.nva.search.common.enums.FieldOperator; -import no.unit.nva.search.common.enums.ParameterKey; -import no.unit.nva.search.common.enums.ParameterKind; -import no.unit.nva.search.common.enums.ValueEncoding; - -import nva.commons.core.JacocoGenerated; - -import org.apache.commons.text.CaseUtils; - import java.util.Arrays; import java.util.LinkedHashSet; import java.util.Locale; @@ -106,6 +96,12 @@ import java.util.StringJoiner; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.search.common.enums.FieldOperator; +import no.unit.nva.search.common.enums.ParameterKey; +import no.unit.nva.search.common.enums.ParameterKind; +import no.unit.nva.search.common.enums.ValueEncoding; +import nva.commons.core.JacocoGenerated; +import org.apache.commons.text.CaseUtils; /** * Enum for all the parameters that can be used to query the search index. This enum needs to diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSearchQuery.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSearchQuery.java index 6677387b8..3a55d13d3 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSearchQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSearchQuery.java @@ -1,5 +1,6 @@ package no.unit.nva.search.resource; +import static java.lang.String.format; import static no.unit.nva.constants.Defaults.DEFAULT_OFFSET; import static no.unit.nva.constants.Defaults.DEFAULT_VALUE_PER_PAGE; import static no.unit.nva.constants.ErrorMessages.INVALID_VALUE_WITH_SORT; @@ -34,14 +35,21 @@ import static no.unit.nva.search.resource.ResourceParameter.SIZE; import static no.unit.nva.search.resource.ResourceParameter.SORT; import static no.unit.nva.search.resource.ResourceSort.INVALID; - import static nva.commons.core.attempt.Try.attempt; import static nva.commons.core.paths.UriWrapper.fromUri; - import static org.opensearch.index.query.QueryBuilders.matchQuery; -import static java.lang.String.format; - +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.constants.Words; import no.unit.nva.search.common.AsType; import no.unit.nva.search.common.OpenSearchClient; @@ -50,27 +58,13 @@ import no.unit.nva.search.common.SearchQuery; import no.unit.nva.search.common.enums.SortKey; import no.unit.nva.search.common.records.HttpResponseFormatter; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.sort.SortOrder; -import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * ResourceSearchQuery is a query for searching resources. * @@ -80,319 +74,313 @@ */ @SuppressWarnings("PMD.GodClass") public final class ResourceSearchQuery extends SearchQuery { - private static final String EXCLUDED_RESOURCE_FIELDS = "entityDescription.contributors"; - private final ResourceStreamBuilders streamBuilders; - private final ResourceAccessFilter filterBuilder; - private final Map additionalQueryParameters = new HashMap<>(); - private UserSettingsClient userSettingsClient; - - private ResourceSearchQuery() { - super(); - assignStatusImpossibleWhiteList(); - setAlwaysExcludedFields(GLOBAL_EXCLUDED_FIELDS); - streamBuilders = new ResourceStreamBuilders(parameters()); - filterBuilder = new ResourceAccessFilter(this); - } - - public static ResourceParameterValidator builder() { - return new ResourceParameterValidator(); + private static final String EXCLUDED_RESOURCE_FIELDS = "entityDescription.contributors"; + private final ResourceStreamBuilders streamBuilders; + private final ResourceAccessFilter filterBuilder; + private final Map additionalQueryParameters = new HashMap<>(); + private UserSettingsClient userSettingsClient; + + private ResourceSearchQuery() { + super(); + assignStatusImpossibleWhiteList(); + setAlwaysExcludedFields(GLOBAL_EXCLUDED_FIELDS); + streamBuilders = new ResourceStreamBuilders(parameters()); + filterBuilder = new ResourceAccessFilter(this); + } + + public static ResourceParameterValidator builder() { + return new ResourceParameterValidator(); + } + + private static boolean missingIdentifier(String sortString) { + return !sortString.contains(IDENTIFIER); + } + + /** + * Calculate the boost for a promoted publication. (4.14 down to 3.14 (PI)) + * + * @param i index of the current publication + * @param size total number of promoted publications + * @return the boost value + */ + private static float calculateBoostValue(AtomicInteger i, int size) { + return PI + 1F - ((float) i.getAndIncrement() / size); + } + + /** + * Add a (default) filter to the query that will never match any document. + * + *

This whitelist the ResourceQuery from any forgetful developer (me) + * + * @apiNote In order to return any results, withRequiredStatus must be set + */ + private void assignStatusImpossibleWhiteList() { + filters() + .set(new TermsQueryBuilder(STATUS_KEYWORD, UUID.randomUUID().toString()).queryName(STATUS)); + } + + @Override + protected ResourceParameter keyFields() { + return NODES_SEARCHED; + } + + @Override + public AsType sort() { + var sortString = parameters().get(SORT).toString(); + if (missingIdentifier(sortString)) { + if (sortString.isBlank()) { + parameters().set(SORT, IDENTIFIER); + } else { + parameters().set(SORT, String.join(COMMA, sortString, IDENTIFIER)); + } } - - private static boolean missingIdentifier(String sortString) { - return !sortString.contains(IDENTIFIER); + return parameters().get(SORT); + } + + @Override + protected String[] exclude() { + return Stream.concat( + parameters().get(NODES_EXCLUDED).asSplitStream(COMMA), getExcludedFields().stream()) + .distinct() + .toArray(String[]::new); + } + + @Override + protected String[] include() { + return getIncludedFields().toArray(String[]::new); + } + + @Override + protected ResourceParameter keyAggregation() { + return AGGREGATION; + } + + @Override + protected ResourceParameter keySearchAfter() { + return SEARCH_AFTER; + } + + @Override + protected ResourceParameter toKey(String keyName) { + return ResourceParameter.keyFromString(keyName); + } + + @Override + protected SortKey toSortKey(String sortName) { + return ResourceSort.fromSortKey(sortName); + } + + @Override + protected List builderAggregations() { + return RESOURCES_AGGREGATIONS; + } + + @JacocoGenerated // default value shouldn't happen, (developer have forgotten to handle a key) + @Override + protected Stream> builderCustomQueryStream( + ResourceParameter key) { + return switch (key) { + case FUNDING -> streamBuilders.fundingQuery(key); + case CRISTIN_IDENTIFIER -> streamBuilders.additionalIdentifierQuery(key, CRISTIN_AS_TYPE); + case SCOPUS_IDENTIFIER -> streamBuilders.additionalIdentifierQuery(key, SCOPUS_AS_TYPE); + case SCIENTIFIC_VALUE -> streamBuilders.scientificValueQuery(key); + case TOP_LEVEL_ORGANIZATION, UNIT, UNIT_NOT -> streamBuilders.subUnitIncludedQuery(key); + case UNIDENTIFIED_NORWEGIAN -> streamBuilders.unIdentifiedNorwegians(key); + case UNIDENTIFIED_CONTRIBUTOR_INSTITUTION -> + streamBuilders.unIdentifiedContributorOrInstitution(key); + case SEARCH_ALL -> + streamBuilders.searchAllWithBoostsQuery( + mapOfPathAndBoost(parameters().get(NODES_SEARCHED))); + default -> throw new IllegalArgumentException("unhandled key -> " + key.name()); + }; + } + + @Override + public > HttpResponseFormatter doSearch( + OpenSearchClient queryClient) { + return super.doSearch(queryClient); + } + + @Override + protected AsType from() { + return parameters().get(FROM); + } + + @Override + protected AsType size() { + return parameters().get(SIZE); + } + + @Override + protected Map facetPaths() { + return facetResourcePaths; + } + + @Override + protected BoolQueryBuilder builderMainQuery() { + var queryBuilder = super.builderMainQuery(); + if (isLookingForOneContributor()) { + addPromotedPublications(this.userSettingsClient, queryBuilder); } - - /** - * Calculate the boost for a promoted publication. (4.14 down to 3.14 (PI)) - * - * @param i index of the current publication - * @param size total number of promoted publications - * @return the boost value - */ - private static float calculateBoostValue(AtomicInteger i, int size) { - return PI + 1F - ((float) i.getAndIncrement() / size); + return queryBuilder; + } + + private boolean isLookingForOneContributor() { + return parameters().get(CONTRIBUTOR).asSplitStream(COMMA).count() == 1; + } + + private void addPromotedPublications(UserSettingsClient client, BoolQueryBuilder builder) { + + var result = attempt(() -> client.doSearch(this)); + if (result.isSuccess()) { + AtomicInteger i = new AtomicInteger(); + var promoted = result.get().promotedPublications(); + promoted.forEach( + identifier -> + builder.should( + matchQuery(IDENTIFIER_KEYWORD, identifier) + .boost(calculateBoostValue(i, promoted.size())))); } - - /** - * Add a (default) filter to the query that will never match any document. - * - *

This whitelist the ResourceQuery from any forgetful developer (me) - * - * @apiNote In order to return any results, withRequiredStatus must be set - */ - private void assignStatusImpossibleWhiteList() { - filters() - .set( - new TermsQueryBuilder(STATUS_KEYWORD, UUID.randomUUID().toString()) - .queryName(STATUS)); + } + + @Override + public URI openSearchUri() { + return fromUri(infrastructureApiUri) + .addChild(Words.RESOURCES, Words.SEARCH) + .addQueryParameters(queryParameters()) + .getUri(); + } + + private Map queryParameters() { + return additionalQueryParameters; + } + + public ResourceAccessFilter withFilter() { + return filterBuilder; + } + + public ResourceSearchQuery withScrollTime(String time) { + this.additionalQueryParameters.put("scroll", time); + return this; + } + + public ResourceSearchQuery withUserSettings(UserSettingsClient userSettingsClient) { + this.userSettingsClient = userSettingsClient; + return this; + } + + /** + * ResourceParameterValidator is a class that validates parameters for a ResourceSearchQuery. + * + * @author Stig Norland + */ + public static class ResourceParameterValidator + extends ParameterValidator { + + ResourceParameterValidator() { + super(new ResourceSearchQuery()); } @Override - protected ResourceParameter keyFields() { - return NODES_SEARCHED; + protected void assignDefaultValues() { + requiredMissing() + .forEach( + key -> { + switch (key) { + case FROM -> setValue(key.name(), DEFAULT_OFFSET); + case SIZE -> setValue(key.name(), DEFAULT_VALUE_PER_PAGE); + case SORT -> setValue(key.name(), DEFAULT_RESOURCE_SORT_FIELDS); + case AGGREGATION -> setValue(key.name(), NONE); + default -> { + /* ignore and continue */ + } + } + }); } + @JacocoGenerated @Override - public AsType sort() { - var sortString = parameters().get(SORT).toString(); - if (missingIdentifier(sortString)) { - if (sortString.isBlank()) { - parameters().set(SORT, IDENTIFIER); - } else { - parameters().set(SORT, String.join(COMMA, sortString, IDENTIFIER)); - } + protected void applyRulesAfterValidation() { + // convert page to offset if offset is not set + if (query.parameters().isPresent(PAGE)) { + if (query.parameters().isPresent(FROM)) { + var page = query.parameters().get(PAGE).as(); + var perPage = query.parameters().get(SIZE).as(); + query.parameters().set(FROM, String.valueOf(page.longValue() * perPage.longValue())); } - return parameters().get(SORT); - } - - @Override - protected String[] exclude() { - return Stream.concat( - parameters().get(NODES_EXCLUDED).asSplitStream(COMMA), - getExcludedFields().stream()) - .distinct() - .toArray(String[]::new); - } - - @Override - protected String[] include() { - return getIncludedFields().toArray(String[]::new); - } - - @Override - protected ResourceParameter keyAggregation() { - return AGGREGATION; + query.parameters().remove(PAGE); + } } @Override - protected ResourceParameter keySearchAfter() { - return SEARCH_AFTER; + protected Collection validKeys() { + return RESOURCE_PARAMETER_SET.stream().map(ResourceParameter::asLowerCase).toList(); } @Override - protected ResourceParameter toKey(String keyName) { - return ResourceParameter.keyFromString(keyName); + protected boolean isKeyValid(String keyName) { + return ResourceParameter.keyFromString(keyName) != ResourceParameter.INVALID; } @Override - protected SortKey toSortKey(String sortName) { - return ResourceSort.fromSortKey(sortName); + protected void validateSortKeyName(String name) { + var nameSort = name.split(COLON_OR_SPACE); + if (nameSort.length == NAME_AND_SORT_LENGTH) { + SortOrder.fromString(nameSort[1]); + } + if (nameSort.length > NAME_AND_SORT_LENGTH) { + throw new IllegalArgumentException(TOO_MANY_ARGUMENTS + name); + } + + if (ResourceSort.fromSortKey(nameSort[0]) == INVALID) { + throw new IllegalArgumentException( + INVALID_VALUE_WITH_SORT.formatted(name, ResourceSort.validSortKeys())); + } } @Override - protected List builderAggregations() { - return RESOURCES_AGGREGATIONS; + protected void setValue(String key, String value) { + var qpKey = ResourceParameter.keyFromString(key); + var decodedValue = getDecodedValue(qpKey, value); + switch (qpKey) { + case INVALID -> invalidKeys.add(key); + case UNIT, UNIT_NOT, TOP_LEVEL_ORGANIZATION -> + mergeToKey(qpKey, identifierToCristinId(decodedValue)); + case CONTRIBUTOR, CONTRIBUTOR_NOT -> + mergeToKey(qpKey, identifierToCristinPersonId(decodedValue)); + case SEARCH_AFTER, FROM, SIZE, PAGE, AGGREGATION -> + query.parameters().set(qpKey, decodedValue); + case NODES_SEARCHED -> query.parameters().set(qpKey, ignoreInvalidFields(decodedValue)); + case SORT -> mergeToKey(SORT, trimSpace(decodedValue)); + case SORT_ORDER -> mergeToKey(SORT, decodedValue); + default -> mergeToKey(qpKey, decodedValue); + } } - @JacocoGenerated // default value shouldn't happen, (developer have forgotten to handle a key) - @Override - protected Stream> builderCustomQueryStream( - ResourceParameter key) { - return switch (key) { - case FUNDING -> streamBuilders.fundingQuery(key); - case CRISTIN_IDENTIFIER -> - streamBuilders.additionalIdentifierQuery(key, CRISTIN_AS_TYPE); - case SCOPUS_IDENTIFIER -> streamBuilders.additionalIdentifierQuery(key, SCOPUS_AS_TYPE); - case SCIENTIFIC_VALUE -> streamBuilders.scientificValueQuery(key); - case TOP_LEVEL_ORGANIZATION, UNIT, UNIT_NOT -> streamBuilders.subUnitIncludedQuery(key); - case UNIDENTIFIED_NORWEGIAN -> streamBuilders.unIdentifiedNorwegians(key); - case UNIDENTIFIED_CONTRIBUTOR_INSTITUTION -> - streamBuilders.unIdentifiedContributorOrInstitution(key); - case SEARCH_ALL -> - streamBuilders.searchAllWithBoostsQuery( - mapOfPathAndBoost(parameters().get(NODES_SEARCHED))); - default -> throw new IllegalArgumentException("unhandled key -> " + key.name()); - }; + private String identifierToCristinId(String decodedValue) { + return Arrays.stream(decodedValue.split(COMMA)) + .map(value -> identifierToUri(value, CRISTIN_ORGANIZATION_PATH)) + .collect(Collectors.joining(COMMA)); } - @Override - public > - HttpResponseFormatter doSearch(OpenSearchClient queryClient) { - return super.doSearch(queryClient); + private String identifierToCristinPersonId(String decodedValue) { + return Arrays.stream(decodedValue.split(COMMA)) + .map(value -> identifierToUri(value, CRISTIN_PERSON_PATH)) + .collect(Collectors.joining(COMMA)); } - @Override - protected AsType from() { - return parameters().get(FROM); + private String currentHost() { + return HTTPS + query.getNvaSearchApiUri().getHost(); } - @Override - protected AsType size() { - return parameters().get(SIZE); + private String identifierToUri(String decodedValue, String uriPath) { + return isUriId(decodedValue) + ? decodedValue + : format("%s%s%s", currentHost(), uriPath, decodedValue); } - @Override - protected Map facetPaths() { - return facetResourcePaths; - } - - @Override - protected BoolQueryBuilder builderMainQuery() { - var queryBuilder = super.builderMainQuery(); - if (isLookingForOneContributor()) { - addPromotedPublications(this.userSettingsClient, queryBuilder); - } - return queryBuilder; - } - - private boolean isLookingForOneContributor() { - return parameters().get(CONTRIBUTOR).asSplitStream(COMMA).count() == 1; - } - - private void addPromotedPublications(UserSettingsClient client, BoolQueryBuilder builder) { - - var result = attempt(() -> client.doSearch(this)); - if (result.isSuccess()) { - AtomicInteger i = new AtomicInteger(); - var promoted = result.get().promotedPublications(); - promoted.forEach( - identifier -> - builder.should( - matchQuery(IDENTIFIER_KEYWORD, identifier) - .boost(calculateBoostValue(i, promoted.size())))); - } - } - - @Override - public URI openSearchUri() { - return fromUri(infrastructureApiUri) - .addChild(Words.RESOURCES, Words.SEARCH) - .addQueryParameters(queryParameters()) - .getUri(); - } - - private Map queryParameters() { - return additionalQueryParameters; - } - - public ResourceAccessFilter withFilter() { - return filterBuilder; - } - - public ResourceSearchQuery withScrollTime(String time) { - this.additionalQueryParameters.put("scroll", time); - return this; - } - - public ResourceSearchQuery withUserSettings(UserSettingsClient userSettingsClient) { - this.userSettingsClient = userSettingsClient; - return this; - } - - /** - * ResourceParameterValidator is a class that validates parameters for a ResourceSearchQuery. - * - * @author Stig Norland - */ - public static class ResourceParameterValidator - extends ParameterValidator { - - ResourceParameterValidator() { - super(new ResourceSearchQuery()); - } - - @Override - protected void assignDefaultValues() { - requiredMissing() - .forEach( - key -> { - switch (key) { - case FROM -> setValue(key.name(), DEFAULT_OFFSET); - case SIZE -> setValue(key.name(), DEFAULT_VALUE_PER_PAGE); - case SORT -> setValue(key.name(), DEFAULT_RESOURCE_SORT_FIELDS); - case AGGREGATION -> setValue(key.name(), NONE); - default -> { - /* ignore and continue */ - } - } - }); - } - - @JacocoGenerated - @Override - protected void applyRulesAfterValidation() { - // convert page to offset if offset is not set - if (query.parameters().isPresent(PAGE)) { - if (query.parameters().isPresent(FROM)) { - var page = query.parameters().get(PAGE).as(); - var perPage = query.parameters().get(SIZE).as(); - query.parameters() - .set(FROM, String.valueOf(page.longValue() * perPage.longValue())); - } - query.parameters().remove(PAGE); - } - } - - @Override - protected Collection validKeys() { - return RESOURCE_PARAMETER_SET.stream().map(ResourceParameter::asLowerCase).toList(); - } - - @Override - protected boolean isKeyValid(String keyName) { - return ResourceParameter.keyFromString(keyName) != ResourceParameter.INVALID; - } - - @Override - protected void validateSortKeyName(String name) { - var nameSort = name.split(COLON_OR_SPACE); - if (nameSort.length == NAME_AND_SORT_LENGTH) { - SortOrder.fromString(nameSort[1]); - } - if (nameSort.length > NAME_AND_SORT_LENGTH) { - throw new IllegalArgumentException(TOO_MANY_ARGUMENTS + name); - } - - if (ResourceSort.fromSortKey(nameSort[0]) == INVALID) { - throw new IllegalArgumentException( - INVALID_VALUE_WITH_SORT.formatted(name, ResourceSort.validSortKeys())); - } - } - - @Override - protected void setValue(String key, String value) { - var qpKey = ResourceParameter.keyFromString(key); - var decodedValue = getDecodedValue(qpKey, value); - switch (qpKey) { - case INVALID -> invalidKeys.add(key); - case UNIT, UNIT_NOT, TOP_LEVEL_ORGANIZATION -> - mergeToKey(qpKey, identifierToCristinId(decodedValue)); - case CONTRIBUTOR, CONTRIBUTOR_NOT -> - mergeToKey(qpKey, identifierToCristinPersonId(decodedValue)); - case SEARCH_AFTER, FROM, SIZE, PAGE, AGGREGATION -> - query.parameters().set(qpKey, decodedValue); - case NODES_SEARCHED -> - query.parameters().set(qpKey, ignoreInvalidFields(decodedValue)); - case SORT -> mergeToKey(SORT, trimSpace(decodedValue)); - case SORT_ORDER -> mergeToKey(SORT, decodedValue); - default -> mergeToKey(qpKey, decodedValue); - } - } - - private String identifierToCristinId(String decodedValue) { - return Arrays.stream(decodedValue.split(COMMA)) - .map(value -> identifierToUri(value, CRISTIN_ORGANIZATION_PATH)) - .collect(Collectors.joining(COMMA)); - } - - private String identifierToCristinPersonId(String decodedValue) { - return Arrays.stream(decodedValue.split(COMMA)) - .map(value -> identifierToUri(value, CRISTIN_PERSON_PATH)) - .collect(Collectors.joining(COMMA)); - } - - private String currentHost() { - return HTTPS + query.getNvaSearchApiUri().getHost(); - } - - private String identifierToUri(String decodedValue, String uriPath) { - return isUriId(decodedValue) - ? decodedValue - : format("%s%s%s", currentHost(), uriPath, decodedValue); - } - - private boolean isUriId(String decodedValue) { - return decodedValue.startsWith(HTTPS); - } + private boolean isUriId(String decodedValue) { + return decodedValue.startsWith(HTTPS); } + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSort.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSort.java index 605236b74..b372404b2 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSort.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceSort.java @@ -7,20 +7,17 @@ import static no.unit.nva.constants.Words.YEAR; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_PIPE; import static no.unit.nva.search.resource.Constants.IDENTIFIER_KEYWORD; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import no.unit.nva.constants.Words; -import no.unit.nva.search.common.enums.SortKey; - -import org.apache.commons.text.CaseUtils; - import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashSet; import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.constants.Words; +import no.unit.nva.search.common.enums.SortKey; +import org.apache.commons.text.CaseUtils; /** * Enum for sorting resources. diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceStreamBuilders.java b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceStreamBuilders.java index 12c3ea9aa..a54fb10c9 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/ResourceStreamBuilders.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/ResourceStreamBuilders.java @@ -35,7 +35,6 @@ import static no.unit.nva.search.resource.ResourceParameter.EXCLUDE_SUBUNITS; import static no.unit.nva.search.resource.ResourceParameter.SEARCH_ALL; import static no.unit.nva.search.resource.ResourceParameter.TITLE; - import static org.opensearch.index.query.QueryBuilders.boolQuery; import static org.opensearch.index.query.QueryBuilders.existsQuery; import static org.opensearch.index.query.QueryBuilders.matchPhrasePrefixQuery; @@ -45,20 +44,18 @@ import static org.opensearch.index.query.QueryBuilders.termQuery; import static org.opensearch.index.query.QueryBuilders.termsQuery; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.search.common.QueryKeys; import no.unit.nva.search.common.builder.FuzzyKeywordQuery; import no.unit.nva.search.common.constant.Functions; - import org.apache.lucene.search.join.ScoreMode; import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; import org.opensearch.join.query.HasParentQueryBuilder; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Stream builders for resource queries. * @@ -66,156 +63,147 @@ */ public class ResourceStreamBuilders { - public static final String COUNTRY_CODE_PATH = - jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, AFFILIATIONS, COUNTRY_CODE, KEYWORD); - public static final String CONTRIBUTOR_ROLE_PATH = - jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, ROLE, TYPE, KEYWORD); - public static final String VERIFICATION_STATUS_PATH = - jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, VERIFICATION_STATUS); - public static final String VERIFICATION_STATUS_KEYWORD = - jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, VERIFICATION_STATUS, KEYWORD); - public static final String ADDITIONAL_IDENTIFIERS_VALUE_PATH = - jsonPath(ADDITIONAL_IDENTIFIERS, VALUE, KEYWORD); - public static final String ADDITIONAL_IDENTIFIERS_NAME_PATH = - jsonPath(ADDITIONAL_IDENTIFIERS, SOURCE_NAME, KEYWORD); - public static final String FUNDING_SOURCE_IDENTIFIER = - jsonPath(FUNDINGS, SOURCE, IDENTIFIER, KEYWORD); - public static final String CONTRIBUTOR_AFFILIATIONS = - jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, AFFILIATIONS); - public static final String FUNDINGS_IDENTIFIER = jsonPath(FUNDINGS, IDENTIFIER, KEYWORD); - private final QueryKeys parameters; - - public ResourceStreamBuilders(QueryKeys parameters) { - this.parameters = parameters; - } - - public Stream> searchAllWithBoostsQuery( - Map fields) { - var sevenValues = - parameters - .get(SEARCH_ALL) - .asSplitStream(SPACE) - .limit(7) - .collect(Collectors.joining(SPACE)); - var fifteenValues = - parameters - .get(SEARCH_ALL) - .asSplitStream(SPACE) - .limit(15) - .collect(Collectors.joining(SPACE)); - - var query = - boolQuery() - .queryName(SEARCH_ALL.asCamelCase()) - .must( - multiMatchQuery(sevenValues) - .fields(fields) - .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) - .operator(Operator.AND)); - - if (fields.containsKey(ENTITY_DESCRIPTION_MAIN_TITLE) || fields.containsKey(ASTERISK)) { - query.should( - matchPhrasePrefixQuery(ENTITY_DESCRIPTION_MAIN_TITLE, fifteenValues) - .boost(TITLE.fieldBoost())); - } - if (fields.containsKey(ENTITY_ABSTRACT) || fields.containsKey(ASTERISK)) { - query.should( - matchPhraseQuery(ENTITY_ABSTRACT, fifteenValues).boost(ABSTRACT.fieldBoost())); - } - return Functions.queryToEntry(SEARCH_ALL, query); + public static final String COUNTRY_CODE_PATH = + jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, AFFILIATIONS, COUNTRY_CODE, KEYWORD); + public static final String CONTRIBUTOR_ROLE_PATH = + jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, ROLE, TYPE, KEYWORD); + public static final String VERIFICATION_STATUS_PATH = + jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, VERIFICATION_STATUS); + public static final String VERIFICATION_STATUS_KEYWORD = + jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, IDENTITY, VERIFICATION_STATUS, KEYWORD); + public static final String ADDITIONAL_IDENTIFIERS_VALUE_PATH = + jsonPath(ADDITIONAL_IDENTIFIERS, VALUE, KEYWORD); + public static final String ADDITIONAL_IDENTIFIERS_NAME_PATH = + jsonPath(ADDITIONAL_IDENTIFIERS, SOURCE_NAME, KEYWORD); + public static final String FUNDING_SOURCE_IDENTIFIER = + jsonPath(FUNDINGS, SOURCE, IDENTIFIER, KEYWORD); + public static final String CONTRIBUTOR_AFFILIATIONS = + jsonPath(ENTITY_DESCRIPTION, CONTRIBUTORS, AFFILIATIONS); + public static final String FUNDINGS_IDENTIFIER = jsonPath(FUNDINGS, IDENTIFIER, KEYWORD); + private final QueryKeys parameters; + + public ResourceStreamBuilders(QueryKeys parameters) { + this.parameters = parameters; + } + + public Stream> searchAllWithBoostsQuery( + Map fields) { + var sevenValues = + parameters.get(SEARCH_ALL).asSplitStream(SPACE).limit(7).collect(Collectors.joining(SPACE)); + var fifteenValues = + parameters + .get(SEARCH_ALL) + .asSplitStream(SPACE) + .limit(15) + .collect(Collectors.joining(SPACE)); + + var query = + boolQuery() + .queryName(SEARCH_ALL.asCamelCase()) + .must( + multiMatchQuery(sevenValues) + .fields(fields) + .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) + .operator(Operator.AND)); + + if (fields.containsKey(ENTITY_DESCRIPTION_MAIN_TITLE) || fields.containsKey(ASTERISK)) { + query.should( + matchPhrasePrefixQuery(ENTITY_DESCRIPTION_MAIN_TITLE, fifteenValues) + .boost(TITLE.fieldBoost())); } - - public Stream> additionalIdentifierQuery( - ResourceParameter key, String source) { - String value = parameters.get(key).as(); - var query = - nestedQuery( - ADDITIONAL_IDENTIFIERS, - boolQuery() - .must(termQuery(ADDITIONAL_IDENTIFIERS_VALUE_PATH, value)) - .must(termQuery(ADDITIONAL_IDENTIFIERS_NAME_PATH, source)), - ScoreMode.None); - - return Functions.queryToEntry(key, query); - } - - public Stream> unIdentifiedNorwegians( - ResourceParameter key) { - var query = - nestedQuery( - ENTITY_CONTRIBUTORS, - boolQuery() - .must(termQuery(COUNTRY_CODE_PATH, NO)) - .must(termQuery(CONTRIBUTOR_ROLE_PATH, CREATOR)) - .should(boolQuery().mustNot(existsQuery(VERIFICATION_STATUS_PATH))) - .should(termQuery(VERIFICATION_STATUS_KEYWORD, NOT_VERIFIED)) - .minimumShouldMatch(1), - ScoreMode.None); - return Functions.queryToEntry(key, query); + if (fields.containsKey(ENTITY_ABSTRACT) || fields.containsKey(ASTERISK)) { + query.should(matchPhraseQuery(ENTITY_ABSTRACT, fifteenValues).boost(ABSTRACT.fieldBoost())); } - - public Stream> unIdentifiedContributorOrInstitution( - ResourceParameter key) { - var query = - boolQuery() - .must(termQuery(CONTRIBUTOR_ROLE_PATH, CREATOR)) - .should( - boolQuery() - .mustNot(termQuery(VERIFICATION_STATUS_KEYWORD, VERIFIED))) - .should(boolQuery().mustNot(existsQuery(CONTRIBUTOR_AFFILIATIONS))) - .minimumShouldMatch(1); - return Functions.queryToEntry(key, query); - } - - public Stream> fundingQuery(ResourceParameter key) { - var values = parameters.get(key).split(COLON); - var query = - nestedQuery( - FUNDINGS, - boolQuery() - .must(termQuery(FUNDINGS_IDENTIFIER, values[1])) - .must(termQuery(FUNDING_SOURCE_IDENTIFIER, values[0])), - ScoreMode.None); - return Functions.queryToEntry(key, query); - } - - public Stream> subUnitIncludedQuery( - ResourceParameter key) { - var searchKey = shouldSearchSpecifiedInstitutionOnly() ? key : EXCLUDE_SUBUNITS; - - return new FuzzyKeywordQuery() - .buildQuery(searchKey, parameters.get(key).toString()) - .map(query -> Map.entry(key, query.getValue())); - } - - public Stream> scientificValueQuery( - ResourceParameter key) { - - var values = parameters.get(key).split(COMMA); - var scientificValuesBaseQuery = + return Functions.queryToEntry(SEARCH_ALL, query); + } + + public Stream> additionalIdentifierQuery( + ResourceParameter key, String source) { + String value = parameters.get(key).as(); + var query = + nestedQuery( + ADDITIONAL_IDENTIFIERS, + boolQuery() + .must(termQuery(ADDITIONAL_IDENTIFIERS_VALUE_PATH, value)) + .must(termQuery(ADDITIONAL_IDENTIFIERS_NAME_PATH, source)), + ScoreMode.None); + + return Functions.queryToEntry(key, query); + } + + public Stream> unIdentifiedNorwegians( + ResourceParameter key) { + var query = + nestedQuery( + ENTITY_CONTRIBUTORS, + boolQuery() + .must(termQuery(COUNTRY_CODE_PATH, NO)) + .must(termQuery(CONTRIBUTOR_ROLE_PATH, CREATOR)) + .should(boolQuery().mustNot(existsQuery(VERIFICATION_STATUS_PATH))) + .should(termQuery(VERIFICATION_STATUS_KEYWORD, NOT_VERIFIED)) + .minimumShouldMatch(1), + ScoreMode.None); + return Functions.queryToEntry(key, query); + } + + public Stream> unIdentifiedContributorOrInstitution( + ResourceParameter key) { + var query = + boolQuery() + .must(termQuery(CONTRIBUTOR_ROLE_PATH, CREATOR)) + .should(boolQuery().mustNot(termQuery(VERIFICATION_STATUS_KEYWORD, VERIFIED))) + .should(boolQuery().mustNot(existsQuery(CONTRIBUTOR_AFFILIATIONS))) + .minimumShouldMatch(1); + return Functions.queryToEntry(key, query); + } + + public Stream> fundingQuery(ResourceParameter key) { + var values = parameters.get(key).split(COLON); + var query = + nestedQuery( + FUNDINGS, + boolQuery() + .must(termQuery(FUNDINGS_IDENTIFIER, values[1])) + .must(termQuery(FUNDING_SOURCE_IDENTIFIER, values[0])), + ScoreMode.None); + return Functions.queryToEntry(key, query); + } + + public Stream> subUnitIncludedQuery( + ResourceParameter key) { + var searchKey = shouldSearchSpecifiedInstitutionOnly() ? key : EXCLUDE_SUBUNITS; + + return new FuzzyKeywordQuery() + .buildQuery(searchKey, parameters.get(key).toString()) + .map(query -> Map.entry(key, query.getValue())); + } + + public Stream> scientificValueQuery( + ResourceParameter key) { + + var values = parameters.get(key).split(COMMA); + var scientificValuesBaseQuery = + boolQuery() + .should( boolQuery() - .should( - boolQuery() - .must(existsQuery(SCIENTIFIC_SERIES)) - .must(termsQuery(SCIENTIFIC_SERIES, values))) - .should( - boolQuery() - .mustNot(existsQuery(SCIENTIFIC_SERIES)) - .must(termsQuery(SCIENTIFIC_PUBLISHER, values))) - .should(termsQuery(SCIENTIFIC_OTHER, values)) - .minimumShouldMatch(1); - var parentChildQuery = + .must(existsQuery(SCIENTIFIC_SERIES)) + .must(termsQuery(SCIENTIFIC_SERIES, values))) + .should( boolQuery() - .should( - new HasParentQueryBuilder( - HAS_PARTS, scientificValuesBaseQuery, true)) - .should(scientificValuesBaseQuery) - .minimumShouldMatch(1); - - return Functions.queryToEntry(key, parentChildQuery); - } - - private Boolean shouldSearchSpecifiedInstitutionOnly() { - return parameters.get(EXCLUDE_SUBUNITS).asBoolean(); - } + .mustNot(existsQuery(SCIENTIFIC_SERIES)) + .must(termsQuery(SCIENTIFIC_PUBLISHER, values))) + .should(termsQuery(SCIENTIFIC_OTHER, values)) + .minimumShouldMatch(1); + var parentChildQuery = + boolQuery() + .should(new HasParentQueryBuilder(HAS_PARTS, scientificValuesBaseQuery, true)) + .should(scientificValuesBaseQuery) + .minimumShouldMatch(1); + + return Functions.queryToEntry(key, parentChildQuery); + } + + private Boolean shouldSearchSpecifiedInstitutionOnly() { + return parameters.get(EXCLUDE_SUBUNITS).asBoolean(); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/SimplifiedMutator.java b/search-commons/src/main/java/no/unit/nva/search/resource/SimplifiedMutator.java index 1cef413ed..6c7533fcb 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/SimplifiedMutator.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/SimplifiedMutator.java @@ -41,13 +41,21 @@ import static no.unit.nva.search.resource.Constants.MANIFESTATIONS; import static no.unit.nva.search.resource.Constants.SCOPUS_IDENTIFIER; import static no.unit.nva.search.resource.Constants.SEQUENCE; - import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.search.common.records.JsonNodeMutator; import no.unit.nva.search.resource.response.Affiliation; import no.unit.nva.search.resource.response.Contributor; @@ -60,21 +68,9 @@ import no.unit.nva.search.resource.response.ResourceSearchResponse; import no.unit.nva.search.resource.response.ResourceSearchResponse.Builder; import no.unit.nva.search.resource.response.Series; - import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - public class SimplifiedMutator implements JsonNodeMutator { public static final String HANDLE_IDENTIFIER = "HandleIdentifier"; diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/UserSettingsClient.java b/search-commons/src/main/java/no/unit/nva/search/resource/UserSettingsClient.java index ed4aca69e..c0429875f 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/UserSettingsClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/UserSettingsClient.java @@ -8,20 +8,10 @@ import static no.unit.nva.search.common.constant.Functions.readApiHost; import static no.unit.nva.search.resource.Constants.PERSON_PREFERENCES; import static no.unit.nva.search.resource.ResourceParameter.CONTRIBUTOR; - import static nva.commons.core.StringUtils.EMPTY_STRING; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.net.MediaType; - -import no.unit.nva.commons.json.JsonSerializable; -import no.unit.nva.search.common.OpenSearchClient; -import no.unit.nva.search.common.jwt.CachedJwtProvider; -import no.unit.nva.search.common.records.UserSettings; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.attempt.FunctionWithException; - import java.net.URI; import java.net.URLEncoder; import java.net.http.HttpClient; @@ -32,6 +22,12 @@ import java.util.List; import java.util.function.BinaryOperator; import java.util.stream.Stream; +import no.unit.nva.commons.json.JsonSerializable; +import no.unit.nva.search.common.OpenSearchClient; +import no.unit.nva.search.common.jwt.CachedJwtProvider; +import no.unit.nva.search.common.records.UserSettings; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.attempt.FunctionWithException; /** * Client for searching user settings. @@ -40,67 +36,67 @@ */ public class UserSettingsClient extends OpenSearchClient { - private URI userSettingUri; - - public UserSettingsClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { - super(client, cachedJwtProvider); - } - - @Override - public UserSettings doSearch(ResourceSearchQuery query) { - queryBuilderStart = Instant.now(); - queryParameters = EMPTY_STRING; - return createQueryBuilderStream(query) - .map(this::createRequest) - .map(super::fetch) - .map(super::handleResponse) - .findFirst() - .orElseThrow() - .join(); - } - - @Override - protected UserSettings jsonToResponse(HttpResponse response) - throws JsonProcessingException { - return singleLineObjectMapper.readValue(response.body(), UserSettings.class); - } - - @Override - @JacocoGenerated - protected BinaryOperator responseAccumulator() { - // not in use - return null; - } - - @Override - protected FunctionWithException - logAndReturnResult() { - return result -> { - logger.info(new UserSettingLog(userSettingUri, result).toJsonString()); - return result; - }; - } - - private Stream createQueryBuilderStream(ResourceSearchQuery query) { - return query.parameters().get(CONTRIBUTOR).asStream(); - } - - private HttpRequest createRequest(String contributorId) { - var personId = URLEncoder.encode(contributorId, Charset.defaultCharset()); - userSettingUri = URI.create(HTTPS + readApiHost() + PERSON_PREFERENCES + personId); - return HttpRequest.newBuilder(userSettingUri) - .headers( - ACCEPT, MediaType.JSON_UTF_8.toString(), - CONTENT_TYPE, MediaType.JSON_UTF_8.toString(), - AUTHORIZATION_HEADER, jwtProvider.getValue().getToken()) - .GET() - .build(); - } - - private record UserSettingLog(URI uri, List promotedPublications) - implements JsonSerializable { - public UserSettingLog(URI uri, UserSettings userSettings) { - this(uri, userSettings.promotedPublications()); - } + private URI userSettingUri; + + public UserSettingsClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { + super(client, cachedJwtProvider); + } + + @Override + public UserSettings doSearch(ResourceSearchQuery query) { + queryBuilderStart = Instant.now(); + queryParameters = EMPTY_STRING; + return createQueryBuilderStream(query) + .map(this::createRequest) + .map(super::fetch) + .map(super::handleResponse) + .findFirst() + .orElseThrow() + .join(); + } + + @Override + protected UserSettings jsonToResponse(HttpResponse response) + throws JsonProcessingException { + return singleLineObjectMapper.readValue(response.body(), UserSettings.class); + } + + @Override + @JacocoGenerated + protected BinaryOperator responseAccumulator() { + // not in use + return null; + } + + @Override + protected FunctionWithException + logAndReturnResult() { + return result -> { + logger.info(new UserSettingLog(userSettingUri, result).toJsonString()); + return result; + }; + } + + private Stream createQueryBuilderStream(ResourceSearchQuery query) { + return query.parameters().get(CONTRIBUTOR).asStream(); + } + + private HttpRequest createRequest(String contributorId) { + var personId = URLEncoder.encode(contributorId, Charset.defaultCharset()); + userSettingUri = URI.create(HTTPS + readApiHost() + PERSON_PREFERENCES + personId); + return HttpRequest.newBuilder(userSettingUri) + .headers( + ACCEPT, MediaType.JSON_UTF_8.toString(), + CONTENT_TYPE, MediaType.JSON_UTF_8.toString(), + AUTHORIZATION_HEADER, jwtProvider.getValue().getToken()) + .GET() + .build(); + } + + private record UserSettingLog(URI uri, List promotedPublications) + implements JsonSerializable { + public UserSettingLog(URI uri, UserSettings userSettings) { + this(uri, userSettings.promotedPublications()); } + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/resource/response/ResourceSearchResponse.java b/search-commons/src/main/java/no/unit/nva/search/resource/response/ResourceSearchResponse.java index 52e823456..0872003b0 100644 --- a/search-commons/src/main/java/no/unit/nva/search/resource/response/ResourceSearchResponse.java +++ b/search-commons/src/main/java/no/unit/nva/search/resource/response/ResourceSearchResponse.java @@ -1,7 +1,6 @@ package no.unit.nva.search.resource.response; import com.fasterxml.jackson.annotation.JsonProperty; - import java.net.URI; import java.util.List; import java.util.Map; diff --git a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollClient.java b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollClient.java index c2f41484e..4e56b5501 100644 --- a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollClient.java @@ -4,19 +4,16 @@ import static no.unit.nva.search.common.jwt.Tools.getCachedJwtProvider; import com.fasterxml.jackson.core.JsonProcessingException; - +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.function.BinaryOperator; import no.unit.nva.search.common.OpenSearchClient; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.records.SwsResponse; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.FunctionWithException; import nva.commons.secrets.SecretsReader; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.function.BinaryOperator; - /** * ScrollClient is a class that sends a request to the search index. * diff --git a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollParameter.java b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollParameter.java index c0746ed63..ce59eea5e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollParameter.java +++ b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollParameter.java @@ -7,23 +7,19 @@ import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_IGNORE_CASE; import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_NONE_OR_ONE; import static no.unit.nva.search.common.enums.FieldOperator.NA; - import static nva.commons.core.StringUtils.EMPTY_STRING; +import java.util.Locale; +import java.util.StringJoiner; +import java.util.stream.Stream; import no.unit.nva.search.common.enums.FieldOperator; import no.unit.nva.search.common.enums.ParameterKey; import no.unit.nva.search.common.enums.ParameterKind; import no.unit.nva.search.common.enums.ValueEncoding; - import nva.commons.core.JacocoGenerated; - import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.text.CaseUtils; -import java.util.Locale; -import java.util.StringJoiner; -import java.util.stream.Stream; - /** * Enum for all the parameters that can be used to query the search index. This enum needs to * implement these parameters cristin diff --git a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollQuery.java b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollQuery.java index ff1a6c0e9..61c77e66b 100644 --- a/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/scroll/ScrollQuery.java @@ -1,30 +1,24 @@ package no.unit.nva.search.scroll; import static com.google.common.net.MediaType.CSV_UTF_8; - +import static java.util.Objects.nonNull; import static nva.commons.core.attempt.Try.attempt; import static nva.commons.core.paths.UriWrapper.fromUri; - import static org.opensearch.core.xcontent.XContentHelper.toXContent; -import static java.util.Objects.nonNull; - import com.fasterxml.jackson.databind.JsonNode; - +import java.net.URI; +import java.util.Objects; +import java.util.stream.Stream; import no.unit.nva.search.common.OpenSearchClient; import no.unit.nva.search.common.Query; import no.unit.nva.search.common.records.HttpResponseFormatter; import no.unit.nva.search.common.records.QueryContentWrapper; import no.unit.nva.search.common.records.SwsResponse; - import org.opensearch.action.search.SearchScrollRequest; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.ToXContent; -import java.net.URI; -import java.util.Objects; -import java.util.stream.Stream; - /** * ScrollQuery is a class that sends a request to the search index. * diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/Constants.java b/search-commons/src/main/java/no/unit/nva/search/ticket/Constants.java index b1c64328a..a89d2fc32 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/Constants.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/Constants.java @@ -25,11 +25,12 @@ import static no.unit.nva.search.common.constant.Functions.filterBranchBuilder; import static no.unit.nva.search.common.constant.Functions.jsonPath; import static no.unit.nva.search.common.constant.Functions.multipleFields; - import static org.opensearch.index.query.QueryBuilders.boolQuery; +import java.util.List; +import java.util.Map; +import java.util.Set; import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilder; @@ -37,10 +38,6 @@ import org.opensearch.index.query.TermsQueryBuilder; import org.opensearch.search.aggregations.AggregationBuilder; -import java.util.List; -import java.util.Map; -import java.util.Set; - /** * Constants for the ticket search. * diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketAccessFilter.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketAccessFilter.java index 1ef9c2aba..02038d20e 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketAccessFilter.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketAccessFilter.java @@ -1,5 +1,6 @@ package no.unit.nva.search.ticket; +import static java.util.Objects.isNull; import static no.unit.nva.search.ticket.Constants.CANNOT_SEARCH_AS_BOTH_ASSIGNEE_AND_OWNER_AT_THE_SAME_TIME; import static no.unit.nva.search.ticket.Constants.FILTER_BY_ORGANIZATION; import static no.unit.nva.search.ticket.Constants.FILTER_BY_OWNER; @@ -14,33 +15,26 @@ import static no.unit.nva.search.ticket.TicketParameter.ASSIGNEE; import static no.unit.nva.search.ticket.TicketParameter.OWNER; import static no.unit.nva.search.ticket.TicketParameter.STATISTICS; - import static nva.commons.apigateway.AccessRight.MANAGE_CUSTOMERS; import static nva.commons.apigateway.AccessRight.MANAGE_DOI; import static nva.commons.apigateway.AccessRight.MANAGE_PUBLISHING_REQUESTS; - import static org.opensearch.index.query.QueryBuilders.boolQuery; -import static java.util.Objects.isNull; - +import java.net.URI; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; import no.unit.nva.search.common.records.FilterBuilder; - import nva.commons.apigateway.AccessRight; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.UnauthorizedException; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.Operator; import org.opensearch.index.query.QueryBuilders; import org.opensearch.index.query.TermsQueryBuilder; -import java.net.URI; -import java.util.EnumSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - /** * TicketAccessFilter is a class that filters tickets based on access rights. * @@ -61,225 +55,223 @@ */ public class TicketAccessFilter implements FilterBuilder { - private final TicketSearchQuery query; - private String currentUser; - private URI organizationId; - private Set accessRightEnumSet = EnumSet.noneOf(AccessRight.class); - - public TicketAccessFilter(TicketSearchQuery query) { - this.query = query; - this.query.filters().set(); - } - - public String getCurrentUser() { - return currentUser; - } - - public Set getAccessRightAsTicketTypes() { - return accessRightsToTicketTypes(accessRightEnumSet); - } - - /** - * Filter on access rights. - * - * @param accessRights access rights - * @return TicketQuery (builder pattern) - * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} - */ - public TicketAccessFilter accessRights(AccessRight... accessRights) { - return accessRights(List.of(accessRights)); - } - - private TicketAccessFilter accessRights(List accessRights) { - this.accessRightEnumSet = - accessRights.stream() - .collect(Collectors.toCollection(() -> EnumSet.noneOf(AccessRight.class))); - return this; - } - - /** - * Filter on owner (user). - * - * @param userName current user - * @return TicketQuery (builder pattern) - * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} - */ - public TicketAccessFilter user(String userName) { - this.currentUser = userName; - return this; - } - - /** - * Filter on organization. - * - * @param organization organization id - * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} - */ - public TicketAccessFilter organization(URI organization) { - this.organizationId = organization; - return this; - } - - /** - * Authorize and set 'ViewScope'. - * - *

Authorize and set filters -> ticketTypes, organization and owner - * - *

This is to avoid the Query to return documents that are not available for the user. - * - * @param requestInfo all required is here - * @return TicketQuery (builder pattern) - */ - @Override - public TicketSearchQuery fromRequestInfo(RequestInfo requestInfo) throws UnauthorizedException { - - var organization = - requestInfo.getTopLevelOrgCristinId().orElse(requestInfo.getPersonAffiliation()); - - return user(requestInfo.getUserName()) - .accessRights(requestInfo.getAccessRights()) - .organization(organization) - .apply(); + private final TicketSearchQuery query; + private String currentUser; + private URI organizationId; + private Set accessRightEnumSet = EnumSet.noneOf(AccessRight.class); + + public TicketAccessFilter(TicketSearchQuery query) { + this.query = query; + this.query.filters().set(); + } + + public String getCurrentUser() { + return currentUser; + } + + public Set getAccessRightAsTicketTypes() { + return accessRightsToTicketTypes(accessRightEnumSet); + } + + /** + * Filter on access rights. + * + * @param accessRights access rights + * @return TicketQuery (builder pattern) + * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} + */ + public TicketAccessFilter accessRights(AccessRight... accessRights) { + return accessRights(List.of(accessRights)); + } + + private TicketAccessFilter accessRights(List accessRights) { + this.accessRightEnumSet = + accessRights.stream() + .collect(Collectors.toCollection(() -> EnumSet.noneOf(AccessRight.class))); + return this; + } + + /** + * Filter on owner (user). + * + * @param userName current user + * @return TicketQuery (builder pattern) + * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} + */ + public TicketAccessFilter user(String userName) { + this.currentUser = userName; + return this; + } + + /** + * Filter on organization. + * + * @param organization organization id + * @apiNote ONLY USE IN TESTS, in handlers use: {@link #fromRequestInfo(RequestInfo)} + */ + public TicketAccessFilter organization(URI organization) { + this.organizationId = organization; + return this; + } + + /** + * Authorize and set 'ViewScope'. + * + *

Authorize and set filters -> ticketTypes, organization and owner + * + *

This is to avoid the Query to return documents that are not available for the user. + * + * @param requestInfo all required is here + * @return TicketQuery (builder pattern) + */ + @Override + public TicketSearchQuery fromRequestInfo(RequestInfo requestInfo) throws UnauthorizedException { + + var organization = + requestInfo.getTopLevelOrgCristinId().orElse(requestInfo.getPersonAffiliation()); + + return user(requestInfo.getUserName()) + .accessRights(requestInfo.getAccessRights()) + .organization(organization) + .apply(); + } + + @Override + public TicketSearchQuery apply() throws UnauthorizedException { + + if (searchAsSiktAdmin() && validateSiktAdmin(accessRightEnumSet)) { + return query; // See everything, NO FILTERS!!! } - @Override - public TicketSearchQuery apply() throws UnauthorizedException { - - if (searchAsSiktAdmin() && validateSiktAdmin(accessRightEnumSet)) { - return query; // See everything, NO FILTERS!!! - } - - validateOrganization(); - validateUser(); - final var curatorTicketTypes = accessRightsToTicketTypes(accessRightEnumSet); - - if (hasNoCuratorRoles(curatorTicketTypes) && searchAsTicketOwner()) { - validateOwner(currentUser); - } - if (searchAsTicketAssignee() && searchAsTicketOwner()) { - validateAssigneeAndOwnerParameters(); - } - - this.query - .filters() - .add(filterByOrganization(organizationId)) - .add(filterByUserAndTicketTypes(currentUser, curatorTicketTypes)) - .add(filterByDeniedUnpublishRequest()); - return query; - } + validateOrganization(); + validateUser(); + final var curatorTicketTypes = accessRightsToTicketTypes(accessRightEnumSet); - /** - * Apply business rules to determine which ticket types are allowed. - * - *

- */ - private Set accessRightsToTicketTypes(Set accessRights) { - var allowed = EnumSet.noneOf(TicketType.class); - if (accessRights.contains(MANAGE_DOI)) { - allowed.add(TicketType.DOI_REQUEST); - } - if (accessRights.contains(AccessRight.SUPPORT)) { - allowed.add(TicketType.GENERAL_SUPPORT_CASE); - } - if (accessRights.contains(MANAGE_PUBLISHING_REQUESTS)) { - allowed.add(TicketType.PUBLISHING_REQUEST); - } - if (allowed.isEmpty()) { - allowed.add(TicketType.NONE); - } - return allowed; + if (hasNoCuratorRoles(curatorTicketTypes) && searchAsTicketOwner()) { + validateOwner(currentUser); } - - private MultiMatchQueryBuilder filterByOrganization(URI organizationId) { - return QueryBuilders.multiMatchQuery(organizationId.toString(), ORGANIZATION_PATHS) - .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) - .operator(Operator.AND) - .queryName(FILTER_BY_ORGANIZATION); + if (searchAsTicketAssignee() && searchAsTicketOwner()) { + validateAssigneeAndOwnerParameters(); } - private BoolQueryBuilder filterByUserAndTicketTypes( - String userName, Set curatorTicketTypes) { - return boolQuery() - .should(filterByOwner(userName)) - .should(filterByTicketTypes(curatorTicketTypes)) - .minimumShouldMatch(1) - .queryName(FILTER_BY_USER_AND_TICKET_TYPES); + this.query + .filters() + .add(filterByOrganization(organizationId)) + .add(filterByUserAndTicketTypes(currentUser, curatorTicketTypes)) + .add(filterByDeniedUnpublishRequest()); + return query; + } + + /** + * Apply business rules to determine which ticket types are allowed. + * + *
    + *
  • manage_doi -> DOI_REQUEST + *
  • support -> GENERAL_SUPPORT_CASE + *
  • manage_publishing_requests -> PUBLISHING_REQUEST + *
+ */ + private Set accessRightsToTicketTypes(Set accessRights) { + var allowed = EnumSet.noneOf(TicketType.class); + if (accessRights.contains(MANAGE_DOI)) { + allowed.add(TicketType.DOI_REQUEST); } - - private MultiMatchQueryBuilder filterByOwner(String userName) { - return QueryBuilders.multiMatchQuery(userName, OWNER.searchFields().toArray(String[]::new)) - .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) - .operator(Operator.AND) - .queryName(FILTER_BY_OWNER); + if (accessRights.contains(AccessRight.SUPPORT)) { + allowed.add(TicketType.GENERAL_SUPPORT_CASE); } - - private TermsQueryBuilder filterByTicketTypes(Set curatorTicketTypes) { - var ticketTypes = - curatorTicketTypes.stream().map(TicketType::toString).toArray(String[]::new); - return new TermsQueryBuilder(TYPE_KEYWORD, ticketTypes).queryName(FILTER_BY_TICKET_TYPES); + if (accessRights.contains(MANAGE_PUBLISHING_REQUESTS)) { + allowed.add(TicketType.PUBLISHING_REQUEST); } - - private BoolQueryBuilder filterByDeniedUnpublishRequest() { - return boolQuery() - .mustNot(filterByTicketTypes(Set.of(TicketType.UNPUBLISH_REQUEST))) - .queryName(FILTER_BY_UN_PUBLISHED); + if (allowed.isEmpty()) { + allowed.add(TicketType.NONE); } - - private void validateOrganization() throws UnauthorizedException { - if (isNull(organizationId)) { - throw new UnauthorizedException(ORGANIZATION_IS_REQUIRED); - } + return allowed; + } + + private MultiMatchQueryBuilder filterByOrganization(URI organizationId) { + return QueryBuilders.multiMatchQuery(organizationId.toString(), ORGANIZATION_PATHS) + .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) + .operator(Operator.AND) + .queryName(FILTER_BY_ORGANIZATION); + } + + private BoolQueryBuilder filterByUserAndTicketTypes( + String userName, Set curatorTicketTypes) { + return boolQuery() + .should(filterByOwner(userName)) + .should(filterByTicketTypes(curatorTicketTypes)) + .minimumShouldMatch(1) + .queryName(FILTER_BY_USER_AND_TICKET_TYPES); + } + + private MultiMatchQueryBuilder filterByOwner(String userName) { + return QueryBuilders.multiMatchQuery(userName, OWNER.searchFields().toArray(String[]::new)) + .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS) + .operator(Operator.AND) + .queryName(FILTER_BY_OWNER); + } + + private TermsQueryBuilder filterByTicketTypes(Set curatorTicketTypes) { + var ticketTypes = curatorTicketTypes.stream().map(TicketType::toString).toArray(String[]::new); + return new TermsQueryBuilder(TYPE_KEYWORD, ticketTypes).queryName(FILTER_BY_TICKET_TYPES); + } + + private BoolQueryBuilder filterByDeniedUnpublishRequest() { + return boolQuery() + .mustNot(filterByTicketTypes(Set.of(TicketType.UNPUBLISH_REQUEST))) + .queryName(FILTER_BY_UN_PUBLISHED); + } + + private void validateOrganization() throws UnauthorizedException { + if (isNull(organizationId)) { + throw new UnauthorizedException(ORGANIZATION_IS_REQUIRED); } + } - private void validateOwner(String userName) throws UnauthorizedException { - if (isCurrentUserNotOwner(userName)) { - throw new UnauthorizedException( - USER_IS_NOT_ALLOWED_TO_SEARCH_FOR_TICKETS_NOT_OWNED_BY_THEMSELVES); - } + private void validateOwner(String userName) throws UnauthorizedException { + if (isCurrentUserNotOwner(userName)) { + throw new UnauthorizedException( + USER_IS_NOT_ALLOWED_TO_SEARCH_FOR_TICKETS_NOT_OWNED_BY_THEMSELVES); } + } - private void validateAssigneeAndOwnerParameters() throws UnauthorizedException { - if (query.parameters().get(OWNER).as().equals(query.parameters().get(ASSIGNEE).as())) { - throw new UnauthorizedException( - CANNOT_SEARCH_AS_BOTH_ASSIGNEE_AND_OWNER_AT_THE_SAME_TIME); - } + private void validateAssigneeAndOwnerParameters() throws UnauthorizedException { + if (query.parameters().get(OWNER).as().equals(query.parameters().get(ASSIGNEE).as())) { + throw new UnauthorizedException(CANNOT_SEARCH_AS_BOTH_ASSIGNEE_AND_OWNER_AT_THE_SAME_TIME); } + } - private void validateUser() throws UnauthorizedException { - if (isNull(currentUser)) { - throw new UnauthorizedException(USER_IS_REQUIRED); - } + private void validateUser() throws UnauthorizedException { + if (isNull(currentUser)) { + throw new UnauthorizedException(USER_IS_REQUIRED); } + } - private boolean validateSiktAdmin(Set accessRights) throws UnauthorizedException { - if (!accessRights.contains(MANAGE_CUSTOMERS)) { - throw new UnauthorizedException( - USER_IS_NOT_ALLOWED_TO_SEARCH_FOR_TICKETS_NOT_OWNED_BY_THEMSELVES); - } - return true; + private boolean validateSiktAdmin(Set accessRights) throws UnauthorizedException { + if (!accessRights.contains(MANAGE_CUSTOMERS)) { + throw new UnauthorizedException( + USER_IS_NOT_ALLOWED_TO_SEARCH_FOR_TICKETS_NOT_OWNED_BY_THEMSELVES); } + return true; + } - private boolean searchAsTicketAssignee() { - return query.parameters().isPresent(ASSIGNEE); - } + private boolean searchAsTicketAssignee() { + return query.parameters().isPresent(ASSIGNEE); + } - private boolean hasNoCuratorRoles(Set curatorTicketTypes) { - return curatorTicketTypes.contains(TicketType.NONE); - } + private boolean hasNoCuratorRoles(Set curatorTicketTypes) { + return curatorTicketTypes.contains(TicketType.NONE); + } - private boolean isCurrentUserNotOwner(String userName) { - return !userName.equalsIgnoreCase(query.parameters().get(OWNER).as()); - } + private boolean isCurrentUserNotOwner(String userName) { + return !userName.equalsIgnoreCase(query.parameters().get(OWNER).as()); + } - private boolean searchAsTicketOwner() { - return query.parameters().isPresent(OWNER); - } + private boolean searchAsTicketOwner() { + return query.parameters().isPresent(OWNER); + } - private boolean searchAsSiktAdmin() { - return query.parameters().isPresent(STATISTICS); - } + private boolean searchAsSiktAdmin() { + return query.parameters().isPresent(STATISTICS); + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketClient.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketClient.java index 56ce01943..6a23da302 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketClient.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketClient.java @@ -5,19 +5,16 @@ import static no.unit.nva.search.common.records.SwsResponse.SwsResponseBuilder.swsResponseBuilder; import com.fasterxml.jackson.core.JsonProcessingException; - +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.function.BinaryOperator; import no.unit.nva.search.common.OpenSearchClient; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.records.SwsResponse; - import nva.commons.core.JacocoGenerated; import nva.commons.core.attempt.FunctionWithException; import nva.commons.secrets.SecretsReader; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.function.BinaryOperator; - /** * TicketClient is a class that sends a request to the search index. * @@ -25,33 +22,32 @@ */ public class TicketClient extends OpenSearchClient { - public TicketClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { - super(client, cachedJwtProvider); - } - - @JacocoGenerated - public static TicketClient defaultClient() { - var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); - return new TicketClient(HttpClient.newHttpClient(), cachedJwtProvider); - } - - @Override - protected SwsResponse jsonToResponse(HttpResponse response) - throws JsonProcessingException { - return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); - } - - @Override - protected BinaryOperator responseAccumulator() { - return (a, b) -> swsResponseBuilder().merge(a).merge(b).build(); - } - - @Override - protected FunctionWithException - logAndReturnResult() { - return result -> { - logger.info(buildLogInfo(result)); - return result; - }; - } + public TicketClient(HttpClient client, CachedJwtProvider cachedJwtProvider) { + super(client, cachedJwtProvider); + } + + @JacocoGenerated + public static TicketClient defaultClient() { + var cachedJwtProvider = getCachedJwtProvider(new SecretsReader()); + return new TicketClient(HttpClient.newHttpClient(), cachedJwtProvider); + } + + @Override + protected SwsResponse jsonToResponse(HttpResponse response) + throws JsonProcessingException { + return singleLineObjectMapper.readValue(response.body(), SwsResponse.class); + } + + @Override + protected BinaryOperator responseAccumulator() { + return (a, b) -> swsResponseBuilder().merge(a).merge(b).build(); + } + + @Override + protected FunctionWithException logAndReturnResult() { + return result -> { + logger.info(buildLogInfo(result)); + return result; + }; + } } diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketParameter.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketParameter.java index 5f2f681fe..53acbd1c4 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketParameter.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketParameter.java @@ -1,5 +1,6 @@ package no.unit.nva.search.ticket; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.ErrorMessages.NOT_IMPLEMENTED_FOR; import static no.unit.nva.constants.Words.CHAR_UNDERSCORE; import static no.unit.nva.constants.Words.COLON; @@ -50,27 +51,22 @@ import static no.unit.nva.search.ticket.Constants.TYPE_KEYWORD; import static no.unit.nva.search.ticket.Constants.VIEWED_BY_FIELDS; -import static java.util.Objects.nonNull; - +import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Locale; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.constants.Words; import no.unit.nva.search.common.enums.FieldOperator; import no.unit.nva.search.common.enums.ParameterKey; import no.unit.nva.search.common.enums.ParameterKind; import no.unit.nva.search.common.enums.ValueEncoding; - import nva.commons.core.JacocoGenerated; - import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.text.CaseUtils; -import java.util.Arrays; -import java.util.LinkedHashSet; -import java.util.Locale; -import java.util.Set; -import java.util.StringJoiner; -import java.util.stream.Collectors; -import java.util.stream.Stream; - /** * Enum for all the parameters that can be used to query the search index. This enum needs to * implement these parameters cristin diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSearchQuery.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSearchQuery.java index 8a4a40e8d..a7729f7d9 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSearchQuery.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSearchQuery.java @@ -38,11 +38,17 @@ import static no.unit.nva.search.ticket.TicketParameter.TYPE; import static no.unit.nva.search.ticket.TicketStatus.NEW; import static no.unit.nva.search.ticket.TicketStatus.PENDING; - import static nva.commons.core.paths.UriWrapper.fromUri; - import static org.opensearch.index.query.QueryBuilders.multiMatchQuery; +import java.net.URI; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.UUID; +import java.util.stream.Stream; import no.unit.nva.search.common.AsType; import no.unit.nva.search.common.ParameterValidator; import no.unit.nva.search.common.SearchQuery; @@ -50,9 +56,7 @@ import no.unit.nva.search.common.builder.KeywordQuery; import no.unit.nva.search.common.constant.Functions; import no.unit.nva.search.common.enums.SortKey; - import nva.commons.core.JacocoGenerated; - import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MultiMatchQueryBuilder; import org.opensearch.index.query.Operator; @@ -62,15 +66,6 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.sort.SortOrder; -import java.net.URI; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.UUID; -import java.util.stream.Stream; - /** * TicketSearchQuery is a class that searches for tickets. * diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSort.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSort.java index b6abc501e..1775a08ee 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSort.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketSort.java @@ -4,19 +4,16 @@ import static no.unit.nva.search.common.constant.Patterns.PATTERN_IS_PIPE; import static no.unit.nva.search.ticket.Constants.STATUS_KEYWORD; import static no.unit.nva.search.ticket.Constants.TYPE_KEYWORD; - import static nva.commons.core.StringUtils.EMPTY_STRING; -import no.unit.nva.constants.Words; -import no.unit.nva.search.common.enums.SortKey; - -import org.apache.commons.text.CaseUtils; - import java.util.Arrays; import java.util.Collection; import java.util.Locale; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.constants.Words; +import no.unit.nva.search.common.enums.SortKey; +import org.apache.commons.text.CaseUtils; /** * TicketSort is an enum for sorting tickets. diff --git a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketStatus.java b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketStatus.java index e184dcc79..45be571ad 100644 --- a/search-commons/src/main/java/no/unit/nva/search/ticket/TicketStatus.java +++ b/search-commons/src/main/java/no/unit/nva/search/ticket/TicketStatus.java @@ -1,7 +1,6 @@ package no.unit.nva.search.ticket; import static no.unit.nva.search.common.enums.SortKey.getIgnoreCaseAndUnderscoreKeyExpression; - import static nva.commons.core.StringUtils.EMPTY_STRING; import java.util.Arrays; diff --git a/search-commons/src/test/java/no/unit/nva/common/Containers.java b/search-commons/src/test/java/no/unit/nva/common/Containers.java index 23d22c680..a8e465386 100644 --- a/search-commons/src/test/java/no/unit/nva/common/Containers.java +++ b/search-commons/src/test/java/no/unit/nva/common/Containers.java @@ -1,5 +1,6 @@ package no.unit.nva.common; +import static java.util.Objects.nonNull; import static no.unit.nva.common.TestConstants.DELAY_AFTER_INDEXING; import static no.unit.nva.common.TestConstants.OPEN_SEARCH_IMAGE; import static no.unit.nva.constants.Words.IDENTIFIER; @@ -7,22 +8,20 @@ import static no.unit.nva.constants.Words.RESOURCES; import static no.unit.nva.constants.Words.TICKETS; import static no.unit.nva.indexing.testutils.MockedJwtProvider.setupMockedCachedJwtProvider; - import static nva.commons.core.attempt.Try.attempt; import static nva.commons.core.ioutils.IoUtils.stringFromResources; -import static java.util.Objects.nonNull; - import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; - +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; import no.unit.nva.commons.json.JsonUtils; import no.unit.nva.identifiers.SortableIdentifier; import no.unit.nva.indexingclient.IndexingClient; import no.unit.nva.indexingclient.models.EventConsumptionAttributes; import no.unit.nva.indexingclient.models.IndexDocument; import no.unit.nva.indexingclient.models.RestHighLevelClientWrapper; - import org.apache.http.HttpHost; import org.opensearch.client.RestClient; import org.opensearch.testcontainers.OpensearchContainer; @@ -30,10 +29,6 @@ import org.slf4j.LoggerFactory; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; -import java.nio.file.Path; -import java.util.Map; - @Testcontainers public class Containers { diff --git a/search-commons/src/test/java/no/unit/nva/common/EntrySetTools.java b/search-commons/src/test/java/no/unit/nva/common/EntrySetTools.java index 74211903b..4e9fa2cf0 100644 --- a/search-commons/src/test/java/no/unit/nva/common/EntrySetTools.java +++ b/search-commons/src/test/java/no/unit/nva/common/EntrySetTools.java @@ -1,20 +1,17 @@ package no.unit.nva.common; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.AMPERSAND; import static no.unit.nva.constants.Words.EQUAL; - import static nva.commons.core.StringUtils.EMPTY_STRING; import static nva.commons.core.attempt.Try.attempt; -import static java.util.Objects.nonNull; - -import nva.commons.core.JacocoGenerated; - import java.net.URI; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Map.Entry; +import nva.commons.core.JacocoGenerated; public final class EntrySetTools { diff --git a/search-commons/src/test/java/no/unit/nva/common/InjectedTestSettings.java b/search-commons/src/test/java/no/unit/nva/common/InjectedTestSettings.java index 2849acea7..751c9fa5f 100644 --- a/search-commons/src/test/java/no/unit/nva/common/InjectedTestSettings.java +++ b/search-commons/src/test/java/no/unit/nva/common/InjectedTestSettings.java @@ -1,5 +1,6 @@ package no.unit.nva.common; +import java.io.IOException; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.config.Configurator; import org.junit.platform.launcher.TestExecutionListener; @@ -7,8 +8,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; - public class InjectedTestSettings implements TestExecutionListener { private static final Logger logger = LoggerFactory.getLogger(InjectedTestSettings.class); diff --git a/search-commons/src/test/java/no/unit/nva/common/MockedHttpResponse.java b/search-commons/src/test/java/no/unit/nva/common/MockedHttpResponse.java index b71860791..37ce2d754 100644 --- a/search-commons/src/test/java/no/unit/nva/common/MockedHttpResponse.java +++ b/search-commons/src/test/java/no/unit/nva/common/MockedHttpResponse.java @@ -1,12 +1,10 @@ package no.unit.nva.common; +import static java.util.Objects.nonNull; import static no.unit.nva.constants.Words.CONTENT_TYPE; import static no.unit.nva.testutils.TestHeaders.APPLICATION_JSON; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; -import static java.util.Objects.nonNull; - import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpHeaders; @@ -17,74 +15,72 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; - import javax.net.ssl.SSLSession; public class MockedHttpResponse { - public static CompletableFuture> mockedFutureHttpResponse(Path path) { - return CompletableFuture.completedFuture(mockedHttpResponse(path)); - } - - public static CompletableFuture> mockedFutureHttpResponse(String body) { - return CompletableFuture.completedFuture(mockedHttpResponse(body)); - } - - public static CompletableFuture> mockedFutureFailed() { - return CompletableFuture.failedFuture(new Throwable("future failed")); - } - - public static HttpResponse mockedHttpResponse(Path path) { - return mockedHttpResponse(stringFromResources(path)); - } - - public static HttpResponse mockedHttpResponse(String body) { - return mockedHttpResponse(body, nonNull(body) ? 200 : 400); - } - - public static HttpResponse mockedHttpResponse(String body, int statusCode) { - return new HttpResponse<>() { - @Override - public int statusCode() { - return statusCode; - } - - @Override - public HttpRequest request() { - return null; - } - - @Override - public Optional> previousResponse() { - return Optional.empty(); - } - - @Override - public HttpHeaders headers() { - return HttpHeaders.of( - Map.of(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), - (s, s2) -> true); - } - - @Override - public String body() { - return body; - } - - @Override - public Optional sslSession() { - return Optional.empty(); - } - - @Override - public URI uri() { - return null; - } - - @Override - public HttpClient.Version version() { - return null; - } - }; - } + public static CompletableFuture> mockedFutureHttpResponse(Path path) { + return CompletableFuture.completedFuture(mockedHttpResponse(path)); + } + + public static CompletableFuture> mockedFutureHttpResponse(String body) { + return CompletableFuture.completedFuture(mockedHttpResponse(body)); + } + + public static CompletableFuture> mockedFutureFailed() { + return CompletableFuture.failedFuture(new Throwable("future failed")); + } + + public static HttpResponse mockedHttpResponse(Path path) { + return mockedHttpResponse(stringFromResources(path)); + } + + public static HttpResponse mockedHttpResponse(String body) { + return mockedHttpResponse(body, nonNull(body) ? 200 : 400); + } + + public static HttpResponse mockedHttpResponse(String body, int statusCode) { + return new HttpResponse<>() { + @Override + public int statusCode() { + return statusCode; + } + + @Override + public HttpRequest request() { + return null; + } + + @Override + public Optional> previousResponse() { + return Optional.empty(); + } + + @Override + public HttpHeaders headers() { + return HttpHeaders.of( + Map.of(CONTENT_TYPE, Collections.singletonList(APPLICATION_JSON)), (s, s2) -> true); + } + + @Override + public String body() { + return body; + } + + @Override + public Optional sslSession() { + return Optional.empty(); + } + + @Override + public URI uri() { + return null; + } + + @Override + public HttpClient.Version version() { + return null; + } + }; + } } diff --git a/search-commons/src/test/java/no/unit/nva/indexingclient/CachedJwtTest.java b/search-commons/src/test/java/no/unit/nva/indexingclient/CachedJwtTest.java index 49916412f..bb6c604bd 100644 --- a/search-commons/src/test/java/no/unit/nva/indexingclient/CachedJwtTest.java +++ b/search-commons/src/test/java/no/unit/nva/indexingclient/CachedJwtTest.java @@ -6,69 +6,66 @@ import static org.mockito.Mockito.when; import com.auth0.jwt.interfaces.DecodedJWT; - -import no.unit.nva.search.common.jwt.CachedJwtProvider; -import no.unit.nva.search.common.jwt.CognitoAuthenticator; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Date; +import no.unit.nva.search.common.jwt.CachedJwtProvider; +import no.unit.nva.search.common.jwt.CognitoAuthenticator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; class CachedJwtTest { - public static final Instant TOKEN_EXPIRE_AT = Instant.parse("2006-12-03T10:15:30.00Z"); - private final DecodedJWT jwt1 = mock(DecodedJWT.class); - private final DecodedJWT jwt2 = mock(DecodedJWT.class); - CachedJwtProvider cachedJwtProvider; - private Clock mockedClock; + public static final Instant TOKEN_EXPIRE_AT = Instant.parse("2006-12-03T10:15:30.00Z"); + private final DecodedJWT jwt1 = mock(DecodedJWT.class); + private final DecodedJWT jwt2 = mock(DecodedJWT.class); + CachedJwtProvider cachedJwtProvider; + private Clock mockedClock; - @BeforeEach - void setup() { + @BeforeEach + void setup() { - when(jwt1.getExpiresAt()).thenReturn(Date.from(TOKEN_EXPIRE_AT)); - when(jwt2.getExpiresAt()).thenReturn(Date.from(TOKEN_EXPIRE_AT)); + when(jwt1.getExpiresAt()).thenReturn(Date.from(TOKEN_EXPIRE_AT)); + when(jwt2.getExpiresAt()).thenReturn(Date.from(TOKEN_EXPIRE_AT)); - mockedClock = mock(Clock.class); + mockedClock = mock(Clock.class); - var cognitoAuthenticator = mock(CognitoAuthenticator.class); - when(cognitoAuthenticator.fetchBearerToken()).thenReturn(jwt1).thenReturn(jwt2); - cachedJwtProvider = new CachedJwtProvider(cognitoAuthenticator, mockedClock); - } + var cognitoAuthenticator = mock(CognitoAuthenticator.class); + when(cognitoAuthenticator.fetchBearerToken()).thenReturn(jwt1).thenReturn(jwt2); + cachedJwtProvider = new CachedJwtProvider(cognitoAuthenticator, mockedClock); + } - @Test - void shouldGetSameTokenOnSequentialCallsWhenTokenIsNotExpired() { - var dateBeforeTokenExpiration = TOKEN_EXPIRE_AT.minus(Duration.ofMinutes(10)); - when(mockedClock.instant()).thenReturn(dateBeforeTokenExpiration); + @Test + void shouldGetSameTokenOnSequentialCallsWhenTokenIsNotExpired() { + var dateBeforeTokenExpiration = TOKEN_EXPIRE_AT.minus(Duration.ofMinutes(10)); + when(mockedClock.instant()).thenReturn(dateBeforeTokenExpiration); - var token1 = cachedJwtProvider.getValue(); - var token2 = cachedJwtProvider.getValue(); + var token1 = cachedJwtProvider.getValue(); + var token2 = cachedJwtProvider.getValue(); - assertEquals(token1, token2); - } + assertEquals(token1, token2); + } - @Test - void shouldGetNewTokenWhenTokenHasExpired() { - var dateAfterTokenExpiration = TOKEN_EXPIRE_AT.plus(Duration.ofMinutes(10)); - when(mockedClock.instant()).thenReturn(dateAfterTokenExpiration); + @Test + void shouldGetNewTokenWhenTokenHasExpired() { + var dateAfterTokenExpiration = TOKEN_EXPIRE_AT.plus(Duration.ofMinutes(10)); + when(mockedClock.instant()).thenReturn(dateAfterTokenExpiration); - var token1 = cachedJwtProvider.getValue(); - var token2 = cachedJwtProvider.getValue(); + var token1 = cachedJwtProvider.getValue(); + var token2 = cachedJwtProvider.getValue(); - assertNotEquals(token1, token2); - } + assertNotEquals(token1, token2); + } - @Test - void shouldGetNewTokenWhenTokenIsAboutToExpire() { - var dateBeforeTokenExpiration = TOKEN_EXPIRE_AT.minus(Duration.ofSeconds(2)); - when(mockedClock.instant()).thenReturn(dateBeforeTokenExpiration); + @Test + void shouldGetNewTokenWhenTokenIsAboutToExpire() { + var dateBeforeTokenExpiration = TOKEN_EXPIRE_AT.minus(Duration.ofSeconds(2)); + when(mockedClock.instant()).thenReturn(dateBeforeTokenExpiration); - var token1 = cachedJwtProvider.getValue(); - var token2 = cachedJwtProvider.getValue(); + var token1 = cachedJwtProvider.getValue(); + var token2 = cachedJwtProvider.getValue(); - assertNotEquals(token1, token2); - } + assertNotEquals(token1, token2); + } } diff --git a/search-commons/src/test/java/no/unit/nva/indexingclient/CognitoAuthenticatorTest.java b/search-commons/src/test/java/no/unit/nva/indexingclient/CognitoAuthenticatorTest.java index 9aa492d08..5a2842392 100644 --- a/search-commons/src/test/java/no/unit/nva/indexingclient/CognitoAuthenticatorTest.java +++ b/search-commons/src/test/java/no/unit/nva/indexingclient/CognitoAuthenticatorTest.java @@ -1,12 +1,13 @@ package no.unit.nva.indexingclient; +import static java.net.HttpURLConnection.HTTP_FORBIDDEN; +import static java.net.HttpURLConnection.HTTP_OK; import static no.unit.nva.auth.AuthorizedBackendClient.APPLICATION_X_WWW_FORM_URLENCODED; import static no.unit.nva.constants.Words.AUTHORIZATION; import static no.unit.nva.indexing.testutils.Constants.TEST_SCOPE; import static no.unit.nva.indexing.testutils.Constants.TEST_TOKEN; import static no.unit.nva.search.common.jwt.CognitoAuthenticator.AUTHORIZATION_ERROR_MESSAGE; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.apache.http.protocol.HTTP.CONTENT_TYPE; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; @@ -18,16 +19,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_FORBIDDEN; -import static java.net.HttpURLConnection.HTTP_OK; - -import no.unit.nva.auth.CognitoCredentials; -import no.unit.nva.indexingclient.utils.HttpRequestMetadataMatcher; -import no.unit.nva.search.common.jwt.CognitoAuthenticator; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - import java.io.IOException; import java.net.URI; import java.net.http.HttpClient; @@ -37,6 +28,11 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.NoSuchElementException; +import no.unit.nva.auth.CognitoCredentials; +import no.unit.nva.indexingclient.utils.HttpRequestMetadataMatcher; +import no.unit.nva.search.common.jwt.CognitoAuthenticator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; @SuppressWarnings({"unchecked"}) class CognitoAuthenticatorTest { diff --git a/search-commons/src/test/java/no/unit/nva/indexingclient/IndexingClientTest.java b/search-commons/src/test/java/no/unit/nva/indexingclient/IndexingClientTest.java index c3357f4f2..5bb798897 100644 --- a/search-commons/src/test/java/no/unit/nva/indexingclient/IndexingClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/indexingclient/IndexingClientTest.java @@ -6,9 +6,7 @@ import static no.unit.nva.indexingclient.IndexingClient.BULK_SIZE; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; @@ -28,7 +26,11 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; - +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import no.unit.nva.identifiers.SortableIdentifier; import no.unit.nva.indexingclient.models.EventConsumptionAttributes; import no.unit.nva.indexingclient.models.IndexDocument; @@ -36,9 +38,7 @@ import no.unit.nva.indexingclient.models.RestHighLevelClientWrapper; import no.unit.nva.search.common.jwt.CachedJwtProvider; import no.unit.nva.search.common.records.UsernamePasswordWrapper; - import nva.commons.secrets.SecretsReader; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; @@ -53,179 +53,171 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.indices.CreateIndexRequest; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - class IndexingClientTest { - public static final int - SET_OF_RESOURCES_THAT_DO_NOT_FIT_EXACTLY_IN_THE_BULK_SIZE_OF_A_BULK_REQUEST = 1256; - public static final IndexResponse UNUSED_INDEX_RESPONSE = null; - private RestHighLevelClientWrapper esClient; - private IndexingClient indexingClient; - private AtomicReference submittedIndexRequest; - - private CachedJwtProvider cachedJwtProvider; - - @BeforeEach - public void init() throws IOException { - cachedJwtProvider = setupMockedCachedJwtProvider(); - esClient = setupMockEsClient(); - indexingClient = new IndexingClient(esClient, cachedJwtProvider); - submittedIndexRequest = new AtomicReference<>(); - } - - @Test - void constructorWithSecretsReaderDefinedShouldCreateInstance() { - var secretsReaderMock = mock(SecretsReader.class); - var testCredentials = new UsernamePasswordWrapper("user", "password"); - when(secretsReaderMock.fetchClassSecret(anyString(), eq(UsernamePasswordWrapper.class))) - .thenReturn(testCredentials); - - IndexingClient indexingClient = IndexingClient.prepareWithSecretReader(secretsReaderMock); - assertNotNull(indexingClient); - } - - @Test - void shouldIndexAllDocumentsInBatchInBulksOfSpecifiedSize() throws IOException { - var indexDocuments = - IntStream.range( - 0, - SET_OF_RESOURCES_THAT_DO_NOT_FIT_EXACTLY_IN_THE_BULK_SIZE_OF_A_BULK_REQUEST) - .boxed() - .map(i -> randomJson()) - .map(this::toIndexDocument) - .toList(); - List provokeExecution = - indexingClient.batchInsert(indexDocuments.stream()).collect(Collectors.toList()); - assertThat(provokeExecution, is(not(nullValue()))); - - int expectedNumberOfBulkRequests = - (int) Math.ceil(((double) indexDocuments.size()) / ((double) BULK_SIZE)); - verify(esClient, times(expectedNumberOfBulkRequests)) - .bulk(any(BulkRequest.class), any(RequestOptions.class)); - } - - @Test - void shouldSendIndexRequestWithIndexNameSpecifiedByIndexDocument() throws IOException { - var indexDocument = sampleIndexDocument(); - var expectedIndex = indexDocument.consumptionAttributes().index(); - indexingClient.addDocumentToIndex(indexDocument); - - assertThat(submittedIndexRequest.get().index(), is(equalTo(expectedIndex))); - assertThat( - extractDocumentFromSubmittedIndexRequest(), is(equalTo(indexDocument.resource()))); - } - - @Test - void shouldCallEsClientCreateIndexRequest() throws IOException { - IndicesClient indicesClient = mock(IndicesClient.class); - var indicesClientWrapper = new IndicesClientWrapper(indicesClient); - when(esClient.indices()).thenReturn(indicesClientWrapper); - indexingClient.createIndex(randomString()); - var expectedNumberOfCreateInvocationsToEs = 1; - verify(indicesClient, times(expectedNumberOfCreateInvocationsToEs)) - .create(any(CreateIndexRequest.class), any(RequestOptions.class)); - } - - @Test - void shouldThrowExceptionWhenEsClientFailedToCreateIndex() throws IOException { - var expectedErrorMessage = randomString(); - var indicesClientWrapper = createMockIndicesClientWrapper(); - when(esClient.indices()).thenReturn(indicesClientWrapper); - when(indicesClientWrapper.create(any(CreateIndexRequest.class), any(RequestOptions.class))) - .thenThrow(new IOException(expectedErrorMessage)); - Executable createIndexAction = () -> indexingClient.createIndex(randomString()); - var actualException = assertThrows(IOException.class, createIndexAction); - assertThat(actualException.getMessage(), containsString(expectedErrorMessage)); - } - - @Test - void shouldThrowExceptionContainingTheCauseWhenIndexDocumentFailsToBeIndexed() - throws IOException { - String expectedMessage = randomString(); - esClient = mock(RestHighLevelClientWrapper.class); - when(esClient.index(any(IndexRequest.class), any(RequestOptions.class))) - .thenThrow(new IOException(expectedMessage)); - - indexingClient = new IndexingClient(esClient, cachedJwtProvider); - - Executable indexingAction = () -> indexingClient.addDocumentToIndex(sampleIndexDocument()); - var exception = assertThrows(IOException.class, indexingAction); - assertThat(exception.getMessage(), containsString(expectedMessage)); - } - - @Test - void shouldNotThrowExceptionWhenTryingToDeleteNonExistingDocument() throws IOException { - RestHighLevelClientWrapper restHighLevelClient = mock(RestHighLevelClientWrapper.class); - DeleteResponse nothingFoundResponse = mock(DeleteResponse.class); - when(nothingFoundResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); - when(restHighLevelClient.delete(any(), any())).thenReturn(nothingFoundResponse); - IndexingClient indexingClient = new IndexingClient(restHighLevelClient, cachedJwtProvider); - assertDoesNotThrow(() -> indexingClient.removeDocumentFromResourcesIndex("1234")); - } - - @Test - void shouldNotThrowExceptionWhenTryingToDeleteNonExistingDocumentFromImportCandidateIndex() - throws IOException { - RestHighLevelClientWrapper restHighLevelClient = mock(RestHighLevelClientWrapper.class); - DeleteResponse nothingFoundResponse = mock(DeleteResponse.class); - when(nothingFoundResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); - when(restHighLevelClient.delete(any(), any())).thenReturn(nothingFoundResponse); - IndexingClient indexingClient = new IndexingClient(restHighLevelClient, cachedJwtProvider); - assertDoesNotThrow(() -> indexingClient.removeDocumentFromImportCandidateIndex("1234")); - } - - @Test - void shouldCallEsClientDeleteIndexRequest() throws IOException { - var indicesClient = mock(IndicesClient.class); - var indicesClientWrapper = new IndicesClientWrapper(indicesClient); - when(esClient.indices()).thenReturn(indicesClientWrapper); - indexingClient.deleteIndex(randomString()); - var expectedNumberOfCreateInvocationsToEs = 1; - verify(indicesClient, times(expectedNumberOfCreateInvocationsToEs)) - .delete(any(DeleteIndexRequest.class), any(RequestOptions.class)); - } - - private IndicesClientWrapper createMockIndicesClientWrapper() { - IndicesClient indicesClient = mock(IndicesClient.class); - return new IndicesClientWrapper(indicesClient); - } - - private RestHighLevelClientWrapper setupMockEsClient() throws IOException { - var esClient = mock(RestHighLevelClientWrapper.class); - when(esClient.index(any(IndexRequest.class), any(RequestOptions.class))) - .thenAnswer( - invocation -> { - var indexRequest = (IndexRequest) invocation.getArgument(0); - submittedIndexRequest.set(indexRequest); - return UNUSED_INDEX_RESPONSE; - }); - return esClient; - } - - private IndexDocument sampleIndexDocument() { - EventConsumptionAttributes consumptionAttributes = - new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); - return new IndexDocument(consumptionAttributes, sampleJsonObject()); - } - - private JsonNode sampleJsonObject() { - return attempt(() -> dtoObjectMapper.readTree(randomJson())).orElseThrow(); - } - - private IndexDocument toIndexDocument(String jsonString) { - var consumptionAttributes = - new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); - var json = attempt(() -> objectMapperWithEmpty.readTree(jsonString)).orElseThrow(); - return new IndexDocument(consumptionAttributes, json); - } - - private JsonNode extractDocumentFromSubmittedIndexRequest() throws JsonProcessingException { - return objectMapperWithEmpty.readTree( - submittedIndexRequest.get().source().toBytesRef().utf8ToString()); - } + public static final int + SET_OF_RESOURCES_THAT_DO_NOT_FIT_EXACTLY_IN_THE_BULK_SIZE_OF_A_BULK_REQUEST = 1256; + public static final IndexResponse UNUSED_INDEX_RESPONSE = null; + private RestHighLevelClientWrapper esClient; + private IndexingClient indexingClient; + private AtomicReference submittedIndexRequest; + + private CachedJwtProvider cachedJwtProvider; + + @BeforeEach + public void init() throws IOException { + cachedJwtProvider = setupMockedCachedJwtProvider(); + esClient = setupMockEsClient(); + indexingClient = new IndexingClient(esClient, cachedJwtProvider); + submittedIndexRequest = new AtomicReference<>(); + } + + @Test + void constructorWithSecretsReaderDefinedShouldCreateInstance() { + var secretsReaderMock = mock(SecretsReader.class); + var testCredentials = new UsernamePasswordWrapper("user", "password"); + when(secretsReaderMock.fetchClassSecret(anyString(), eq(UsernamePasswordWrapper.class))) + .thenReturn(testCredentials); + + IndexingClient indexingClient = IndexingClient.prepareWithSecretReader(secretsReaderMock); + assertNotNull(indexingClient); + } + + @Test + void shouldIndexAllDocumentsInBatchInBulksOfSpecifiedSize() throws IOException { + var indexDocuments = + IntStream.range( + 0, SET_OF_RESOURCES_THAT_DO_NOT_FIT_EXACTLY_IN_THE_BULK_SIZE_OF_A_BULK_REQUEST) + .boxed() + .map(i -> randomJson()) + .map(this::toIndexDocument) + .toList(); + List provokeExecution = + indexingClient.batchInsert(indexDocuments.stream()).collect(Collectors.toList()); + assertThat(provokeExecution, is(not(nullValue()))); + + int expectedNumberOfBulkRequests = + (int) Math.ceil(((double) indexDocuments.size()) / ((double) BULK_SIZE)); + verify(esClient, times(expectedNumberOfBulkRequests)) + .bulk(any(BulkRequest.class), any(RequestOptions.class)); + } + + @Test + void shouldSendIndexRequestWithIndexNameSpecifiedByIndexDocument() throws IOException { + var indexDocument = sampleIndexDocument(); + var expectedIndex = indexDocument.consumptionAttributes().index(); + indexingClient.addDocumentToIndex(indexDocument); + + assertThat(submittedIndexRequest.get().index(), is(equalTo(expectedIndex))); + assertThat(extractDocumentFromSubmittedIndexRequest(), is(equalTo(indexDocument.resource()))); + } + + @Test + void shouldCallEsClientCreateIndexRequest() throws IOException { + IndicesClient indicesClient = mock(IndicesClient.class); + var indicesClientWrapper = new IndicesClientWrapper(indicesClient); + when(esClient.indices()).thenReturn(indicesClientWrapper); + indexingClient.createIndex(randomString()); + var expectedNumberOfCreateInvocationsToEs = 1; + verify(indicesClient, times(expectedNumberOfCreateInvocationsToEs)) + .create(any(CreateIndexRequest.class), any(RequestOptions.class)); + } + + @Test + void shouldThrowExceptionWhenEsClientFailedToCreateIndex() throws IOException { + var expectedErrorMessage = randomString(); + var indicesClientWrapper = createMockIndicesClientWrapper(); + when(esClient.indices()).thenReturn(indicesClientWrapper); + when(indicesClientWrapper.create(any(CreateIndexRequest.class), any(RequestOptions.class))) + .thenThrow(new IOException(expectedErrorMessage)); + Executable createIndexAction = () -> indexingClient.createIndex(randomString()); + var actualException = assertThrows(IOException.class, createIndexAction); + assertThat(actualException.getMessage(), containsString(expectedErrorMessage)); + } + + @Test + void shouldThrowExceptionContainingTheCauseWhenIndexDocumentFailsToBeIndexed() + throws IOException { + String expectedMessage = randomString(); + esClient = mock(RestHighLevelClientWrapper.class); + when(esClient.index(any(IndexRequest.class), any(RequestOptions.class))) + .thenThrow(new IOException(expectedMessage)); + + indexingClient = new IndexingClient(esClient, cachedJwtProvider); + + Executable indexingAction = () -> indexingClient.addDocumentToIndex(sampleIndexDocument()); + var exception = assertThrows(IOException.class, indexingAction); + assertThat(exception.getMessage(), containsString(expectedMessage)); + } + + @Test + void shouldNotThrowExceptionWhenTryingToDeleteNonExistingDocument() throws IOException { + RestHighLevelClientWrapper restHighLevelClient = mock(RestHighLevelClientWrapper.class); + DeleteResponse nothingFoundResponse = mock(DeleteResponse.class); + when(nothingFoundResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + when(restHighLevelClient.delete(any(), any())).thenReturn(nothingFoundResponse); + IndexingClient indexingClient = new IndexingClient(restHighLevelClient, cachedJwtProvider); + assertDoesNotThrow(() -> indexingClient.removeDocumentFromResourcesIndex("1234")); + } + + @Test + void shouldNotThrowExceptionWhenTryingToDeleteNonExistingDocumentFromImportCandidateIndex() + throws IOException { + RestHighLevelClientWrapper restHighLevelClient = mock(RestHighLevelClientWrapper.class); + DeleteResponse nothingFoundResponse = mock(DeleteResponse.class); + when(nothingFoundResponse.getResult()).thenReturn(DocWriteResponse.Result.NOT_FOUND); + when(restHighLevelClient.delete(any(), any())).thenReturn(nothingFoundResponse); + IndexingClient indexingClient = new IndexingClient(restHighLevelClient, cachedJwtProvider); + assertDoesNotThrow(() -> indexingClient.removeDocumentFromImportCandidateIndex("1234")); + } + + @Test + void shouldCallEsClientDeleteIndexRequest() throws IOException { + var indicesClient = mock(IndicesClient.class); + var indicesClientWrapper = new IndicesClientWrapper(indicesClient); + when(esClient.indices()).thenReturn(indicesClientWrapper); + indexingClient.deleteIndex(randomString()); + var expectedNumberOfCreateInvocationsToEs = 1; + verify(indicesClient, times(expectedNumberOfCreateInvocationsToEs)) + .delete(any(DeleteIndexRequest.class), any(RequestOptions.class)); + } + + private IndicesClientWrapper createMockIndicesClientWrapper() { + IndicesClient indicesClient = mock(IndicesClient.class); + return new IndicesClientWrapper(indicesClient); + } + + private RestHighLevelClientWrapper setupMockEsClient() throws IOException { + var esClient = mock(RestHighLevelClientWrapper.class); + when(esClient.index(any(IndexRequest.class), any(RequestOptions.class))) + .thenAnswer( + invocation -> { + var indexRequest = (IndexRequest) invocation.getArgument(0); + submittedIndexRequest.set(indexRequest); + return UNUSED_INDEX_RESPONSE; + }); + return esClient; + } + + private IndexDocument sampleIndexDocument() { + EventConsumptionAttributes consumptionAttributes = + new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); + return new IndexDocument(consumptionAttributes, sampleJsonObject()); + } + + private JsonNode sampleJsonObject() { + return attempt(() -> dtoObjectMapper.readTree(randomJson())).orElseThrow(); + } + + private IndexDocument toIndexDocument(String jsonString) { + var consumptionAttributes = + new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); + var json = attempt(() -> objectMapperWithEmpty.readTree(jsonString)).orElseThrow(); + return new IndexDocument(consumptionAttributes, json); + } + + private JsonNode extractDocumentFromSubmittedIndexRequest() throws JsonProcessingException { + return objectMapperWithEmpty.readTree( + submittedIndexRequest.get().source().toBytesRef().utf8ToString()); + } } diff --git a/search-commons/src/test/java/no/unit/nva/indexingclient/models/IndexDocumentTest.java b/search-commons/src/test/java/no/unit/nva/indexingclient/models/IndexDocumentTest.java index 97899f760..77278fabb 100644 --- a/search-commons/src/test/java/no/unit/nva/indexingclient/models/IndexDocumentTest.java +++ b/search-commons/src/test/java/no/unit/nva/indexingclient/models/IndexDocumentTest.java @@ -11,9 +11,7 @@ import static no.unit.nva.indexingclient.models.IndexDocument.TICKET; import static no.unit.nva.testutils.RandomDataGenerator.randomJson; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.attempt.Try.attempt; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.hamcrest.core.IsEqual.equalTo; @@ -23,112 +21,106 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.util.stream.Stream; import no.unit.nva.identifiers.SortableIdentifier; - import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import java.util.stream.Stream; - class IndexDocumentTest { - public static Stream invalidConsumptionAttributes() { - var consumptionAttributesMissingIndexName = - new EventConsumptionAttributes(null, SortableIdentifier.next()); - var consumptionAttributesMissingDocumentIdentifier = - new EventConsumptionAttributes(randomString(), null); - return Stream.of( - consumptionAttributesMissingIndexName, - consumptionAttributesMissingDocumentIdentifier) - .map( - consumptionAttributes -> - new IndexDocument(consumptionAttributes, randomJsonObject())); - } - - private static ObjectNode randomJsonObject() { - String json = randomJson(); - return attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(json)).orElseThrow(); - } - - static Stream nameProvider() { - return Stream.of( - Arguments.of(IMPORT_CANDIDATE, IMPORT_CANDIDATES_INDEX), - Arguments.of(TICKET, TICKETS), - Arguments.of(RESOURCE, RESOURCES)); - } - - @Test - void shouldReturnOpenSearchIndexRequestWithIndexNameSpecifiedByConsumptionAttributes() { - var consumptionAttributes = randomConsumptionAttributes(); - var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); - var indexRequest = indexDocument.toIndexRequest(); - assertThat(indexRequest.index(), is(equalTo(consumptionAttributes.index()))); - } - - @Test - void shouldReturnObjectWhenInputIsValidJsonString() throws JsonProcessingException { - var indexDocument = new IndexDocument(randomConsumptionAttributes(), randomJsonObject()); - var json = objectMapperWithEmpty.writeValueAsString(indexDocument); - var deserialized = IndexDocument.fromJsonString(json); - assertThat(deserialized, is(equalTo(indexDocument))); - } - - @Test - void - shouldReturnDocumentIdentifierOfContainedObjectWhenEventConsumptionAttributesContainIdentifier() { - var indexDocument = new IndexDocument(randomConsumptionAttributes(), randomJsonObject()); - assertThat( - indexDocument.getDocumentIdentifier(), - is(equalTo(indexDocument.consumptionAttributes().documentIdentifier().toString()))); - } - - @Test - void shouldThrowExceptionWhenEventConsumptionAttributesDoNotContainIndexName() { - var consumptionAttributes = new EventConsumptionAttributes(null, SortableIdentifier.next()); - var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); - var error = assertThrows(RuntimeException.class, indexDocument::getIndexName); - assertThat(error.getMessage(), containsString(MISSING_INDEX_NAME_IN_RESOURCE)); - } - - @Test - void shouldThrowExceptionWhenEventConsumptionAttributesDoNotContainDocumentIdentifier() { - var consumptionAttributes = new EventConsumptionAttributes(randomString(), null); - var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); - var error = assertThrows(RuntimeException.class, indexDocument::getDocumentIdentifier); - assertThat(error.getMessage(), containsString(MISSING_IDENTIFIER_IN_RESOURCE)); - } - - @Test - void shouldFailWithInvalidType() { - var consumptionAttributes = - new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); - var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); - assertThrows(IllegalArgumentException.class, indexDocument::getType); - assertNotNull(indexDocument.validate()); - } - - @ParameterizedTest(name = "Checking type:{0}") - @MethodSource("nameProvider") - void shouldUseGetTypeAsWell(String name, String indexName) { - var consumptionAttributes = - new EventConsumptionAttributes(indexName, SortableIdentifier.next()); - var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); - assertThat(indexDocument.getType(), is(name)); - assertNotNull(indexDocument.validate()); - } - - @ParameterizedTest( - name = "should throw exception when validating and missing mandatory fields:{0}") - @MethodSource("invalidConsumptionAttributes") - void shouldThrowExceptionWhenValidatingAndMissingMandatoryFields( - IndexDocument invalidIndexDocument) { - assertThrows(Exception.class, invalidIndexDocument::validate); - } - - private EventConsumptionAttributes randomConsumptionAttributes() { - return new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); - } + public static Stream invalidConsumptionAttributes() { + var consumptionAttributesMissingIndexName = + new EventConsumptionAttributes(null, SortableIdentifier.next()); + var consumptionAttributesMissingDocumentIdentifier = + new EventConsumptionAttributes(randomString(), null); + return Stream.of( + consumptionAttributesMissingIndexName, consumptionAttributesMissingDocumentIdentifier) + .map(consumptionAttributes -> new IndexDocument(consumptionAttributes, randomJsonObject())); + } + + private static ObjectNode randomJsonObject() { + String json = randomJson(); + return attempt(() -> (ObjectNode) objectMapperWithEmpty.readTree(json)).orElseThrow(); + } + + static Stream nameProvider() { + return Stream.of( + Arguments.of(IMPORT_CANDIDATE, IMPORT_CANDIDATES_INDEX), + Arguments.of(TICKET, TICKETS), + Arguments.of(RESOURCE, RESOURCES)); + } + + @Test + void shouldReturnOpenSearchIndexRequestWithIndexNameSpecifiedByConsumptionAttributes() { + var consumptionAttributes = randomConsumptionAttributes(); + var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); + var indexRequest = indexDocument.toIndexRequest(); + assertThat(indexRequest.index(), is(equalTo(consumptionAttributes.index()))); + } + + @Test + void shouldReturnObjectWhenInputIsValidJsonString() throws JsonProcessingException { + var indexDocument = new IndexDocument(randomConsumptionAttributes(), randomJsonObject()); + var json = objectMapperWithEmpty.writeValueAsString(indexDocument); + var deserialized = IndexDocument.fromJsonString(json); + assertThat(deserialized, is(equalTo(indexDocument))); + } + + @Test + void + shouldReturnDocumentIdentifierOfContainedObjectWhenEventConsumptionAttributesContainIdentifier() { + var indexDocument = new IndexDocument(randomConsumptionAttributes(), randomJsonObject()); + assertThat( + indexDocument.getDocumentIdentifier(), + is(equalTo(indexDocument.consumptionAttributes().documentIdentifier().toString()))); + } + + @Test + void shouldThrowExceptionWhenEventConsumptionAttributesDoNotContainIndexName() { + var consumptionAttributes = new EventConsumptionAttributes(null, SortableIdentifier.next()); + var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); + var error = assertThrows(RuntimeException.class, indexDocument::getIndexName); + assertThat(error.getMessage(), containsString(MISSING_INDEX_NAME_IN_RESOURCE)); + } + + @Test + void shouldThrowExceptionWhenEventConsumptionAttributesDoNotContainDocumentIdentifier() { + var consumptionAttributes = new EventConsumptionAttributes(randomString(), null); + var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); + var error = assertThrows(RuntimeException.class, indexDocument::getDocumentIdentifier); + assertThat(error.getMessage(), containsString(MISSING_IDENTIFIER_IN_RESOURCE)); + } + + @Test + void shouldFailWithInvalidType() { + var consumptionAttributes = + new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); + var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); + assertThrows(IllegalArgumentException.class, indexDocument::getType); + assertNotNull(indexDocument.validate()); + } + + @ParameterizedTest(name = "Checking type:{0}") + @MethodSource("nameProvider") + void shouldUseGetTypeAsWell(String name, String indexName) { + var consumptionAttributes = + new EventConsumptionAttributes(indexName, SortableIdentifier.next()); + var indexDocument = new IndexDocument(consumptionAttributes, randomJsonObject()); + assertThat(indexDocument.getType(), is(name)); + assertNotNull(indexDocument.validate()); + } + + @ParameterizedTest( + name = "should throw exception when validating and missing mandatory fields:{0}") + @MethodSource("invalidConsumptionAttributes") + void shouldThrowExceptionWhenValidatingAndMissingMandatoryFields( + IndexDocument invalidIndexDocument) { + assertThrows(Exception.class, invalidIndexDocument::validate); + } + + private EventConsumptionAttributes randomConsumptionAttributes() { + return new EventConsumptionAttributes(randomString(), SortableIdentifier.next()); + } } diff --git a/search-commons/src/test/java/no/unit/nva/indexingclient/utils/HttpRequestMetadataMatcher.java b/search-commons/src/test/java/no/unit/nva/indexingclient/utils/HttpRequestMetadataMatcher.java index b4e3d5e2a..94e7371f6 100644 --- a/search-commons/src/test/java/no/unit/nva/indexingclient/utils/HttpRequestMetadataMatcher.java +++ b/search-commons/src/test/java/no/unit/nva/indexingclient/utils/HttpRequestMetadataMatcher.java @@ -1,8 +1,7 @@ package no.unit.nva.indexingclient.utils; -import org.mockito.ArgumentMatcher; - import java.net.http.HttpRequest; +import org.mockito.ArgumentMatcher; /** * A ArgumentMatcher that will match whenever 2 HttpRequests have the same URI, Headers and Method. @@ -10,16 +9,16 @@ */ public class HttpRequestMetadataMatcher implements ArgumentMatcher { - private final HttpRequest sourceRequest; + private final HttpRequest sourceRequest; - public HttpRequestMetadataMatcher(HttpRequest sourceRequest) { - this.sourceRequest = sourceRequest; - } + public HttpRequestMetadataMatcher(HttpRequest sourceRequest) { + this.sourceRequest = sourceRequest; + } - @Override - public boolean matches(HttpRequest request) { - return request.method().equals(sourceRequest.method()) - && request.uri().equals(sourceRequest.uri()) - && request.headers().equals(sourceRequest.headers()); - } + @Override + public boolean matches(HttpRequest request) { + return request.method().equals(sourceRequest.method()) + && request.uri().equals(sourceRequest.uri()) + && request.headers().equals(sourceRequest.headers()); + } } diff --git a/search-commons/src/test/java/no/unit/nva/search/HardToHitFunctionsTest.java b/search-commons/src/test/java/no/unit/nva/search/HardToHitFunctionsTest.java index a47515def..30a070e6a 100644 --- a/search-commons/src/test/java/no/unit/nva/search/HardToHitFunctionsTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/HardToHitFunctionsTest.java @@ -3,12 +3,14 @@ import static no.unit.nva.constants.Words.COMMA; import static no.unit.nva.constants.Words.SPACE; import static no.unit.nva.search.common.enums.FieldOperator.BETWEEN; - import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.search.common.builder.PartOfQuery; import no.unit.nva.search.common.builder.RangeQuery; import no.unit.nva.search.common.enums.ParameterKey; @@ -19,15 +21,10 @@ import no.unit.nva.search.resource.ResourceSort; import no.unit.nva.search.ticket.TicketParameter; import no.unit.nva.search.ticket.TicketSort; - import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.stream.Collectors; -import java.util.stream.Stream; - class HardToHitFunctionsTest { private static final Logger logger = LoggerFactory.getLogger(HardToHitFunctionsTest.class); private static final String TEST = "test"; diff --git a/search-commons/src/test/java/no/unit/nva/search/common/ContentTypeUtilsTest.java b/search-commons/src/test/java/no/unit/nva/search/common/ContentTypeUtilsTest.java index 4bef7d5c4..96bed2b65 100644 --- a/search-commons/src/test/java/no/unit/nva/search/common/ContentTypeUtilsTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/common/ContentTypeUtilsTest.java @@ -4,55 +4,51 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import java.util.Map; import nva.commons.apigateway.RequestInfo; - import org.junit.jupiter.api.Test; -import java.util.Map; - class ContentTypeUtilsTest { - public static final String ACCEPT_HEADER_VALUE = "application/json; version=2023-05-10"; - public static final String ACCEPT_HEADER_VALUE_WITH_QUOTES = - "application/json; version=\"2023-05-10\""; - public static final String ACCEPT_HEADER_VALUE_WITHOUT_VERSION = "application/json"; - public static final String VERSION_VALUE = "2023-05-10"; - public static final String MIME_TYPE = "application/json"; - - @Test - void asssertThatMimeTypeAndVersionIsExtractedWhenProvided() { - RequestInfo requestInfo = new RequestInfo(); - requestInfo.setHeaders(Map.of(ACCEPT, ACCEPT_HEADER_VALUE)); - - var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); - var mimeType = - ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo).getMimeType(); - - assertThat(mimeType, equalTo(MIME_TYPE)); - assertThat(version, equalTo(VERSION_VALUE)); - } - - @Test - void asssertThatMimeTypeAndVersionIsExtractedWhenProvidedAndVersionHasQuotes() { - RequestInfo requestInfo = new RequestInfo(); - requestInfo.setHeaders(Map.of(ACCEPT, ACCEPT_HEADER_VALUE_WITH_QUOTES)); - - var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); - var mimeType = - ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo).getMimeType(); - - assertThat(mimeType, equalTo(MIME_TYPE)); - assertThat(version, equalTo(VERSION_VALUE)); - } - - @Test - void asssertThatMimeTypeAndVersionAreNullWhenNotProvided() { - RequestInfo requestInfo = new RequestInfo(); - requestInfo.setHeaders(Map.of()); - - var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); - var mimeType = ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo); - - assertThat(mimeType, equalTo(null)); - assertThat(version, equalTo(null)); - } + public static final String ACCEPT_HEADER_VALUE = "application/json; version=2023-05-10"; + public static final String ACCEPT_HEADER_VALUE_WITH_QUOTES = + "application/json; version=\"2023-05-10\""; + public static final String ACCEPT_HEADER_VALUE_WITHOUT_VERSION = "application/json"; + public static final String VERSION_VALUE = "2023-05-10"; + public static final String MIME_TYPE = "application/json"; + + @Test + void asssertThatMimeTypeAndVersionIsExtractedWhenProvided() { + RequestInfo requestInfo = new RequestInfo(); + requestInfo.setHeaders(Map.of(ACCEPT, ACCEPT_HEADER_VALUE)); + + var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); + var mimeType = ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo).getMimeType(); + + assertThat(mimeType, equalTo(MIME_TYPE)); + assertThat(version, equalTo(VERSION_VALUE)); + } + + @Test + void asssertThatMimeTypeAndVersionIsExtractedWhenProvidedAndVersionHasQuotes() { + RequestInfo requestInfo = new RequestInfo(); + requestInfo.setHeaders(Map.of(ACCEPT, ACCEPT_HEADER_VALUE_WITH_QUOTES)); + + var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); + var mimeType = ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo).getMimeType(); + + assertThat(mimeType, equalTo(MIME_TYPE)); + assertThat(version, equalTo(VERSION_VALUE)); + } + + @Test + void asssertThatMimeTypeAndVersionAreNullWhenNotProvided() { + RequestInfo requestInfo = new RequestInfo(); + requestInfo.setHeaders(Map.of()); + + var version = ContentTypeUtils.extractVersionFromRequestInfo(requestInfo); + var mimeType = ContentTypeUtils.extractContentTypeFromRequestInfo(requestInfo); + + assertThat(mimeType, equalTo(null)); + assertThat(version, equalTo(null)); + } } diff --git a/search-commons/src/test/java/no/unit/nva/search/common/builder/PartOfQueryTest.java b/search-commons/src/test/java/no/unit/nva/search/common/builder/PartOfQueryTest.java index 912c3c52c..28c807319 100644 --- a/search-commons/src/test/java/no/unit/nva/search/common/builder/PartOfQueryTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/common/builder/PartOfQueryTest.java @@ -3,16 +3,14 @@ import static org.junit.Assert.assertNotNull; import no.unit.nva.search.resource.ResourceParameter; - import org.junit.jupiter.api.Test; class PartOfQueryTest { - @Test - void checkPartOfQuery() { - var partOfQuery = new PartOfQuery(); - assertNotNull( - partOfQuery.buildMatchAnyValueQuery( - ResourceParameter.CONTRIBUTORS_OF_CHILD, "ewsrdf")); - } + @Test + void checkPartOfQuery() { + var partOfQuery = new PartOfQuery(); + assertNotNull( + partOfQuery.buildMatchAnyValueQuery(ResourceParameter.CONTRIBUTORS_OF_CHILD, "ewsrdf")); + } } diff --git a/search-commons/src/test/java/no/unit/nva/search/importcandidate/ImportCandidateClientTest.java b/search-commons/src/test/java/no/unit/nva/search/importcandidate/ImportCandidateClientTest.java index 0f1664655..a8450886d 100644 --- a/search-commons/src/test/java/no/unit/nva/search/importcandidate/ImportCandidateClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/importcandidate/ImportCandidateClientTest.java @@ -17,7 +17,6 @@ import static no.unit.nva.search.importcandidate.ImportCandidateParameter.SORT; import static no.unit.nva.search.importcandidate.ImportCandidateParameter.TOP_LEVEL_ORGANIZATION; import static no.unit.nva.search.importcandidate.ImportCandidateParameter.TYPE; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -29,23 +28,20 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpResponse; +import java.util.stream.Stream; import no.unit.nva.constants.Words; - import nva.commons.apigateway.exceptions.ApiGatewayException; import nva.commons.apigateway.exceptions.BadRequestException; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpResponse; -import java.util.stream.Stream; - class ImportCandidateClientTest { public static final String REQUEST_BASE_URL = "https://example.com/?"; diff --git a/search-commons/src/test/java/no/unit/nva/search/resource/ResourceClientTest.java b/search-commons/src/test/java/no/unit/nva/search/resource/ResourceClientTest.java index 8be95d458..ea8eb5a3a 100644 --- a/search-commons/src/test/java/no/unit/nva/search/resource/ResourceClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/resource/ResourceClientTest.java @@ -55,11 +55,9 @@ import static no.unit.nva.search.resource.ResourceParameter.SORT; import static no.unit.nva.search.resource.ResourceParameter.STATISTICS; import static no.unit.nva.search.resource.ResourceParameter.UNIT; - import static nva.commons.apigateway.AccessRight.MANAGE_CUSTOMERS; import static nva.commons.apigateway.AccessRight.MANAGE_DEGREE; import static nva.commons.apigateway.AccessRight.MANAGE_RESOURCES_ALL; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -83,7 +81,18 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; - +import java.io.IOException; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.commons.json.JsonUtils; import no.unit.nva.constants.Words; import no.unit.nva.identifiers.SortableIdentifier; @@ -96,14 +105,12 @@ import no.unit.nva.search.common.records.HttpResponseFormatter; import no.unit.nva.search.scroll.ScrollClient; import no.unit.nva.search.scroll.ScrollQuery; - import nva.commons.apigateway.AccessRight; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.ApiGatewayException; import nva.commons.apigateway.exceptions.BadRequestException; import nva.commons.apigateway.exceptions.UnauthorizedException; import nva.commons.core.paths.UriWrapper; - import org.apache.http.HttpHost; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -116,19 +123,6 @@ import org.slf4j.LoggerFactory; import org.testcontainers.junit.jupiter.Testcontainers; -import java.io.IOException; -import java.net.URI; -import java.net.URLEncoder; -import java.net.http.HttpClient; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Testcontainers class ResourceClientTest { @@ -290,12 +284,12 @@ private static IndexDocument indexDocumentWithIdentifier() throws JsonProcessing var identifier = SortableIdentifier.next(); var document = """ - { - "type": "Publication", - "status": "PUBLISHED", - "identifier": "__ID__" - } - """ + { + "type": "Publication", + "status": "PUBLISHED", + "identifier": "__ID__" + } + """ .replace("__ID__", identifier.toString()); var jsonNode = JsonUtils.dtoObjectMapper.readTree(document); return new IndexDocument(new EventConsumptionAttributes(RESOURCES, identifier), jsonNode); diff --git a/search-commons/src/test/java/no/unit/nva/search/resource/ResourceSearchQueryTest.java b/search-commons/src/test/java/no/unit/nva/search/resource/ResourceSearchQueryTest.java index 0faa529e0..7ee44cbec 100644 --- a/search-commons/src/test/java/no/unit/nva/search/resource/ResourceSearchQueryTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/resource/ResourceSearchQueryTest.java @@ -1,5 +1,6 @@ package no.unit.nva.search.resource; +import static java.util.Objects.nonNull; import static no.unit.nva.common.Containers.container; import static no.unit.nva.common.EntrySetTools.queryToMapEntries; import static no.unit.nva.common.MockedHttpResponse.mockedFutureHttpResponse; @@ -16,7 +17,6 @@ import static no.unit.nva.search.resource.ResourceParameter.SCIENTIFIC_REPORT_PERIOD_SINCE; import static no.unit.nva.search.resource.ResourceParameter.SIZE; import static no.unit.nva.search.resource.ResourceParameter.SORT; - import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -27,13 +27,16 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.util.Objects.nonNull; - +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.search.common.records.PagedSearch; - import nva.commons.apigateway.exceptions.BadRequestException; import nva.commons.core.paths.UriWrapper; - import org.joda.time.DateTime; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -41,223 +44,210 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.http.HttpClient; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; - class ResourceSearchQueryTest { - private static final Logger logger = LoggerFactory.getLogger(ResourceSearchQueryTest.class); - - static Stream uriProvider() { - return Stream.of( - URI.create("https://example.com/"), - URI.create("https://example.com/?fields=category,title,created_date"), - URI.create("https://example.com/?query=Muhammad+Yahya&fields=CONTRIBUTOR"), - URI.create( - "https://example.com/?CONTRIBUTOR=https://api.dev.nva.aws.unit.no/cristin/person/1136254"), - URI.create( - "https://example.com/?CONTRIBUTOR_NOT=" - + "https://api.dev.nva.aws.unit.no/cristin/person/1136254," - + "https://api.dev.nva.aws.unit.no/cristin/person/1135555"), - URI.create("https://example.com/?fields=all"), - URI.create("https://example.com/?category=hello+world&page=1&user=12%203"), - URI.create("https://example.com/?category=hello+world&sort=created_date&order=asc"), - URI.create("https://example.com/?category=hello+world&sort=created_date:ASC"), - URI.create("https://example.com/?category=hello+world&sort=created_date"), - URI.create("https://example.com/?category=hello+world&user=12%203&page=2"), - URI.create("https://example.com/?category=hello+world&user=12%203&offset=30"), - URI.create( - "https://example.com/?category=hello+world&user=12%203&from=30&results=10"), - URI.create( - "https://example.com/?PARENT_PUBLICATION=https://api.dev.nva.aws.unit" - + ".no/publication/018b80c90f4a-75942f6d-544e-4d5b-8129-7b81b957678c"), - URI.create("https://example.com/?published_before=2020-01-01&user=1%2023"), - URI.create( - "https://example.com/?published_since=2019-01-01&institution=uib&funding_source=NFR&user=Per" - + "%20Eplekjekk")); - } - - static Stream uriSortingProvider() { - return Stream.of( - URI.create( - "https://example.com/?sort=category&sortOrder=asc&sort=created_date&order=desc"), - URI.create("https://example.com/?orderBy=category:asc,created_date:desc"), - URI.create("https://example.com/?sort=category+asc&sort=created_date+desc")); - } - - static Stream uriDatesProvider() { - return Stream.of( - URI.create( - "https://example.com/?category=hello&modified_before=2020-01-01&modified_since=2019-01-01"), - URI.create( - "https://example.com/?published_before=2020-01-02&published_since=2019-01-02"), - URI.create("https://example.com/?published_before=2020&published_since=2019"), - URI.create( - "https://example.com/?created_before=2020-01-01T23:59:59&created_since=2019-01-01")); + private static final Logger logger = LoggerFactory.getLogger(ResourceSearchQueryTest.class); + + static Stream uriProvider() { + return Stream.of( + URI.create("https://example.com/"), + URI.create("https://example.com/?fields=category,title,created_date"), + URI.create("https://example.com/?query=Muhammad+Yahya&fields=CONTRIBUTOR"), + URI.create( + "https://example.com/?CONTRIBUTOR=https://api.dev.nva.aws.unit.no/cristin/person/1136254"), + URI.create( + "https://example.com/?CONTRIBUTOR_NOT=" + + "https://api.dev.nva.aws.unit.no/cristin/person/1136254," + + "https://api.dev.nva.aws.unit.no/cristin/person/1135555"), + URI.create("https://example.com/?fields=all"), + URI.create("https://example.com/?category=hello+world&page=1&user=12%203"), + URI.create("https://example.com/?category=hello+world&sort=created_date&order=asc"), + URI.create("https://example.com/?category=hello+world&sort=created_date:ASC"), + URI.create("https://example.com/?category=hello+world&sort=created_date"), + URI.create("https://example.com/?category=hello+world&user=12%203&page=2"), + URI.create("https://example.com/?category=hello+world&user=12%203&offset=30"), + URI.create("https://example.com/?category=hello+world&user=12%203&from=30&results=10"), + URI.create( + "https://example.com/?PARENT_PUBLICATION=https://api.dev.nva.aws.unit" + + ".no/publication/018b80c90f4a-75942f6d-544e-4d5b-8129-7b81b957678c"), + URI.create("https://example.com/?published_before=2020-01-01&user=1%2023"), + URI.create( + "https://example.com/?published_since=2019-01-01&institution=uib&funding_source=NFR&user=Per" + + "%20Eplekjekk")); + } + + static Stream uriSortingProvider() { + return Stream.of( + URI.create("https://example.com/?sort=category&sortOrder=asc&sort=created_date&order=desc"), + URI.create("https://example.com/?orderBy=category:asc,created_date:desc"), + URI.create("https://example.com/?sort=category+asc&sort=created_date+desc")); + } + + static Stream uriDatesProvider() { + return Stream.of( + URI.create( + "https://example.com/?category=hello&modified_before=2020-01-01&modified_since=2019-01-01"), + URI.create("https://example.com/?published_before=2020-01-02&published_since=2019-01-02"), + URI.create("https://example.com/?published_before=2020&published_since=2019"), + URI.create( + "https://example.com/?created_before=2020-01-01T23:59:59&created_since=2019-01-01")); + } + + static Stream invalidUriProvider() { + return Stream.of( + URI.create("https://example.com/?dcategory=hello+world&page=0"), + URI.create("https://example.com/?publishedbefore=202t0&lang=en&user="), + URI.create("https://example.com/?publishedbefore=202t0&lang=en&"), + URI.create("https://example.com/?publishedbefore=2020&sort=category:BESC"), + URI.create("https://example.com/?publishedbefore=2020&sort=category:BESC:AS"), + URI.create("https://example.com/?institutions=uib&funding=NFR&lang=en")); + } + + private static void debugLogging(ResourceParameter key, ResourceSearchQuery query) { + logger.debug("{} : {}", key.asCamelCase(), query.parameters().get(key).as()); + } + + @Test + void emptyPageSearch() { + var page = new PagedSearch(null, 0, null, null, null, null, null); + assertEquals(page.aggregations(), Map.of()); + } + + @Test + void removeKeySuccessfully() throws BadRequestException { + var response = + ResourceSearchQuery.builder() + .fromTestParameterMap( + Map.of( + SCIENTIFIC_REPORT_PERIOD_SINCE.asCamelCase(), + "2019", + SCIENTIFIC_REPORT_PERIOD_BEFORE.asCamelCase(), + "2020")) + .withRequiredParameters(FROM, SIZE, AGGREGATION) + .withDockerHostUri(URI.create(container.getHttpHostAddress())) + .build(); + + var key = response.parameters().remove(SCIENTIFIC_REPORT_PERIOD_SINCE); + + assertEquals(SCIENTIFIC_REPORT_PERIOD_SINCE, key.getKey()); + } + + @Test + void openSearchFailedResponse() { + HttpClient httpClient = mock(HttpClient.class); + when(httpClient.sendAsync(any(), any())).thenReturn(mockedFutureHttpResponse((String) null)); + var toMapEntries = queryToMapEntries(URI.create("https://example.com/?size=2")); + var resourceClient = new ResourceClient(httpClient, setupMockedCachedJwtProvider()); + assertThrows( + RuntimeException.class, + () -> + ResourceSearchQuery.builder() + .withRequiredParameters(SIZE, FROM) + .fromTestQueryParameters(toMapEntries) + .build() + .doSearch(resourceClient)); + } + + @Test + void missingRequiredException() { + var toMapEntries = queryToMapEntries(URI.create("https://example.com/?doi=2&Title=wqerasdfg")); + assertThrows( + BadRequestException.class, + () -> + ResourceSearchQuery.builder() + .withRequiredParameters(ABSTRACT, FUNDING) + .fromTestQueryParameters(toMapEntries) + .validate() + .build()); + } + + @ParameterizedTest + @MethodSource("uriProvider") + void buildOpenSearchSwsUriFromGatewayUri(URI uri) + throws BadRequestException, MalformedURLException, URISyntaxException { + var resource = + ResourceSearchQuery.builder() + .fromTestQueryParameters(queryToMapEntries(uri)) + .withRequiredParameters(FROM, SIZE) + .build(); + assertNotNull(resource.parameters().get(FROM).as()); + assertNotNull(resource.parameters().get(SIZE).as()); + var uri2 = + UriWrapper.fromUri(resource.getNvaSearchApiUri()) + .addQueryParameters(resource.parameters().asMap()) + .getUri(); + assertDoesNotThrow(uri2::toURL, "MalformedURLException"); + var url2 = uri2.toURL(); + assertDoesNotThrow(url2::toURI, "URISyntaxException"); + logger.debug( + resource.parameters().asMap().entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining("&"))); + logger.debug(uri2.toString()); + assertNotEquals(uri, resource.getNvaSearchApiUri()); + } + + @ParameterizedTest + @MethodSource("uriDatesProvider") + void uriParamsDateToResourceParams(URI uri) throws BadRequestException { + var query = + ResourceSearchQuery.builder() + .fromTestQueryParameters(queryToMapEntries(uri)) + .withRequiredParameters(FROM, SIZE) + .build(); + + query.parameters().getSearchKeys().forEach(key -> debugLogging(key, query)); + + // two ways to access keys + + var modified = query.parameters().get(MODIFIED_BEFORE); + if (!modified.isEmpty()) { + logger.debug("Modified-1: {}", modified.asDateTime()); } - - static Stream invalidUriProvider() { - return Stream.of( - URI.create("https://example.com/?dcategory=hello+world&page=0"), - URI.create("https://example.com/?publishedbefore=202t0&lang=en&user="), - URI.create("https://example.com/?publishedbefore=202t0&lang=en&"), - URI.create("https://example.com/?publishedbefore=2020&sort=category:BESC"), - URI.create("https://example.com/?publishedbefore=2020&sort=category:BESC:AS"), - URI.create("https://example.com/?institutions=uib&funding=NFR&lang=en")); - } - - private static void debugLogging(ResourceParameter key, ResourceSearchQuery query) { - logger.debug("{} : {}", key.asCamelCase(), query.parameters().get(key).as()); - } - - @Test - void emptyPageSearch() { - var page = new PagedSearch(null, 0, null, null, null, null, null); - assertEquals(page.aggregations(), Map.of()); - } - - @Test - void removeKeySuccessfully() throws BadRequestException { - var response = - ResourceSearchQuery.builder() - .fromTestParameterMap( - Map.of( - SCIENTIFIC_REPORT_PERIOD_SINCE.asCamelCase(), - "2019", - SCIENTIFIC_REPORT_PERIOD_BEFORE.asCamelCase(), - "2020")) - .withRequiredParameters(FROM, SIZE, AGGREGATION) - .withDockerHostUri(URI.create(container.getHttpHostAddress())) - .build(); - - var key = response.parameters().remove(SCIENTIFIC_REPORT_PERIOD_SINCE); - - assertEquals(SCIENTIFIC_REPORT_PERIOD_SINCE, key.getKey()); - } - - @Test - void openSearchFailedResponse() { - HttpClient httpClient = mock(HttpClient.class); - when(httpClient.sendAsync(any(), any())) - .thenReturn(mockedFutureHttpResponse((String) null)); - var toMapEntries = queryToMapEntries(URI.create("https://example.com/?size=2")); - var resourceClient = new ResourceClient(httpClient, setupMockedCachedJwtProvider()); - assertThrows( - RuntimeException.class, - () -> - ResourceSearchQuery.builder() - .withRequiredParameters(SIZE, FROM) - .fromTestQueryParameters(toMapEntries) - .build() - .doSearch(resourceClient)); - } - - @Test - void missingRequiredException() { - var toMapEntries = - queryToMapEntries(URI.create("https://example.com/?doi=2&Title=wqerasdfg")); - assertThrows( - BadRequestException.class, - () -> - ResourceSearchQuery.builder() - .withRequiredParameters(ABSTRACT, FUNDING) - .fromTestQueryParameters(toMapEntries) - .validate() - .build()); - } - - @ParameterizedTest - @MethodSource("uriProvider") - void buildOpenSearchSwsUriFromGatewayUri(URI uri) - throws BadRequestException, MalformedURLException, URISyntaxException { - var resource = - ResourceSearchQuery.builder() - .fromTestQueryParameters(queryToMapEntries(uri)) - .withRequiredParameters(FROM, SIZE) - .build(); - assertNotNull(resource.parameters().get(FROM).as()); - assertNotNull(resource.parameters().get(SIZE).as()); - var uri2 = - UriWrapper.fromUri(resource.getNvaSearchApiUri()) - .addQueryParameters(resource.parameters().asMap()) - .getUri(); - assertDoesNotThrow(uri2::toURL, "MalformedURLException"); - var url2 = uri2.toURL(); - assertDoesNotThrow(url2::toURI, "URISyntaxException"); - logger.debug( - resource.parameters().asMap().entrySet().stream() - .map(entry -> entry.getKey() + "=" + entry.getValue()) - .collect(Collectors.joining("&"))); - logger.debug(uri2.toString()); - assertNotEquals(uri, resource.getNvaSearchApiUri()); - } - - @ParameterizedTest - @MethodSource("uriDatesProvider") - void uriParamsDateToResourceParams(URI uri) throws BadRequestException { - var query = - ResourceSearchQuery.builder() - .fromTestQueryParameters(queryToMapEntries(uri)) - .withRequiredParameters(FROM, SIZE) - .build(); - - query.parameters().getSearchKeys().forEach(key -> debugLogging(key, query)); - - // two ways to access keys - - var modified = query.parameters().get(MODIFIED_BEFORE); - if (!modified.isEmpty()) { - logger.debug("Modified-1: {}", modified.asDateTime()); - } - var published = query.parameters().ifPresent(PUBLISHED_BEFORE); - if (nonNull(published)) { - logger.debug("Published-1: {}", published.as()); - } - assertNotNull(query.parameters()); - } - - @ParameterizedTest - @MethodSource("uriSortingProvider") - void uriParamsToResourceParams(URI uri) throws BadRequestException { - var resource = - ResourceSearchQuery.builder() - .fromTestQueryParameters(queryToMapEntries(uri)) - .withRequiredParameters(FROM, SIZE) - .build(); - assertNotNull(resource.parameters().get(FROM).as()); - assertNull(resource.parameters().get(PAGE).as()); - assertNotNull(resource.parameters().get(SORT).as()); - } - - @ParameterizedTest - @MethodSource("uriProvider") - void failToBuildOpenSearchSwsUriFromMissingRequired(URI uri) { - assertThrows( - BadRequestException.class, - () -> - ResourceSearchQuery.builder() - .fromTestQueryParameters(queryToMapEntries(uri)) - .withRequiredParameters(FROM, SIZE, DOI) - .validate() - .build() - .openSearchUri()); - } - - @ParameterizedTest - @MethodSource("invalidUriProvider") - void failToBuildOpenSearchSwsUriFromInvalidGatewayUri(URI uri) { - assertThrows( - BadRequestException.class, - () -> - ResourceSearchQuery.builder() - .fromTestQueryParameters(queryToMapEntries(uri)) - .withRequiredParameters(FROM, SIZE) - .build() - .openSearchUri()); + var published = query.parameters().ifPresent(PUBLISHED_BEFORE); + if (nonNull(published)) { + logger.debug("Published-1: {}", published.as()); } + assertNotNull(query.parameters()); + } + + @ParameterizedTest + @MethodSource("uriSortingProvider") + void uriParamsToResourceParams(URI uri) throws BadRequestException { + var resource = + ResourceSearchQuery.builder() + .fromTestQueryParameters(queryToMapEntries(uri)) + .withRequiredParameters(FROM, SIZE) + .build(); + assertNotNull(resource.parameters().get(FROM).as()); + assertNull(resource.parameters().get(PAGE).as()); + assertNotNull(resource.parameters().get(SORT).as()); + } + + @ParameterizedTest + @MethodSource("uriProvider") + void failToBuildOpenSearchSwsUriFromMissingRequired(URI uri) { + assertThrows( + BadRequestException.class, + () -> + ResourceSearchQuery.builder() + .fromTestQueryParameters(queryToMapEntries(uri)) + .withRequiredParameters(FROM, SIZE, DOI) + .validate() + .build() + .openSearchUri()); + } + + @ParameterizedTest + @MethodSource("invalidUriProvider") + void failToBuildOpenSearchSwsUriFromInvalidGatewayUri(URI uri) { + assertThrows( + BadRequestException.class, + () -> + ResourceSearchQuery.builder() + .fromTestQueryParameters(queryToMapEntries(uri)) + .withRequiredParameters(FROM, SIZE) + .build() + .openSearchUri()); + } } diff --git a/search-commons/src/test/java/no/unit/nva/search/resource/SimplifiedMutatorTest.java b/search-commons/src/test/java/no/unit/nva/search/resource/SimplifiedMutatorTest.java index 6a6e98cda..3a31576f1 100644 --- a/search-commons/src/test/java/no/unit/nva/search/resource/SimplifiedMutatorTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/resource/SimplifiedMutatorTest.java @@ -2,9 +2,7 @@ import static no.unit.nva.testutils.RandomDataGenerator.objectMapper; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.hasItem; import static org.hamcrest.CoreMatchers.is; @@ -18,13 +16,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; - +import java.nio.file.Path; import no.unit.nva.search.resource.response.ResourceSearchResponse; - import org.junit.jupiter.api.Test; -import java.nio.file.Path; - class SimplifiedMutatorTest { @Test diff --git a/search-commons/src/test/java/no/unit/nva/search/resource/UserSettingsClientTest.java b/search-commons/src/test/java/no/unit/nva/search/resource/UserSettingsClientTest.java index 27dd9d241..a7ee6c747 100644 --- a/search-commons/src/test/java/no/unit/nva/search/resource/UserSettingsClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/resource/UserSettingsClientTest.java @@ -6,33 +6,28 @@ import static no.unit.nva.indexing.testutils.MockedJwtProvider.setupMockedCachedJwtProvider; import static no.unit.nva.search.resource.ResourceParameter.FROM; import static no.unit.nva.search.resource.ResourceParameter.SIZE; - import static nva.commons.core.StringUtils.EMPTY_STRING; import static nva.commons.core.attempt.Try.attempt; - import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.net.URI; +import java.net.http.HttpClient; +import java.nio.file.Path; +import java.util.List; +import java.util.stream.Stream; import no.unit.nva.search.common.records.UserSettings; - import nva.commons.apigateway.exceptions.ApiGatewayException; import nva.commons.core.attempt.Failure; import nva.commons.core.attempt.FunctionWithException; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.URI; -import java.net.http.HttpClient; -import java.nio.file.Path; -import java.util.List; -import java.util.stream.Stream; - class UserSettingsClientTest { public static final String SAMPLE_USER_SETTINGS_RESPONSE = "user_settings.json"; diff --git a/search-commons/src/test/java/no/unit/nva/search/scroll/ScrollClientTest.java b/search-commons/src/test/java/no/unit/nva/search/scroll/ScrollClientTest.java index 3f7a5b8b9..bdeaa6f6c 100644 --- a/search-commons/src/test/java/no/unit/nva/search/scroll/ScrollClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/scroll/ScrollClientTest.java @@ -3,53 +3,50 @@ import static no.unit.nva.common.MockedHttpResponse.mockedFutureHttpResponse; import static no.unit.nva.indexing.testutils.MockedJwtProvider.setupMockedCachedJwtProvider; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import java.net.http.HttpClient; +import java.nio.file.Path; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.net.http.HttpClient; -import java.nio.file.Path; - class ScrollClientTest { - public static final String SAMPLE_PUBLICATION_SEARCH = - "resource_mocked_sws_search_response.json"; - private static final Logger logger = LoggerFactory.getLogger(ScrollClientTest.class); - private ScrollClient scrollClient; - - @BeforeEach - public void setUp() { - var httpClient = mock(HttpClient.class); - var cachedJwtProvider = setupMockedCachedJwtProvider(); - scrollClient = new ScrollClient(httpClient, cachedJwtProvider); - when(httpClient.sendAsync(any(), any())) - .thenReturn(mockedFutureHttpResponse(Path.of(SAMPLE_PUBLICATION_SEARCH))); - } - - @Test - void searchWithUriReturnsOpenSearchAwsResponse() { - var scrollId = randomString(); - var resourceAwsQuery = new ScrollQuery(scrollId, "1m"); - var result = scrollClient.doSearch(resourceAwsQuery); - logger.debug(result.toString()); - assertNotNull(result); - assertNotNull(ScrollParameter.INVALID.asCamelCase()); - assertNotNull(ScrollParameter.INVALID.asLowerCase()); - assertNotNull(ScrollParameter.INVALID.errorMessage()); - assertNotNull(ScrollParameter.INVALID.fieldBoost()); - assertNotNull(ScrollParameter.INVALID.fieldPattern()); - assertNotNull(ScrollParameter.INVALID.fieldPattern()); - assertNotNull(ScrollParameter.INVALID.fieldType()); - assertNotNull(ScrollParameter.INVALID.searchOperator()); - assertNotNull(ScrollParameter.INVALID.searchFields(true)); - assertNotNull(ScrollParameter.INVALID.valueEncoding()); - assertNotNull(ScrollParameter.INVALID.valuePattern()); - } + public static final String SAMPLE_PUBLICATION_SEARCH = "resource_mocked_sws_search_response.json"; + private static final Logger logger = LoggerFactory.getLogger(ScrollClientTest.class); + private ScrollClient scrollClient; + + @BeforeEach + public void setUp() { + var httpClient = mock(HttpClient.class); + var cachedJwtProvider = setupMockedCachedJwtProvider(); + scrollClient = new ScrollClient(httpClient, cachedJwtProvider); + when(httpClient.sendAsync(any(), any())) + .thenReturn(mockedFutureHttpResponse(Path.of(SAMPLE_PUBLICATION_SEARCH))); + } + + @Test + void searchWithUriReturnsOpenSearchAwsResponse() { + var scrollId = randomString(); + var resourceAwsQuery = new ScrollQuery(scrollId, "1m"); + var result = scrollClient.doSearch(resourceAwsQuery); + logger.debug(result.toString()); + assertNotNull(result); + assertNotNull(ScrollParameter.INVALID.asCamelCase()); + assertNotNull(ScrollParameter.INVALID.asLowerCase()); + assertNotNull(ScrollParameter.INVALID.errorMessage()); + assertNotNull(ScrollParameter.INVALID.fieldBoost()); + assertNotNull(ScrollParameter.INVALID.fieldPattern()); + assertNotNull(ScrollParameter.INVALID.fieldPattern()); + assertNotNull(ScrollParameter.INVALID.fieldType()); + assertNotNull(ScrollParameter.INVALID.searchOperator()); + assertNotNull(ScrollParameter.INVALID.searchFields(true)); + assertNotNull(ScrollParameter.INVALID.valueEncoding()); + assertNotNull(ScrollParameter.INVALID.valuePattern()); + } } diff --git a/search-commons/src/test/java/no/unit/nva/search/ticket/TicketClientTest.java b/search-commons/src/test/java/no/unit/nva/search/ticket/TicketClientTest.java index 2a624d89b..74953296c 100644 --- a/search-commons/src/test/java/no/unit/nva/search/ticket/TicketClientTest.java +++ b/search-commons/src/test/java/no/unit/nva/search/ticket/TicketClientTest.java @@ -1,5 +1,6 @@ package no.unit.nva.search.ticket; +import static java.util.Objects.nonNull; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.common.Containers.container; import static no.unit.nva.common.Containers.indexingClient; @@ -20,7 +21,6 @@ import static no.unit.nva.search.ticket.TicketParameter.SORT; import static no.unit.nva.search.ticket.TicketParameter.STATISTICS; import static no.unit.nva.testutils.RandomDataGenerator.randomString; - import static nva.commons.apigateway.AccessRight.MANAGE_CUSTOMERS; import static nva.commons.apigateway.AccessRight.MANAGE_DOI; import static nva.commons.apigateway.AccessRight.MANAGE_PUBLISHING_REQUESTS; @@ -28,7 +28,6 @@ import static nva.commons.core.attempt.Try.attempt; import static nva.commons.core.ioutils.IoUtils.stringFromResources; import static nva.commons.core.paths.UriWrapper.fromUri; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -42,20 +41,25 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.util.Objects.nonNull; - import com.fasterxml.jackson.core.type.TypeReference; - +import java.net.URI; +import java.net.http.HttpClient; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import java.util.stream.Stream; import no.unit.nva.commons.json.JsonUtils; import no.unit.nva.constants.Words; import no.unit.nva.search.common.records.Facet; - import nva.commons.apigateway.AccessRight; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.ApiGatewayException; import nva.commons.apigateway.exceptions.BadRequestException; import nva.commons.apigateway.exceptions.UnauthorizedException; - import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; @@ -66,17 +70,6 @@ import org.slf4j.LoggerFactory; import org.testcontainers.junit.jupiter.Testcontainers; -import java.net.URI; -import java.net.http.HttpClient; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; -import java.util.stream.Stream; - @Testcontainers class TicketClientTest { diff --git a/search-handlers/src/main/java/no/unit/nva/search/ExportResourceHandler.java b/search-handlers/src/main/java/no/unit/nva/search/ExportResourceHandler.java index ddc71a7dd..628da0dc6 100644 --- a/search-handlers/src/main/java/no/unit/nva/search/ExportResourceHandler.java +++ b/search-handlers/src/main/java/no/unit/nva/search/ExportResourceHandler.java @@ -11,18 +11,15 @@ import static no.unit.nva.search.resource.ResourceParameter.SORT; import com.amazonaws.services.lambda.runtime.Context; - import no.unit.nva.search.common.csv.ResourceCsvTransformer; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.search.resource.ResourceSearchQuery; import no.unit.nva.search.scroll.ScrollClient; import no.unit.nva.search.scroll.ScrollQuery; - import nva.commons.apigateway.ApiS3GatewayHandler; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.BadRequestException; import nva.commons.core.JacocoGenerated; - import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.presigner.S3Presigner; @@ -34,74 +31,74 @@ */ public class ExportResourceHandler extends ApiS3GatewayHandler { - public static final String MAX_HITS_PER_PAGE = "2500"; - public static final String SCROLL_TTL = "1m"; - public static final String INCLUDED_NODES = - String.join(COMMA, ResourceCsvTransformer.getJsonFields()); - private final ResourceClient opensearchClient; - private final ScrollClient scrollClient; + public static final String MAX_HITS_PER_PAGE = "2500"; + public static final String SCROLL_TTL = "1m"; + public static final String INCLUDED_NODES = + String.join(COMMA, ResourceCsvTransformer.getJsonFields()); + private final ResourceClient opensearchClient; + private final ScrollClient scrollClient; - @JacocoGenerated - public ExportResourceHandler() { - this( - ResourceClient.defaultClient(), - ScrollClient.defaultClient(), - defaultS3Client(), - defaultS3Presigner()); - } + @JacocoGenerated + public ExportResourceHandler() { + this( + ResourceClient.defaultClient(), + ScrollClient.defaultClient(), + defaultS3Client(), + defaultS3Presigner()); + } - public ExportResourceHandler( - ResourceClient resourceClient, - ScrollClient scrollClient, - S3Client s3Client, - S3Presigner s3Presigner) { - super(Void.class, s3Client, s3Presigner); - this.opensearchClient = resourceClient; - this.scrollClient = scrollClient; - } + public ExportResourceHandler( + ResourceClient resourceClient, + ScrollClient scrollClient, + S3Client s3Client, + S3Presigner s3Presigner) { + super(Void.class, s3Client, s3Presigner); + this.opensearchClient = resourceClient; + this.scrollClient = scrollClient; + } - @Override - public String processS3Input(Void input, RequestInfo requestInfo, Context context) - throws BadRequestException { - var initialResponse = - ResourceSearchQuery.builder() - .fromRequestInfo(requestInfo) - .withParameter(FROM, ZERO) - .withParameter(SIZE, MAX_HITS_PER_PAGE) - .withParameter(AGGREGATION, NONE) - .withParameter(NODES_INCLUDED, INCLUDED_NODES) - .withRequiredParameters(SORT) - .build() - .withFilter() - .requiredStatus(PUBLISHED, PUBLISHED_METADATA) - .apply() - .withScrollTime(SCROLL_TTL) - .doSearch(opensearchClient) - .swsResponse(); + @Override + public String processS3Input(Void input, RequestInfo requestInfo, Context context) + throws BadRequestException { + var initialResponse = + ResourceSearchQuery.builder() + .fromRequestInfo(requestInfo) + .withParameter(FROM, ZERO) + .withParameter(SIZE, MAX_HITS_PER_PAGE) + .withParameter(AGGREGATION, NONE) + .withParameter(NODES_INCLUDED, INCLUDED_NODES) + .withRequiredParameters(SORT) + .build() + .withFilter() + .requiredStatus(PUBLISHED, PUBLISHED_METADATA) + .apply() + .withScrollTime(SCROLL_TTL) + .doSearch(opensearchClient) + .swsResponse(); - return ScrollQuery.builder() - .withInitialResponse(initialResponse) - .withScrollTime(SCROLL_TTL) - .build() - .doSearch(scrollClient) - .toCsvText(); - } + return ScrollQuery.builder() + .withInitialResponse(initialResponse) + .withScrollTime(SCROLL_TTL) + .build() + .doSearch(scrollClient) + .toCsvText(); + } - @JacocoGenerated - @Override - protected String getContentType() { - return "text/csv"; - } + @JacocoGenerated + @Override + protected String getContentType() { + return "text/csv"; + } - @JacocoGenerated - @Override - protected void validateRequest(Void unused, RequestInfo requestInfo, Context context) { - // Do nothing - } + @JacocoGenerated + @Override + protected void validateRequest(Void unused, RequestInfo requestInfo, Context context) { + // Do nothing + } - @JacocoGenerated - @Override - protected Integer getSuccessStatusCode(Void input, Void output) { - return super.getSuccessStatusCode(input, output); - } + @JacocoGenerated + @Override + protected Integer getSuccessStatusCode(Void input, Void output) { + return super.getSuccessStatusCode(input, output); + } } diff --git a/search-handlers/src/main/java/no/unit/nva/search/SearchImportCandidateAuthHandler.java b/search-handlers/src/main/java/no/unit/nva/search/SearchImportCandidateAuthHandler.java index f4e70a27c..0ffd10e34 100644 --- a/search-handlers/src/main/java/no/unit/nva/search/SearchImportCandidateAuthHandler.java +++ b/search-handlers/src/main/java/no/unit/nva/search/SearchImportCandidateAuthHandler.java @@ -8,10 +8,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.google.common.net.MediaType; - +import java.net.HttpURLConnection; +import java.util.List; import no.unit.nva.search.importcandidate.ImportCandidateClient; import no.unit.nva.search.importcandidate.ImportCandidateSearchQuery; - import nva.commons.apigateway.ApiGatewayHandler; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.ApiGatewayException; @@ -19,9 +19,6 @@ import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; -import java.net.HttpURLConnection; -import java.util.List; - /** * Handler for searching import candidates. * diff --git a/search-handlers/src/main/java/no/unit/nva/search/SearchResourceAuthHandler.java b/search-handlers/src/main/java/no/unit/nva/search/SearchResourceAuthHandler.java index f09f65bd4..4f2c78ecb 100644 --- a/search-handlers/src/main/java/no/unit/nva/search/SearchResourceAuthHandler.java +++ b/search-handlers/src/main/java/no/unit/nva/search/SearchResourceAuthHandler.java @@ -14,14 +14,14 @@ import com.amazonaws.services.lambda.runtime.Context; import com.google.common.net.MediaType; - +import java.net.HttpURLConnection; +import java.util.List; import no.unit.nva.search.common.ContentTypeUtils; import no.unit.nva.search.common.records.JsonNodeMutator; import no.unit.nva.search.resource.LegacyMutator; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.search.resource.ResourceSearchQuery; import no.unit.nva.search.resource.SimplifiedMutator; - import nva.commons.apigateway.AccessRight; import nva.commons.apigateway.ApiGatewayHandler; import nva.commons.apigateway.RequestInfo; @@ -31,9 +31,6 @@ import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; -import java.net.HttpURLConnection; -import java.util.List; - /** * Handler for searching resources. * diff --git a/search-handlers/src/main/java/no/unit/nva/search/SearchResourceHandler.java b/search-handlers/src/main/java/no/unit/nva/search/SearchResourceHandler.java index 0a0fb3d79..38df6f88a 100644 --- a/search-handlers/src/main/java/no/unit/nva/search/SearchResourceHandler.java +++ b/search-handlers/src/main/java/no/unit/nva/search/SearchResourceHandler.java @@ -12,27 +12,23 @@ import com.amazonaws.services.lambda.runtime.Context; import com.google.common.net.MediaType; - +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; import no.unit.nva.search.common.ContentTypeUtils; import no.unit.nva.search.common.records.JsonNodeMutator; import no.unit.nva.search.resource.LegacyMutator; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.search.resource.ResourceSearchQuery; import no.unit.nva.search.resource.SimplifiedMutator; - import nva.commons.apigateway.ApiGatewayHandler; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.BadRequestException; import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; - import org.apache.http.HttpHeaders; -import java.net.HttpURLConnection; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - /** * Handler for searching resources. * diff --git a/search-handlers/src/main/java/no/unit/nva/search/SearchTicketAuthHandler.java b/search-handlers/src/main/java/no/unit/nva/search/SearchTicketAuthHandler.java index a209926c0..1a57f6541 100644 --- a/search-handlers/src/main/java/no/unit/nva/search/SearchTicketAuthHandler.java +++ b/search-handlers/src/main/java/no/unit/nva/search/SearchTicketAuthHandler.java @@ -8,10 +8,10 @@ import com.amazonaws.services.lambda.runtime.Context; import com.google.common.net.MediaType; - +import java.net.HttpURLConnection; +import java.util.List; import no.unit.nva.search.ticket.TicketClient; import no.unit.nva.search.ticket.TicketSearchQuery; - import nva.commons.apigateway.ApiGatewayHandler; import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.ApiGatewayException; @@ -20,9 +20,6 @@ import nva.commons.core.Environment; import nva.commons.core.JacocoGenerated; -import java.net.HttpURLConnection; -import java.util.List; - /** * Handler for searching tickets. * diff --git a/search-handlers/src/test/java/no/unit/nva/search/ExportResourceHandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/ExportResourceHandlerTest.java index dff0d4f64..5fdcc5123 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/ExportResourceHandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/ExportResourceHandlerTest.java @@ -5,7 +5,6 @@ import static no.unit.nva.search.resource.ResourceParameter.SEARCH_ALL; import static no.unit.nva.testutils.RandomDataGenerator.randomString; import static no.unit.nva.testutils.RandomDataGenerator.randomUri; - import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; import static org.mockito.ArgumentMatchers.any; @@ -14,102 +13,96 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Map; import no.unit.nva.indexing.testutils.FakeSearchResponse; import no.unit.nva.search.common.csv.ExportCsv; import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.search.scroll.ScrollClient; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.apigateway.RequestInfo; import nva.commons.apigateway.exceptions.BadRequestException; - import org.apache.commons.lang3.StringUtils; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; - class ExportResourceHandlerTest { - private static final String SAMPLE_PATH = "search"; - private static final String SAMPLE_DOMAIN_NAME = "localhost"; - private ResourceClient mockedResourceClient; - private ScrollClient mockedScrollClient; - private ExportResourceHandler handler; - - private static SwsResponse csvToSwsResponse(ExportCsv csv, String scrollId) - throws JsonProcessingException { - var jsonResponse = FakeSearchResponse.generateSearchResponseString(List.of(csv), scrollId); - return objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - } - - private static ExportCsv csvWithFullDate(String title) { - var id = randomUri().toString(); - var type = "AcademicArticle"; - var contributors = List.of(randomString(), randomString(), randomString()); - var date = "2022-01-22"; - - return new ExportCsv() - .withId(id) - .withMainTitle(title) - .withPublicationInstance(type) - .withPublicationDate(date) - .withContributors(String.join(COMMA, contributors)); - } - - @BeforeEach - void setUp() { - mockedResourceClient = mock(ResourceClient.class); - mockedScrollClient = mock(ScrollClient.class); - handler = new ExportResourceHandler(mockedResourceClient, mockedScrollClient, null, null); - } - - @Test - void shouldReturnCsvWithTitleField() throws IOException, BadRequestException { - var expectedTitle1 = randomString(); - var expectedTitle2 = randomString(); - var expectedTitle3 = randomString(); - prepareRestHighLevelClientOkResponse( - csvWithFullDate(expectedTitle1), - csvWithFullDate(expectedTitle2), - csvWithFullDate(expectedTitle3)); - - var s3data = - handler.processS3Input( - null, RequestInfo.fromRequest(getRequestInputStreamAccepting()), null); - - assertThat(StringUtils.countMatches(s3data, expectedTitle1), is(1)); - assertThat(StringUtils.countMatches(s3data, expectedTitle2), is(1)); - assertThat(StringUtils.countMatches(s3data, expectedTitle3), is(1)); - } - - private void prepareRestHighLevelClientOkResponse( - ExportCsv initialSearchResult, - ExportCsv scroll1SearchResult, - ExportCsv scroll2SearchResult) - throws IOException { - - when(mockedResourceClient.doSearch(any())) - .thenReturn(csvToSwsResponse(initialSearchResult, "scrollId1")); - - when(mockedScrollClient.doSearch(any())) - .thenReturn(csvToSwsResponse(scroll1SearchResult, "scrollId2")) - .thenReturn(csvToSwsResponse(scroll2SearchResult, null)); - } - - private InputStream getRequestInputStreamAccepting() throws JsonProcessingException { - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withQueryParameters(Map.of(SEARCH_ALL.asCamelCase(), "*")) - .withRequestContext(getRequestContext()) - .build(); - } - - private ObjectNode getRequestContext() { - return objectMapperWithEmpty.convertValue( - Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); - } + private static final String SAMPLE_PATH = "search"; + private static final String SAMPLE_DOMAIN_NAME = "localhost"; + private ResourceClient mockedResourceClient; + private ScrollClient mockedScrollClient; + private ExportResourceHandler handler; + + private static SwsResponse csvToSwsResponse(ExportCsv csv, String scrollId) + throws JsonProcessingException { + var jsonResponse = FakeSearchResponse.generateSearchResponseString(List.of(csv), scrollId); + return objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + } + + private static ExportCsv csvWithFullDate(String title) { + var id = randomUri().toString(); + var type = "AcademicArticle"; + var contributors = List.of(randomString(), randomString(), randomString()); + var date = "2022-01-22"; + + return new ExportCsv() + .withId(id) + .withMainTitle(title) + .withPublicationInstance(type) + .withPublicationDate(date) + .withContributors(String.join(COMMA, contributors)); + } + + @BeforeEach + void setUp() { + mockedResourceClient = mock(ResourceClient.class); + mockedScrollClient = mock(ScrollClient.class); + handler = new ExportResourceHandler(mockedResourceClient, mockedScrollClient, null, null); + } + + @Test + void shouldReturnCsvWithTitleField() throws IOException, BadRequestException { + var expectedTitle1 = randomString(); + var expectedTitle2 = randomString(); + var expectedTitle3 = randomString(); + prepareRestHighLevelClientOkResponse( + csvWithFullDate(expectedTitle1), + csvWithFullDate(expectedTitle2), + csvWithFullDate(expectedTitle3)); + + var s3data = + handler.processS3Input( + null, RequestInfo.fromRequest(getRequestInputStreamAccepting()), null); + + assertThat(StringUtils.countMatches(s3data, expectedTitle1), is(1)); + assertThat(StringUtils.countMatches(s3data, expectedTitle2), is(1)); + assertThat(StringUtils.countMatches(s3data, expectedTitle3), is(1)); + } + + private void prepareRestHighLevelClientOkResponse( + ExportCsv initialSearchResult, ExportCsv scroll1SearchResult, ExportCsv scroll2SearchResult) + throws IOException { + + when(mockedResourceClient.doSearch(any())) + .thenReturn(csvToSwsResponse(initialSearchResult, "scrollId1")); + + when(mockedScrollClient.doSearch(any())) + .thenReturn(csvToSwsResponse(scroll1SearchResult, "scrollId2")) + .thenReturn(csvToSwsResponse(scroll2SearchResult, null)); + } + + private InputStream getRequestInputStreamAccepting() throws JsonProcessingException { + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withQueryParameters(Map.of(SEARCH_ALL.asCamelCase(), "*")) + .withRequestContext(getRequestContext()) + .build(); + } + + private ObjectNode getRequestContext() { + return objectMapperWithEmpty.convertValue( + Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); + } } diff --git a/search-handlers/src/test/java/no/unit/nva/search/SearchImportCandidateAuthHandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/SearchImportCandidateAuthHandlerTest.java index 284718f73..74138cf12 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/SearchImportCandidateAuthHandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/SearchImportCandidateAuthHandlerTest.java @@ -1,14 +1,14 @@ package no.unit.nva.search; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.util.Objects.nonNull; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.constants.Words.COMMA; import static no.unit.nva.search.importcandidate.ImportCandidateParameter.SEARCH_ALL; import static no.unit.nva.testutils.RandomDataGenerator.randomString; import static no.unit.nva.testutils.RandomDataGenerator.randomUri; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.collection.IsEmptyCollection.empty; @@ -20,13 +20,16 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.util.Objects.nonNull; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import no.unit.nva.constants.Words; import no.unit.nva.indexing.testutils.FakeSearchResponse; import no.unit.nva.search.common.FakeGatewayResponse; @@ -35,24 +38,14 @@ import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.importcandidate.ImportCandidateClient; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.apigateway.GatewayResponse; import nva.commons.core.Environment; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - class SearchImportCandidateAuthHandlerTest { public static final String SAMPLE_PATH = "search"; diff --git a/search-handlers/src/test/java/no/unit/nva/search/SearchResource20241201HandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/SearchResource20241201HandlerTest.java index 2229a70e4..99619cec9 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/SearchResource20241201HandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/SearchResource20241201HandlerTest.java @@ -1,10 +1,9 @@ package no.unit.nva.search; +import static java.net.HttpURLConnection.HTTP_OK; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.search.resource.ResourceParameter.SEARCH_ALL; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -13,30 +12,24 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_OK; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.util.Map; import no.unit.nva.search.common.FakeGatewayResponse; import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.search.resource.response.ResourceSearchResponse; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.core.Environment; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.file.Path; -import java.util.Map; - class SearchResource20241201HandlerTest { public static final String SAMPLE_PATH = "search"; public static final String SAMPLE_DOMAIN_NAME = "localhost"; diff --git a/search-handlers/src/test/java/no/unit/nva/search/SearchResourceAuthHandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/SearchResourceAuthHandlerTest.java index 762ec02c4..8bad18ed6 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/SearchResourceAuthHandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/SearchResourceAuthHandlerTest.java @@ -1,12 +1,12 @@ package no.unit.nva.search; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.testutils.RandomDataGenerator.randomString; import static no.unit.nva.testutils.RandomDataGenerator.randomUri; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; @@ -16,32 +16,25 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Map; import no.unit.nva.search.common.FakeGatewayResponse; import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.apigateway.AccessRight; import nva.commons.core.Environment; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.util.Map; - class SearchResourceAuthHandlerTest { public static final String SAMPLE_PATH = "search"; diff --git a/search-handlers/src/test/java/no/unit/nva/search/SearchResourceLegacyHandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/SearchResourceLegacyHandlerTest.java index fd9bf7167..5558bfd34 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/SearchResourceLegacyHandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/SearchResourceLegacyHandlerTest.java @@ -1,5 +1,7 @@ package no.unit.nva.search; +import static java.net.HttpURLConnection.HTTP_OK; +import static java.util.Objects.nonNull; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.constants.Words.COMMA; @@ -7,9 +9,7 @@ import static no.unit.nva.search.resource.ResourceParameter.SEARCH_ALL; import static no.unit.nva.testutils.RandomDataGenerator.randomString; import static no.unit.nva.testutils.RandomDataGenerator.randomUri; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.collection.IsEmptyCollection.empty; @@ -22,13 +22,16 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.util.Objects.nonNull; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; import no.unit.nva.constants.Words; import no.unit.nva.indexing.testutils.FakeSearchResponse; import no.unit.nva.search.common.FakeGatewayResponse; @@ -36,264 +39,250 @@ import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.resource.ResourceClient; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.apigateway.GatewayResponse; import nva.commons.core.Environment; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - class SearchResourceLegacyHandlerTest { - public static final String SAMPLE_PATH = "search"; - public static final String SAMPLE_DOMAIN_NAME = "localhost"; - public static final String SAMPLE_SEARCH_TERM = "searchTerm"; - public static final String SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON = - "sample_opensearch_response.json"; - public static final String EMPTY_OPENSEARCH_RESPONSE_JSON = "empty_opensearch_response.json"; + public static final String SAMPLE_PATH = "search"; + public static final String SAMPLE_DOMAIN_NAME = "localhost"; + public static final String SAMPLE_SEARCH_TERM = "searchTerm"; + public static final String SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON = + "sample_opensearch_response.json"; + public static final String EMPTY_OPENSEARCH_RESPONSE_JSON = "empty_opensearch_response.json"; private SearchResourceHandler handler; - private Context contextMock; - private ByteArrayOutputStream outputStream; - private ResourceClient mockedSearchClient; - - private static ExportCsv csvWithFullDate() { - var id = randomUri().toString(); - var title = randomString(); - var type = "AcademicArticle"; - var contributors = List.of(randomString(), randomString(), randomString()); - var date = "2022-01-22"; - - return new ExportCsv() - .withId(id) - .withMainTitle(title) - .withPublicationInstance(type) - .withPublicationDate(date) - .withContributors(String.join(COMMA, contributors)); - } - - public static Stream acceptHeaderValuesProducingTextCsvProvider() { - return Stream.of("text/*", Words.TEXT_CSV); - } - - public static Stream acceptHeaderValuesProducingApplicationJsonProvider() { - return Stream.of(null, "application/json", "application/json; charset=utf-8"); - } - - @BeforeEach - void setUp() { - - mockedSearchClient = mock(ResourceClient.class); + private Context contextMock; + private ByteArrayOutputStream outputStream; + private ResourceClient mockedSearchClient; + + private static ExportCsv csvWithFullDate() { + var id = randomUri().toString(); + var title = randomString(); + var type = "AcademicArticle"; + var contributors = List.of(randomString(), randomString(), randomString()); + var date = "2022-01-22"; + + return new ExportCsv() + .withId(id) + .withMainTitle(title) + .withPublicationInstance(type) + .withPublicationDate(date) + .withContributors(String.join(COMMA, contributors)); + } + + public static Stream acceptHeaderValuesProducingTextCsvProvider() { + return Stream.of("text/*", Words.TEXT_CSV); + } + + public static Stream acceptHeaderValuesProducingApplicationJsonProvider() { + return Stream.of(null, "application/json", "application/json; charset=utf-8"); + } + + @BeforeEach + void setUp() { + + mockedSearchClient = mock(ResourceClient.class); handler = new SearchResourceHandler(new Environment(), mockedSearchClient); - contextMock = mock(Context.class); - outputStream = new ByteArrayOutputStream(); - } - - @ParameterizedTest(name = "should return text/csv for accept header {0}") - @MethodSource("acceptHeaderValuesProducingTextCsvProvider") - void shouldReturnTextCsvWithGivenAcceptHeader(String acceptHeaderValue) throws IOException { - prepareRestHighLevelClientOkResponse(List.of(csvWithFullDate(), csvWithYearOnly())); - handler.handleRequest( - getRequestInputStreamAccepting(acceptHeaderValue), - outputStream, - mock(Context.class)); - - GatewayResponse gatewayResponse = - GatewayResponse.fromOutputStream(outputStream, String.class); - assertThat( - gatewayResponse.getHeaders().get("Content-Type"), - is(equalTo("text/csv; charset=utf-8"))); - } - - private ExportCsv csvWithYearOnly() { - var id = randomUri().toString(); - var title = randomString(); - var type = "AcademicArticle"; - var contributors = List.of(randomString(), randomString(), randomString()); - var date = "2022"; - - return new ExportCsv() - .withId(id) - .withMainTitle(title) - .withPublicationInstance(type) - .withPublicationDate(date) - .withContributors(String.join(COMMA, contributors)); - } - - @Test - void shouldReturnSortedSearchResultsWhenSendingContributorId() throws IOException { - prepareRestHighLevelClientOkResponse(); - handler.handleRequest(getInputStreamWithContributorId(), outputStream, contextMock); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - } - - @Test - void shouldNotReturnSortedSearchResultsWhenSendingMultipleContributorId() throws IOException { - prepareRestHighLevelClientOkResponse(); - - handler.handleRequest(getInputStreamWithMultipleContributorId(), outputStream, contextMock); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - } - - @Test - void shouldReturnResultWithEqualContributorAndContributorPreview() throws IOException { - prepareRestHighLevelClientOkResponse(); - - handler.handleRequest(getInputStreamWithMultipleContributorId(), outputStream, contextMock); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - var actualBody = gatewayResponse.body(); - var firstHit = actualBody.hits().getFirst(); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - assertFalse(firstHit.path("entityDescription").path("contributors").isMissingNode()); - assertNotNull(firstHit.path("entityDescription").path("contributors").get(0)); - assertEquals( - firstHit.path("entityDescription").path("contributors"), - firstHit.path("entityDescription").path("contributorsPreview")); - } - - @Test - void shouldReturnSearchResultsWithEmptyHitsWhenQueryResultIsEmpty() throws IOException { - prepareRestHighLevelClientEmptyResponse(); - - handler.handleRequest(getInputStream(), outputStream, mock(Context.class)); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - var actualBody = gatewayResponse.body(); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - assertThat(actualBody.totalHits(), is(equalTo(0))); - assertThat(actualBody.hits(), is(empty())); - assertDoesNotThrow(() -> gatewayResponse.body().id().normalize()); - } - - @Test - void shouldReturn200WhenSortOrderIsNotSpecified() throws IOException { - prepareRestHighLevelClientEmptyResponseForSortOrder(); - - handler.handleRequest(getInputStream(), outputStream, mock(Context.class)); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - var actualBody = gatewayResponse.body(); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - assertThat(actualBody.totalHits(), is(equalTo(0))); - assertThat(actualBody.hits(), is(empty())); - assertDoesNotThrow(() -> gatewayResponse.body().id().normalize()); - } - - @ParameterizedTest(name = "Should return application/json for accept header {0}") - @MethodSource("acceptHeaderValuesProducingApplicationJsonProvider") - void shouldProduceWithHeader(String acceptHeaderValue) throws IOException { - prepareRestHighLevelClientOkResponse(); - var requestInput = - nonNull(acceptHeaderValue) - ? getRequestInputStreamAccepting(acceptHeaderValue) - : getInputStream(); - handler.handleRequest(requestInput, outputStream, mock(Context.class)); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - assertThat( - gatewayResponse.headers().get("Content-Type"), - is(equalTo("application/json; charset=utf-8"))); - } - - private InputStream getInputStream() throws JsonProcessingException { - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withQueryParameters(Map.of(SEARCH_ALL.name(), SAMPLE_SEARCH_TERM)) - .withRequestContext(getRequestContext()) - .build(); - } - - private InputStream getInputStreamWithContributorId() throws JsonProcessingException { - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withQueryParameters( - Map.of( - SEARCH_ALL.name(), - "entityDescription.contributors.identity.id:12345", - "results", - "10", - "from", - "0")) - .withHeaders(Map.of(ACCEPT, "application/json")) - .withRequestContext(getRequestContext()) - .withUserName(randomString()) - .build(); - } - - private InputStream getInputStreamWithMultipleContributorId() throws JsonProcessingException { - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withMultiValueQueryParameters(Map.of(CONTRIBUTOR, List.of("12345", "54321"))) - .withRequestContext(getRequestContext()) - .withHeaders(Map.of(ACCEPT, "application/json")) - .withUserName(randomString()) - .build(); - } - - private InputStream getRequestInputStreamAccepting(String contentType) - throws JsonProcessingException { - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withQueryParameters(Map.of(SEARCH_ALL.asCamelCase(), SAMPLE_SEARCH_TERM)) - .withHeaders(Map.of(ACCEPT, contentType)) - .withRequestContext(getRequestContext()) - .build(); - } - - private ObjectNode getRequestContext() { - return objectMapperWithEmpty.convertValue( - Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); - } - - private void prepareRestHighLevelClientOkResponse(List exportCsvs) - throws IOException { - var jsonResponse = FakeSearchResponse.generateSearchResponseString(exportCsvs, null); - var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - - when(mockedSearchClient.doSearch(any())).thenReturn(body); - // var searchResponse = createSearchResponseWithHits(json); - // when(restHighLevelClientMock.search(any(), any())).thenReturn(searchResponse); - } - - private void prepareRestHighLevelClientOkResponse() throws IOException { - var jsonResponse = - stringFromResources(Path.of(SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON)); - var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - - when(mockedSearchClient.doSearch(any())).thenReturn(body); - } - - private void prepareRestHighLevelClientEmptyResponse() throws IOException { - var jsonResponse = stringFromResources(Path.of(EMPTY_OPENSEARCH_RESPONSE_JSON)); - var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - - when(mockedSearchClient.doSearch(any())).thenReturn(body); - } - - private void prepareRestHighLevelClientEmptyResponseForSortOrder() throws IOException { - var jsonResponse = stringFromResources(Path.of(EMPTY_OPENSEARCH_RESPONSE_JSON)); - var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - - when(mockedSearchClient.doSearch(any())).thenReturn(body); - } + contextMock = mock(Context.class); + outputStream = new ByteArrayOutputStream(); + } + + @ParameterizedTest(name = "should return text/csv for accept header {0}") + @MethodSource("acceptHeaderValuesProducingTextCsvProvider") + void shouldReturnTextCsvWithGivenAcceptHeader(String acceptHeaderValue) throws IOException { + prepareRestHighLevelClientOkResponse(List.of(csvWithFullDate(), csvWithYearOnly())); + handler.handleRequest( + getRequestInputStreamAccepting(acceptHeaderValue), outputStream, mock(Context.class)); + + GatewayResponse gatewayResponse = + GatewayResponse.fromOutputStream(outputStream, String.class); + assertThat( + gatewayResponse.getHeaders().get("Content-Type"), is(equalTo("text/csv; charset=utf-8"))); + } + + private ExportCsv csvWithYearOnly() { + var id = randomUri().toString(); + var title = randomString(); + var type = "AcademicArticle"; + var contributors = List.of(randomString(), randomString(), randomString()); + var date = "2022"; + + return new ExportCsv() + .withId(id) + .withMainTitle(title) + .withPublicationInstance(type) + .withPublicationDate(date) + .withContributors(String.join(COMMA, contributors)); + } + + @Test + void shouldReturnSortedSearchResultsWhenSendingContributorId() throws IOException { + prepareRestHighLevelClientOkResponse(); + handler.handleRequest(getInputStreamWithContributorId(), outputStream, contextMock); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + } + + @Test + void shouldNotReturnSortedSearchResultsWhenSendingMultipleContributorId() throws IOException { + prepareRestHighLevelClientOkResponse(); + + handler.handleRequest(getInputStreamWithMultipleContributorId(), outputStream, contextMock); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + } + + @Test + void shouldReturnResultWithEqualContributorAndContributorPreview() throws IOException { + prepareRestHighLevelClientOkResponse(); + + handler.handleRequest(getInputStreamWithMultipleContributorId(), outputStream, contextMock); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + var actualBody = gatewayResponse.body(); + var firstHit = actualBody.hits().getFirst(); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + assertFalse(firstHit.path("entityDescription").path("contributors").isMissingNode()); + assertNotNull(firstHit.path("entityDescription").path("contributors").get(0)); + assertEquals( + firstHit.path("entityDescription").path("contributors"), + firstHit.path("entityDescription").path("contributorsPreview")); + } + + @Test + void shouldReturnSearchResultsWithEmptyHitsWhenQueryResultIsEmpty() throws IOException { + prepareRestHighLevelClientEmptyResponse(); + + handler.handleRequest(getInputStream(), outputStream, mock(Context.class)); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + var actualBody = gatewayResponse.body(); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + assertThat(actualBody.totalHits(), is(equalTo(0))); + assertThat(actualBody.hits(), is(empty())); + assertDoesNotThrow(() -> gatewayResponse.body().id().normalize()); + } + + @Test + void shouldReturn200WhenSortOrderIsNotSpecified() throws IOException { + prepareRestHighLevelClientEmptyResponseForSortOrder(); + + handler.handleRequest(getInputStream(), outputStream, mock(Context.class)); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + var actualBody = gatewayResponse.body(); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + assertThat(actualBody.totalHits(), is(equalTo(0))); + assertThat(actualBody.hits(), is(empty())); + assertDoesNotThrow(() -> gatewayResponse.body().id().normalize()); + } + + @ParameterizedTest(name = "Should return application/json for accept header {0}") + @MethodSource("acceptHeaderValuesProducingApplicationJsonProvider") + void shouldProduceWithHeader(String acceptHeaderValue) throws IOException { + prepareRestHighLevelClientOkResponse(); + var requestInput = + nonNull(acceptHeaderValue) + ? getRequestInputStreamAccepting(acceptHeaderValue) + : getInputStream(); + handler.handleRequest(requestInput, outputStream, mock(Context.class)); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + assertThat( + gatewayResponse.headers().get("Content-Type"), + is(equalTo("application/json; charset=utf-8"))); + } + + private InputStream getInputStream() throws JsonProcessingException { + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withQueryParameters(Map.of(SEARCH_ALL.name(), SAMPLE_SEARCH_TERM)) + .withRequestContext(getRequestContext()) + .build(); + } + + private InputStream getInputStreamWithContributorId() throws JsonProcessingException { + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withQueryParameters( + Map.of( + SEARCH_ALL.name(), + "entityDescription.contributors.identity.id:12345", + "results", + "10", + "from", + "0")) + .withHeaders(Map.of(ACCEPT, "application/json")) + .withRequestContext(getRequestContext()) + .withUserName(randomString()) + .build(); + } + + private InputStream getInputStreamWithMultipleContributorId() throws JsonProcessingException { + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withMultiValueQueryParameters(Map.of(CONTRIBUTOR, List.of("12345", "54321"))) + .withRequestContext(getRequestContext()) + .withHeaders(Map.of(ACCEPT, "application/json")) + .withUserName(randomString()) + .build(); + } + + private InputStream getRequestInputStreamAccepting(String contentType) + throws JsonProcessingException { + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withQueryParameters(Map.of(SEARCH_ALL.asCamelCase(), SAMPLE_SEARCH_TERM)) + .withHeaders(Map.of(ACCEPT, contentType)) + .withRequestContext(getRequestContext()) + .build(); + } + + private ObjectNode getRequestContext() { + return objectMapperWithEmpty.convertValue( + Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); + } + + private void prepareRestHighLevelClientOkResponse(List exportCsvs) throws IOException { + var jsonResponse = FakeSearchResponse.generateSearchResponseString(exportCsvs, null); + var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + + when(mockedSearchClient.doSearch(any())).thenReturn(body); + // var searchResponse = createSearchResponseWithHits(json); + // when(restHighLevelClientMock.search(any(), any())).thenReturn(searchResponse); + } + + private void prepareRestHighLevelClientOkResponse() throws IOException { + var jsonResponse = + stringFromResources(Path.of(SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON)); + var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + + when(mockedSearchClient.doSearch(any())).thenReturn(body); + } + + private void prepareRestHighLevelClientEmptyResponse() throws IOException { + var jsonResponse = stringFromResources(Path.of(EMPTY_OPENSEARCH_RESPONSE_JSON)); + var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + + when(mockedSearchClient.doSearch(any())).thenReturn(body); + } + + private void prepareRestHighLevelClientEmptyResponseForSortOrder() throws IOException { + var jsonResponse = stringFromResources(Path.of(EMPTY_OPENSEARCH_RESPONSE_JSON)); + var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + + when(mockedSearchClient.doSearch(any())).thenReturn(body); + } } diff --git a/search-handlers/src/test/java/no/unit/nva/search/SearchTicketAuthHandlerTest.java b/search-handlers/src/test/java/no/unit/nva/search/SearchTicketAuthHandlerTest.java index 95a23f1d2..1d08ed62c 100644 --- a/search-handlers/src/test/java/no/unit/nva/search/SearchTicketAuthHandlerTest.java +++ b/search-handlers/src/test/java/no/unit/nva/search/SearchTicketAuthHandlerTest.java @@ -1,12 +1,11 @@ package no.unit.nva.search; +import static java.net.HttpURLConnection.HTTP_OK; import static no.unit.nva.auth.uriretriever.UriRetriever.ACCEPT; import static no.unit.nva.constants.Defaults.objectMapperWithEmpty; import static no.unit.nva.testutils.RandomDataGenerator.randomString; import static no.unit.nva.testutils.RandomDataGenerator.randomUri; - import static nva.commons.core.ioutils.IoUtils.stringFromResources; - import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.Is.is; @@ -16,111 +15,103 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.net.HttpURLConnection.HTTP_OK; - import com.amazonaws.services.lambda.runtime.Context; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.node.ObjectNode; - +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Map; import no.unit.nva.search.common.FakeGatewayResponse; import no.unit.nva.search.common.records.SwsResponse; import no.unit.nva.search.ticket.TicketClient; import no.unit.nva.testutils.HandlerRequestBuilder; - import nva.commons.apigateway.AccessRight; import nva.commons.core.Environment; - import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.util.Map; - class SearchTicketAuthHandlerTest { - public static final String SAMPLE_PATH = "search"; - public static final String SAMPLE_DOMAIN_NAME = "localhost"; - public static final String SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON = - "sample_opensearch_response.json"; - private SearchTicketAuthHandler handler; - private Context contextMock; - private ByteArrayOutputStream outputStream; - private TicketClient mockedSearchClient; - - @BeforeEach - void setUp() { - - mockedSearchClient = mock(TicketClient.class); - handler = new SearchTicketAuthHandler(new Environment(), mockedSearchClient); - contextMock = mock(Context.class); - outputStream = new ByteArrayOutputStream(); - } - - @Test - void shouldOnlyReturnPublicationsFromCuratorsOrganizationWhenQuerying() - throws IOException, URISyntaxException { - prepareRestHighLevelClientOkResponse(); - - var organization = - new URI("https://api.dev.nva.aws.unit.no/cristin/organization/20754.0.0.0"); - var input = getInputStreamWithAccessRight(organization, AccessRight.MANAGE_DOI); - - handler.handleRequest(input, outputStream, contextMock); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - var actualBody = gatewayResponse.body(); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - assertThat(actualBody.hits().size(), is(equalTo(2))); - } - - @Test - void shouldReturnOkWhenUserIsEditor() throws IOException { - prepareRestHighLevelClientOkResponse(); - - var input = getInputStreamWithAccessRight(randomUri(), AccessRight.SUPPORT); - handler.handleRequest(input, outputStream, contextMock); - - var gatewayResponse = FakeGatewayResponse.of(outputStream); - - assertNotNull(gatewayResponse.headers()); - assertEquals(HTTP_OK, gatewayResponse.statusCode()); - } - - private void prepareRestHighLevelClientOkResponse() throws IOException { - var jsonResponse = - stringFromResources(Path.of(SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON)); - var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); - - when(mockedSearchClient.doSearch(any())).thenReturn(body); - } - - private InputStream getInputStreamWithAccessRight(URI organization, AccessRight accessRight) - throws JsonProcessingException { - var personAffiliationId = - "https://api.dev.nva.aws.unit.no/cristin/organization/20754.1.0.0"; - var personAffiliation = "custom:personAffiliation"; - var topLevelOrgCristinId = - URI.create("https://api.dev.nva.aws.unit.no/cristin/organization/20754.0.0.0"); - return new HandlerRequestBuilder(objectMapperWithEmpty) - .withHeaders(Map.of(ACCEPT, "application/json")) - .withRequestContext(getRequestContext()) - .withTopLevelCristinOrgId(topLevelOrgCristinId) - .withAuthorizerClaim(personAffiliation, personAffiliationId) - .withUserName(randomString()) - .withCurrentCustomer(organization) - .withAccessRights(organization, accessRight) - .build(); - } - - private ObjectNode getRequestContext() { - return objectMapperWithEmpty.convertValue( - Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); - } + public static final String SAMPLE_PATH = "search"; + public static final String SAMPLE_DOMAIN_NAME = "localhost"; + public static final String SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON = + "sample_opensearch_response.json"; + private SearchTicketAuthHandler handler; + private Context contextMock; + private ByteArrayOutputStream outputStream; + private TicketClient mockedSearchClient; + + @BeforeEach + void setUp() { + + mockedSearchClient = mock(TicketClient.class); + handler = new SearchTicketAuthHandler(new Environment(), mockedSearchClient); + contextMock = mock(Context.class); + outputStream = new ByteArrayOutputStream(); + } + + @Test + void shouldOnlyReturnPublicationsFromCuratorsOrganizationWhenQuerying() + throws IOException, URISyntaxException { + prepareRestHighLevelClientOkResponse(); + + var organization = new URI("https://api.dev.nva.aws.unit.no/cristin/organization/20754.0.0.0"); + var input = getInputStreamWithAccessRight(organization, AccessRight.MANAGE_DOI); + + handler.handleRequest(input, outputStream, contextMock); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + var actualBody = gatewayResponse.body(); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + assertThat(actualBody.hits().size(), is(equalTo(2))); + } + + @Test + void shouldReturnOkWhenUserIsEditor() throws IOException { + prepareRestHighLevelClientOkResponse(); + + var input = getInputStreamWithAccessRight(randomUri(), AccessRight.SUPPORT); + handler.handleRequest(input, outputStream, contextMock); + + var gatewayResponse = FakeGatewayResponse.of(outputStream); + + assertNotNull(gatewayResponse.headers()); + assertEquals(HTTP_OK, gatewayResponse.statusCode()); + } + + private void prepareRestHighLevelClientOkResponse() throws IOException { + var jsonResponse = + stringFromResources(Path.of(SAMPLE_OPENSEARCH_RESPONSE_WITH_AGGREGATION_JSON)); + var body = objectMapperWithEmpty.readValue(jsonResponse, SwsResponse.class); + + when(mockedSearchClient.doSearch(any())).thenReturn(body); + } + + private InputStream getInputStreamWithAccessRight(URI organization, AccessRight accessRight) + throws JsonProcessingException { + var personAffiliationId = "https://api.dev.nva.aws.unit.no/cristin/organization/20754.1.0.0"; + var personAffiliation = "custom:personAffiliation"; + var topLevelOrgCristinId = + URI.create("https://api.dev.nva.aws.unit.no/cristin/organization/20754.0.0.0"); + return new HandlerRequestBuilder(objectMapperWithEmpty) + .withHeaders(Map.of(ACCEPT, "application/json")) + .withRequestContext(getRequestContext()) + .withTopLevelCristinOrgId(topLevelOrgCristinId) + .withAuthorizerClaim(personAffiliation, personAffiliationId) + .withUserName(randomString()) + .withCurrentCustomer(organization) + .withAccessRights(organization, accessRight) + .build(); + } + + private ObjectNode getRequestContext() { + return objectMapperWithEmpty.convertValue( + Map.of("path", SAMPLE_PATH, "domainName", SAMPLE_DOMAIN_NAME), ObjectNode.class); + } } diff --git a/search-testing/src/main/java/no/unit/nva/LogAppender.java b/search-testing/src/main/java/no/unit/nva/LogAppender.java index 81c4833f3..50a525663 100644 --- a/search-testing/src/main/java/no/unit/nva/LogAppender.java +++ b/search-testing/src/main/java/no/unit/nva/LogAppender.java @@ -1,19 +1,16 @@ package no.unit.nva; -import static no.unit.nva.constants.Words.SPACE; - import static java.util.Objects.nonNull; +import static no.unit.nva.constants.Words.SPACE; +import java.util.List; import nva.commons.core.JacocoGenerated; - import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.test.appender.ListAppender; import org.slf4j.LoggerFactory; -import java.util.List; - @SuppressWarnings({"PMD.CloseResource"}) public final class LogAppender { diff --git a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeIndexingClient.java b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeIndexingClient.java index 458698b84..eebfbbc9f 100644 --- a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeIndexingClient.java +++ b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeIndexingClient.java @@ -3,15 +3,6 @@ import static nva.commons.core.attempt.Try.attempt; import com.fasterxml.jackson.databind.JsonNode; - -import no.unit.nva.indexingclient.IndexingClient; -import no.unit.nva.indexingclient.models.IndexDocument; - -import org.opensearch.action.DocWriteRequest.OpType; -import org.opensearch.action.DocWriteResponse; -import org.opensearch.action.bulk.BulkItemResponse; -import org.opensearch.action.bulk.BulkResponse; - import java.io.IOException; import java.util.Collection; import java.util.HashMap; @@ -22,6 +13,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; +import no.unit.nva.indexingclient.IndexingClient; +import no.unit.nva.indexingclient.models.IndexDocument; +import org.opensearch.action.DocWriteRequest.OpType; +import org.opensearch.action.DocWriteResponse; +import org.opensearch.action.bulk.BulkItemResponse; +import org.opensearch.action.bulk.BulkResponse; /** * Faking the Indexing Client instead of the OpenSearch client because faking the OpenSearch client diff --git a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSearchResponse.java b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSearchResponse.java index 6212fb40b..658fdab82 100644 --- a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSearchResponse.java +++ b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSearchResponse.java @@ -1,15 +1,13 @@ package no.unit.nva.indexing.testutils; -import no.unit.nva.search.common.csv.ExportCsv; - -import nva.commons.core.JacocoGenerated; -import nva.commons.core.ioutils.IoUtils; - import java.nio.file.Path; import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; +import no.unit.nva.search.common.csv.ExportCsv; +import nva.commons.core.JacocoGenerated; +import nva.commons.core.ioutils.IoUtils; public final class FakeSearchResponse { diff --git a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSqsClient.java b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSqsClient.java index 8ccfdab77..213aef10f 100644 --- a/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSqsClient.java +++ b/search-testing/src/main/java/no/unit/nva/indexing/testutils/FakeSqsClient.java @@ -1,11 +1,9 @@ package no.unit.nva.indexing.testutils; -import no.unit.nva.indexingclient.models.QueueClient; - -import software.amazon.awssdk.services.sqs.model.SendMessageRequest; - import java.util.ArrayList; import java.util.List; +import no.unit.nva.indexingclient.models.QueueClient; +import software.amazon.awssdk.services.sqs.model.SendMessageRequest; public class FakeSqsClient implements QueueClient { diff --git a/search-testing/src/main/java/no/unit/nva/indexing/testutils/MockedJwtProvider.java b/search-testing/src/main/java/no/unit/nva/indexing/testutils/MockedJwtProvider.java index 2254440df..0c30b713c 100644 --- a/search-testing/src/main/java/no/unit/nva/indexing/testutils/MockedJwtProvider.java +++ b/search-testing/src/main/java/no/unit/nva/indexing/testutils/MockedJwtProvider.java @@ -1,34 +1,30 @@ package no.unit.nva.indexing.testutils; import static no.unit.nva.indexing.testutils.Constants.TEST_TOKEN; - import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.auth0.jwt.interfaces.DecodedJWT; - -import no.unit.nva.search.common.jwt.CachedJwtProvider; -import no.unit.nva.search.common.jwt.CognitoAuthenticator; - -import nva.commons.core.JacocoGenerated; - import java.time.Duration; import java.time.Instant; import java.util.Date; +import no.unit.nva.search.common.jwt.CachedJwtProvider; +import no.unit.nva.search.common.jwt.CognitoAuthenticator; +import nva.commons.core.JacocoGenerated; public final class MockedJwtProvider { - @JacocoGenerated - private MockedJwtProvider() {} + @JacocoGenerated + private MockedJwtProvider() {} - public static CachedJwtProvider setupMockedCachedJwtProvider() { - var jwt = mock(DecodedJWT.class); - var cogintoAuthenticatorMock = mock(CognitoAuthenticator.class); + public static CachedJwtProvider setupMockedCachedJwtProvider() { + var jwt = mock(DecodedJWT.class); + var cogintoAuthenticatorMock = mock(CognitoAuthenticator.class); - when(jwt.getToken()).thenReturn(TEST_TOKEN); - when(jwt.getExpiresAt()).thenReturn(Date.from(Instant.now().plus(Duration.ofMinutes(5)))); - when(cogintoAuthenticatorMock.fetchBearerToken()).thenReturn(jwt); + when(jwt.getToken()).thenReturn(TEST_TOKEN); + when(jwt.getExpiresAt()).thenReturn(Date.from(Instant.now().plus(Duration.ofMinutes(5)))); + when(cogintoAuthenticatorMock.fetchBearerToken()).thenReturn(jwt); - return CachedJwtProvider.prepareWithAuthenticator(cogintoAuthenticatorMock); - } + return CachedJwtProvider.prepareWithAuthenticator(cogintoAuthenticatorMock); + } } diff --git a/search-testing/src/main/java/no/unit/nva/search/common/FakeGatewayResponse.java b/search-testing/src/main/java/no/unit/nva/search/common/FakeGatewayResponse.java index 9a81ff4c6..2295c46f4 100644 --- a/search-testing/src/main/java/no/unit/nva/search/common/FakeGatewayResponse.java +++ b/search-testing/src/main/java/no/unit/nva/search/common/FakeGatewayResponse.java @@ -4,12 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; - -import no.unit.nva.search.common.records.PagedSearch; - import java.io.IOException; import java.io.OutputStream; import java.util.Map; +import no.unit.nva.search.common.records.PagedSearch; @SuppressWarnings("PMD.ShortMethodName") public record FakeGatewayResponse(T body, int statusCode, Map headers) {