diff --git a/.mvn/maven.config b/.mvn/maven.config index f4b88dce..de4cdebd 100644 --- a/.mvn/maven.config +++ b/.mvn/maven.config @@ -1,3 +1,3 @@ --Drevision=2.0.0 +-Drevision=3.0.0 -Dsha1= -Dchangelist=-SNAPSHOT diff --git a/docs/features.md b/docs/features.md index 1845923a..89726f69 100644 --- a/docs/features.md +++ b/docs/features.md @@ -13,9 +13,9 @@ As an example, you can see in the `repositories` section of the following docume "access": "controlled", "study": "ABCD-CA", "analysis": { - "id": "EGAZ", - "type": "sequencingRead", - "state": "PUBLISHED", + "analysis_id": "EGAZ", + "analysis_type": "sequencingRead", + "analysis_state": "PUBLISHED", "study": "ABCD-CA", "experiment": { "analysisId": "EGAZ", diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java b/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java index 28296661..9dd23f43 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/Maestro.java @@ -31,7 +31,7 @@ @SpringBootApplication @Import({RootConfiguration.class}) public class Maestro { - public static void main(String[] args) { - SpringApplication.run(Maestro.class, args); - } + public static void main(String[] args) { + SpringApplication.run(Maestro.class, args); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java index 421f9918..a2f28d06 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexAnalysisMessage.java @@ -23,13 +23,10 @@ @ToString @AllArgsConstructor class IndexAnalysisMessage { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String repositoryCode; - /** if callers set this flag it will do a remove instead of add.*/ - private Boolean removeAnalysis = false; + /** if callers set this flag it will do a remove instead of add. */ + private Boolean removeAnalysis = false; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java index f14067cc..df2d0a6e 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessage.java @@ -26,10 +26,9 @@ @ToString @AllArgsConstructor class IndexMessage { - private String analysisId; - private String studyId; - @NonNull - private String repositoryCode; - /** if callers set this flag it will do a remove instead of add.*/ - private Boolean removeAnalysis = false; + private String analysisId; + private String studyId; + @NonNull private String repositoryCode; + /** if callers set this flag it will do a remove instead of add. */ + private Boolean removeAnalysis = false; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java index 604fb4a9..a35d84d5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexMessagesHelper.java @@ -19,45 +19,46 @@ import bio.overture.maestro.domain.api.message.IndexResult; import io.vavr.Tuple2; +import java.util.function.Supplier; import lombok.extern.slf4j.Slf4j; import lombok.val; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.function.Supplier; - @Slf4j public class IndexMessagesHelper { - public static void handleIndexRepository(Supplier>> resultSupplier) { - val result = resultSupplier.get().blockOptional(); - val tuple = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); - if (!tuple._2().isSuccessful()) { - log.error("failed to process message : {} successfully", tuple._1()); - throw new RuntimeException("failed to process the message"); - } + public static void handleIndexRepository( + Supplier>> resultSupplier) { + val result = resultSupplier.get().blockOptional(); + val tuple = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); + if (!tuple._2().isSuccessful()) { + log.error("failed to process message : {} successfully", tuple._1()); + throw new RuntimeException("failed to process the message"); } + } - public static void handleIndexResult(Supplier>> resultSupplier) { - /* - * Why Blocking? - * - * - this is a stream consumer, it's supposed to process one message at a time - * the value of reactive processing diminishes since the queue provides a buffering level, - * without blocking it will async process the messages and if one fails we can - * async add it to a DLQ in the subscriber, However, I opted to use blocking because of the next point. - * - * - spring reactive cloud stream is deprecated in favor of spring cloud functions that support - * stream processing: https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/2.2.0.RELEASE/spring-cloud-stream.html#spring_cloud_function - * so I don't want to use a deprecated library, and if needed we can switch to cloud function in future - * https://stackoverflow.com/questions/53438208/spring-cloud-stream-reactive-how-to-do-the-error-handling-in-case-of-reactive - */ - val result = resultSupplier.get().collectList().blockOptional(); - val tupleList = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); - tupleList.forEach(tuple -> { - if (!tuple._2().isSuccessful()) { - log.error("failed to process message : {} successfully", tuple._1()); - throw new RuntimeException("failed to process the message"); - } + public static void handleIndexResult(Supplier>> resultSupplier) { + /* + * Why Blocking? + * + * - this is a stream consumer, it's supposed to process one message at a time + * the value of reactive processing diminishes since the queue provides a buffering level, + * without blocking it will async process the messages and if one fails we can + * async add it to a DLQ in the subscriber, However, I opted to use blocking because of the next point. + * + * - spring reactive cloud stream is deprecated in favor of spring cloud functions that support + * stream processing: https://cloud.spring.io/spring-cloud-static/spring-cloud-stream/2.2.0.RELEASE/spring-cloud-stream.html#spring_cloud_function + * so I don't want to use a deprecated library, and if needed we can switch to cloud function in future + * https://stackoverflow.com/questions/53438208/spring-cloud-stream-reactive-how-to-do-the-error-handling-in-case-of-reactive + */ + val result = resultSupplier.get().collectList().blockOptional(); + val tupleList = result.orElseThrow(() -> new RuntimeException("failed to obtain result")); + tupleList.forEach( + tuple -> { + if (!tuple._2().isSuccessful()) { + log.error("failed to process message : {} successfully", tuple._1()); + throw new RuntimeException("failed to process the message"); + } }); - } + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java index a626f3d5..dae56011 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexRepositoryMessage.java @@ -23,6 +23,5 @@ @ToString @AllArgsConstructor class IndexRepositoryMessage { - @NonNull - private String repositoryCode; + @NonNull private String repositoryCode; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java index ccc44cb3..1548701f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexStudyMessage.java @@ -23,8 +23,6 @@ @ToString @AllArgsConstructor class IndexStudyMessage { - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java index dbeae9ea..bd4d3fe1 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/IndexingMessagesStreamListener.java @@ -17,6 +17,9 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexRepository; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; + import bio.overture.maestro.domain.api.Indexer; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.message.*; @@ -32,111 +35,118 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexRepository; -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; - @Slf4j @EnableBinding(Sink.class) public class IndexingMessagesStreamListener { - private final Indexer indexer; - - public IndexingMessagesStreamListener(@NonNull Indexer indexer) { - this.indexer = indexer; - } - - @StreamListener(Sink.INPUT) - public void handleAnalysisMessage(@Payload IndexMessage indexMessage) { - if (isAnalysisReq(indexMessage)) { - val indexAnalysisMessage = new IndexAnalysisMessage(indexMessage.getAnalysisId(), - indexMessage.getStudyId(), - indexMessage.getRepositoryCode(), - indexMessage.getRemoveAnalysis()); - handleIndexResult(() -> this.indexOrRemoveAnalysis(indexAnalysisMessage)); - } else if (isStudyMsg(indexMessage)) { - val indexStudyMessage = new IndexStudyMessage(indexMessage.getStudyId(), indexMessage.getRepositoryCode()); - handleIndexResult(() -> this.indexStudy(indexStudyMessage)); - } else if (isRepoMsg(indexMessage)) { - val indexRepositoryMessage = new IndexRepositoryMessage(indexMessage.getRepositoryCode()); - handleIndexRepository(() -> this.indexRepository(indexRepositoryMessage)); - } else { - throw new IllegalArgumentException("invalid message format"); - } - } - - private boolean isAnalysisReq(IndexMessage indexMessage) { - return !StringUtils.isEmpty(indexMessage.getAnalysisId()) - && !StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); - } - - private boolean isStudyMsg(IndexMessage indexMessage) { - return StringUtils.isEmpty(indexMessage.getAnalysisId()) - && !StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); - } - - private boolean isRepoMsg(IndexMessage indexMessage) { - return StringUtils.isEmpty(indexMessage.getAnalysisId()) - && StringUtils.isEmpty(indexMessage.getStudyId()) - && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + private final Indexer indexer; + + public IndexingMessagesStreamListener(@NonNull Indexer indexer) { + this.indexer = indexer; + } + + @StreamListener(Sink.INPUT) + public void handleAnalysisMessage(@Payload IndexMessage indexMessage) { + if (isAnalysisReq(indexMessage)) { + val indexAnalysisMessage = + new IndexAnalysisMessage( + indexMessage.getAnalysisId(), + indexMessage.getStudyId(), + indexMessage.getRepositoryCode(), + indexMessage.getRemoveAnalysis()); + handleIndexResult(() -> this.indexOrRemoveAnalysis(indexAnalysisMessage)); + } else if (isStudyMsg(indexMessage)) { + val indexStudyMessage = + new IndexStudyMessage(indexMessage.getStudyId(), indexMessage.getRepositoryCode()); + handleIndexResult(() -> this.indexStudy(indexStudyMessage)); + } else if (isRepoMsg(indexMessage)) { + val indexRepositoryMessage = new IndexRepositoryMessage(indexMessage.getRepositoryCode()); + handleIndexRepository(() -> this.indexRepository(indexRepositoryMessage)); + } else { + throw new IllegalArgumentException("invalid message format"); } - - private Flux> indexOrRemoveAnalysis(IndexAnalysisMessage msg) { - if (msg.getRemoveAnalysis()) { - return Flux.from(removeAnalysis(msg)); - } else { - return indexAnalysis(msg); - } + } + + private boolean isAnalysisReq(IndexMessage indexMessage) { + return !StringUtils.isEmpty(indexMessage.getAnalysisId()) + && !StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private boolean isStudyMsg(IndexMessage indexMessage) { + return StringUtils.isEmpty(indexMessage.getAnalysisId()) + && !StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private boolean isRepoMsg(IndexMessage indexMessage) { + return StringUtils.isEmpty(indexMessage.getAnalysisId()) + && StringUtils.isEmpty(indexMessage.getStudyId()) + && !StringUtils.isEmpty(indexMessage.getRepositoryCode()); + } + + private Flux> indexOrRemoveAnalysis( + IndexAnalysisMessage msg) { + if (msg.getRemoveAnalysis()) { + return Flux.from(removeAnalysis(msg)); + } else { + return indexAnalysis(msg); } - - private Mono> removeAnalysis(IndexAnalysisMessage msg) { - return indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) - .repositoryCode(msg.getRepositoryCode()) - .build()) + } + + private Flux> removeAnalysis(IndexAnalysisMessage msg) { + return indexer + .removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .repositoryCode(msg.getRepositoryCode()) + .build()) .build()) - .map(out -> new Tuple2<>(msg, out)) - .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } + .map(out -> new Tuple2<>(msg, out)) + .onErrorResume((e) -> catchUnhandledErrors(msg, e)); + } + + private Flux> indexAnalysis(IndexAnalysisMessage msg) { + return indexer + .indexAnalysis( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .repositoryCode(msg.getRepositoryCode()) + .build()) + .build()) + .map(out -> new Tuple2<>(msg, out)) + .onErrorResume((e) -> catchUnhandledErrors(msg, e)); + } - private Flux> indexAnalysis(IndexAnalysisMessage msg) { - return indexer.indexAnalysis(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() + private Flux> indexStudy(IndexStudyMessage msg) { + return indexer + .indexStudy( + IndexStudyCommand.builder() .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) .repositoryCode(msg.getRepositoryCode()) - .build() - ).build()) + .build()) + .map(out -> new Tuple2<>(msg, out)); + } + + private Mono> indexRepository( + IndexRepositoryMessage msg) { + return indexer + .indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode(msg.getRepositoryCode()).build()) .map(out -> new Tuple2<>(msg, out)) .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } - - private Flux> indexStudy(IndexStudyMessage msg) { - return indexer.indexStudy(IndexStudyCommand.builder() - .studyId(msg.getStudyId()) - .repositoryCode(msg.getRepositoryCode()) - .build()) - .map(out -> new Tuple2<>(msg, out)); - } - - private Mono> indexRepository(IndexRepositoryMessage msg) { - return indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode(msg.getRepositoryCode()) - .build()) - .map(out -> new Tuple2<>(msg, out)) - .onErrorResume((e) -> catchUnhandledErrors(msg, e)); - } - - private Mono> catchUnhandledErrors(T msg, Throwable e) { - log.error("failed processing message: {} ", msg, e); - val indexResult = IndexResult.builder() - .successful(false) - .failureData(FailureData.builder().build()) - .build(); - return Mono.just(new Tuple2<>(msg, indexResult)); - } - + } + + private Mono> catchUnhandledErrors(T msg, Throwable e) { + log.error("failed processing message: {} ", msg, e); + val indexResult = + IndexResult.builder().successful(false).failureData(FailureData.builder().build()).build(); + return Mono.just(new Tuple2<>(msg, indexResult)); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java index 9ea4d77e..86c2b6fa 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/MessagingConfig.java @@ -20,9 +20,8 @@ import bio.overture.maestro.app.infra.adapter.inbound.messaging.song.SongAnalysisStreamListener; import org.springframework.context.annotation.Import; - @Import({ - IndexingMessagesStreamListener.class, - SongAnalysisStreamListener.class, + IndexingMessagesStreamListener.class, + SongAnalysisStreamListener.class, }) -public class MessagingConfig { } +public class MessagingConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java index 34e244e3..2afa60b0 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/AnalysisMessage.java @@ -26,12 +26,8 @@ @ToString @AllArgsConstructor class AnalysisMessage { - @NonNull - private final String analysisId; - @NonNull - private final String studyId; - @NonNull - private final String state; - @NonNull - private final String songServerId; + @NonNull private final String analysisId; + @NonNull private final String studyId; + @NonNull private final String state; + @NonNull private final String songServerId; } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java index 9be8584d..6ba9769d 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisSink.java @@ -18,19 +18,13 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.messaging.Sink; import org.springframework.messaging.SubscribableChannel; public interface SongAnalysisSink { - /** - * Input channel name. - */ - String NAME = "songInput"; - - /** - * @return input channel. - */ - @Input(SongAnalysisSink.NAME) - SubscribableChannel songInput(); + /** Input channel name. */ + String NAME = "songInput"; + /** @return input channel. */ + @Input(SongAnalysisSink.NAME) + SubscribableChannel songInput(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java index 3d48297c..da81dbf5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListener.java @@ -17,12 +17,16 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; +import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; + +import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.Indexer; import bio.overture.maestro.domain.api.message.AnalysisIdentifier; import bio.overture.maestro.domain.api.message.IndexAnalysisCommand; import bio.overture.maestro.domain.api.message.IndexResult; import bio.overture.maestro.domain.api.message.RemoveAnalysisCommand; import io.vavr.Tuple2; +import java.util.List; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.cloud.stream.annotation.EnableBinding; @@ -30,51 +34,55 @@ import org.springframework.messaging.handler.annotation.Payload; import reactor.core.publisher.Flux; -import static bio.overture.maestro.app.infra.adapter.inbound.messaging.IndexMessagesHelper.handleIndexResult; - @Slf4j @EnableBinding(SongAnalysisSink.class) public class SongAnalysisStreamListener { - private static final String PUBLISHED = "PUBLISHED"; + private Indexer indexer; + private final List indexableStudyStatuses; - private Indexer indexer; + public SongAnalysisStreamListener(Indexer indexer, ApplicationProperties properties) { + this.indexer = indexer; + this.indexableStudyStatuses = List.of(properties.indexableStudyStatuses().split(",")); + } - public SongAnalysisStreamListener(Indexer indexer) { - this.indexer = indexer; - } - - @StreamListener(SongAnalysisSink.NAME) - public void handleMessage(@Payload AnalysisMessage analysisMessage) { - log.info("received message : {}", analysisMessage); - handleIndexResult(() -> this.doHandle(analysisMessage)); - } + @StreamListener(SongAnalysisSink.NAME) + public void handleMessage(@Payload AnalysisMessage analysisMessage) { + log.info("received message : {}", analysisMessage); + handleIndexResult(() -> this.doHandle(analysisMessage)); + } - private Flux> doHandle(AnalysisMessage msg) { - Flux result; - try { - if (msg.getState().equals(PUBLISHED)) { - result = indexer.indexAnalysis(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .repositoryCode(msg.getSongServerId()) - .studyId(msg.getStudyId()) - .analysisId(msg.getAnalysisId()) - .build()) + private Flux> doHandle(AnalysisMessage msg) { + Flux result; + try { + if (this.indexableStudyStatuses.contains(msg.getState())) { + result = + indexer.indexAnalysis( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .repositoryCode(msg.getSongServerId()) + .studyId(msg.getStudyId()) + .analysisId(msg.getAnalysisId()) + .build()) + .build()); + } else { + val mono = + indexer.removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId(msg.getAnalysisId()) + .studyId(msg.getStudyId()) + .repositoryCode(msg.getSongServerId()) + .build()) .build()); - } else { // UNPUBLISHED, SUPPRESSED - val mono = indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId(msg.getAnalysisId()) - .studyId(msg.getStudyId()) - .repositoryCode(msg.getSongServerId()) - .build() - ).build()); - result = Flux.from(mono); - } - return result.map(indexResult -> new Tuple2<>(msg, indexResult)); - } catch (Exception e) { - log.error("failed reading message: {} ", msg, e); - return Flux.just(new Tuple2<>(msg, IndexResult.builder().successful(false).build())); - } + result = Flux.from(mono); + } + return result.map(indexResult -> new Tuple2<>(msg, indexResult)); + } catch (Exception e) { + log.error("failed reading message: {} ", msg, e); + return Flux.just(new Tuple2<>(msg, IndexResult.builder().successful(false).build())); } + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java index 6b491a93..b15562e3 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ErrorDetails.java @@ -17,18 +17,17 @@ package bio.overture.maestro.app.infra.adapter.inbound.webapi; +import java.util.Date; import lombok.AllArgsConstructor; import lombok.Getter; -import java.util.Date; - /** * This class is a generic representation for errors to be returned by the GlobalWebExceptionHandler */ @AllArgsConstructor @Getter class ErrorDetails { - private Date timestamp; - private String message; - private String details; -} \ No newline at end of file + private Date timestamp; + private String message; + private String details; +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java index 5fdb3682..44e1b446 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/GlobalWebExceptionHandler.java @@ -17,8 +17,8 @@ package bio.overture.maestro.app.infra.adapter.inbound.webapi; - import bio.overture.maestro.domain.api.exception.NotFoundException; +import java.util.Date; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; @@ -28,32 +28,28 @@ import org.springframework.web.bind.annotation.ResponseStatus; import reactor.core.publisher.Mono; -import java.util.Date; - @Slf4j @ControllerAdvice public class GlobalWebExceptionHandler { - @ExceptionHandler(NotFoundException.class) - @ResponseStatus(HttpStatus.NOT_FOUND) - @ResponseBody - public Mono resourceNotFoundException(NotFoundException ex, ServerHttpRequest request) { - log.error("Resource not found exception", ex); - return getErrorDetails(ex, request); - } - - @ExceptionHandler(Exception.class) - @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) - @ResponseBody - public Mono globalExceptionHandler(Exception ex, ServerHttpRequest request) { - log.error("Unhandled exception", ex); - return getErrorDetails(ex, request); - } - - private Mono getErrorDetails(Exception ex, ServerHttpRequest request) { - return Mono.just(new ErrorDetails(new Date(), - ex.getMessage(), request.getPath().toString()) - ); - } - -} \ No newline at end of file + @ExceptionHandler(NotFoundException.class) + @ResponseStatus(HttpStatus.NOT_FOUND) + @ResponseBody + public Mono resourceNotFoundException( + NotFoundException ex, ServerHttpRequest request) { + log.error("Resource not found exception", ex); + return getErrorDetails(ex, request); + } + + @ExceptionHandler(Exception.class) + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + @ResponseBody + public Mono globalExceptionHandler(Exception ex, ServerHttpRequest request) { + log.error("Unhandled exception", ex); + return getErrorDetails(ex, request); + } + + private Mono getErrorDetails(Exception ex, ServerHttpRequest request) { + return Mono.just(new ErrorDetails(new Date(), ex.getMessage(), request.getPath().toString())); + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java index e5020648..5668a7c2 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/inbound/webapi/ManagementController.java @@ -51,7 +51,7 @@ public Mono> ping() { @DeleteMapping("/index/repository/{repositoryCode}/study/{studyId}/analysis/{analysisId}") @ResponseStatus(HttpStatus.OK) - public Mono removeAnalysis( + public Flux removeAnalysis( @PathVariable String analysisId, @PathVariable String studyId, @PathVariable String repositoryCode) { @@ -71,8 +71,7 @@ public Mono removeAnalysis( .build()); } - @PostMapping( - "/index/repository/{repositoryCode}/study/{studyId}/analysis/{analysisId}") + @PostMapping("/index/repository/{repositoryCode}/study/{studyId}/analysis/{analysisId}") @ResponseStatus(HttpStatus.CREATED) public Flux indexAnalysis( @PathVariable String analysisId, diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java index bcc0be42..1a3d7955 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/AnalysisCentricElasticSearchAdapter.java @@ -14,7 +14,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.databind.PropertyNamingStrategy; - +import io.github.resilience4j.retry.Retry; +import io.github.resilience4j.retry.RetryConfig; +import java.io.IOException; +import java.time.Duration; import java.util.*; import javax.inject.Inject; import lombok.NonNull; @@ -29,11 +32,14 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.core.io.Resource; import org.springframework.retry.annotation.Backoff; import org.springframework.retry.annotation.Retryable; import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; @Slf4j public class AnalysisCentricElasticSearchAdapter implements AnalysisCentricIndexAdapter { @@ -92,8 +98,13 @@ public Mono batchUpsertAnalysisRepositories( this.indexName, this.elasticsearchRestClient, AnalysisCentricDocument::getAnalysisId, - this::mapAnalysisToUpsertRepositoryQuery - ); + this::mapAnalysisToUpsertRepositoryQuery); + } + + @Override + public Mono removeAnalysisDocs(String analysisId) { + return Mono.fromSupplier(() -> this.deleteByAnalysisId(analysisId)) + .subscribeOn(Schedulers.elastic()); } @SneakyThrows @@ -155,4 +166,28 @@ private String loadIndexSourceAsString(String typeName) { log.trace("in loadIndexSourceAsString: {}", typeName); return inputStreamToString(analysisCentricIndex.getInputStream()); } + + @SneakyThrows + private Void deleteByAnalysisId(String analysisId) { + val retryConfig = + RetryConfig.custom() + .maxAttempts(this.maxRetriesAttempts) + .retryExceptions(IOException.class) + .waitDuration(Duration.ofMillis(this.retriesWaitDuration)) + .build(); + val retry = Retry.of("deleteByAnalysisId", retryConfig); + val decorated = + Retry.decorateCheckedRunnable(retry, () -> this.deleteByAnalysisIdRunnable(analysisId)); + decorated.run(); + return null; + } + + @SneakyThrows + private void deleteByAnalysisIdRunnable(@NonNull String analysisId) { + log.trace("deleteByAnalysisId called, analysis_id {} ", analysisId); + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(this.alias); + deleteByQueryRequest.setQuery( + QueryBuilders.boolQuery().must(QueryBuilders.termQuery("analysis_id", analysisId))); + this.elasticsearchRestClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java index cf00ccd1..c3077289 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/ElasticSearchConfig.java @@ -17,9 +17,16 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.elasticsearch; +import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.stream.Collectors; import lombok.val; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; @@ -36,90 +43,85 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.stream.Collectors; - -import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; - /** * Elasticsearch related configuration this allows us to keep the beans package private to avoid - * other packages using them instead of the interface, and be more explicit about configuration scope. + * other packages using them instead of the interface, and be more explicit about configuration + * scope. */ @Configuration @Import({ - FileCentricElasticSearchAdapter.class, - AnalysisCentricElasticSearchAdapter.class, - SnakeCaseJacksonSearchResultMapper.class + FileCentricElasticSearchAdapter.class, + AnalysisCentricElasticSearchAdapter.class, + SnakeCaseJacksonSearchResultMapper.class }) public class ElasticSearchConfig { - /** - * this bean executes when the application starts it's used to initialize the - * indexes in elastic search server, can be extended as needed. - */ - @Bean - CommandLineRunner elasticsearchBootstrapper(FileCentricElasticSearchAdapter adapter) { - return (args) -> adapter.initialize(); - } + /** + * this bean executes when the application starts it's used to initialize the indexes in elastic + * search server, can be extended as needed. + */ + @Bean + CommandLineRunner elasticsearchBootstrapper(FileCentricElasticSearchAdapter adapter) { + return (args) -> adapter.initialize(); + } - @Bean - CommandLineRunner analysisElasticsearchBootstrapper(AnalysisCentricElasticSearchAdapter adapter) { - return (args) -> adapter.initialize(); - } + @Bean + CommandLineRunner analysisElasticsearchBootstrapper(AnalysisCentricElasticSearchAdapter adapter) { + return (args) -> adapter.initialize(); + } - @Bean - RestHighLevelClient client(ApplicationProperties properties) { - val httpHostArrayList = new ArrayList(properties.elasticSearchClusterNodes() - .stream() - .map(HttpHost::create) - .collect(Collectors.toUnmodifiableList())); - val builder = RestClient.builder(httpHostArrayList.toArray(new HttpHost[]{})); + @Bean + RestHighLevelClient client(ApplicationProperties properties) { + val httpHostArrayList = + new ArrayList( + properties.elasticSearchClusterNodes().stream() + .map(HttpHost::create) + .collect(Collectors.toUnmodifiableList())); + val builder = RestClient.builder(httpHostArrayList.toArray(new HttpHost[] {})); - builder.setHttpClientConfigCallback((httpAsyncClientBuilder) -> { - val connectTimeout = properties.elasticSearchClientConnectionTimeoutMillis(); - val timeout = properties.elasticSearchClientSocketTimeoutMillis(); - val requestConfigBuilder = RequestConfig.custom(); + builder.setHttpClientConfigCallback( + (httpAsyncClientBuilder) -> { + val connectTimeout = properties.elasticSearchClientConnectionTimeoutMillis(); + val timeout = properties.elasticSearchClientSocketTimeoutMillis(); + val requestConfigBuilder = RequestConfig.custom(); - if (connectTimeout > 0) { - requestConfigBuilder.setConnectTimeout(connectTimeout); - requestConfigBuilder.setConnectionRequestTimeout(connectTimeout); - } - if (timeout > 0) { - requestConfigBuilder.setSocketTimeout(timeout); - } - httpAsyncClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); + if (connectTimeout > 0) { + requestConfigBuilder.setConnectTimeout(connectTimeout); + requestConfigBuilder.setConnectionRequestTimeout(connectTimeout); + } + if (timeout > 0) { + requestConfigBuilder.setSocketTimeout(timeout); + } + httpAsyncClientBuilder.setDefaultRequestConfig(requestConfigBuilder.build()); - if (properties.elasticSearchTlsTrustSelfSigned()) { - SSLContextBuilder sslCtxBuilder = new SSLContextBuilder(); - try { - sslCtxBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); - httpAsyncClientBuilder.setSSLContext(sslCtxBuilder.build()); - } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { - throw new RuntimeException("failed to build Elastic rest client"); - } + if (properties.elasticSearchTlsTrustSelfSigned()) { + SSLContextBuilder sslCtxBuilder = new SSLContextBuilder(); + try { + sslCtxBuilder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + httpAsyncClientBuilder.setSSLContext(sslCtxBuilder.build()); + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new RuntimeException("failed to build Elastic rest client"); } + } - // set the credentials provider for auth - if (properties.elasticSearchBasicAuthEnabled()) { - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, - new UsernamePasswordCredentials(properties.elasticSearchAuthUser(), - properties.elasticSearchAuthPassword())); - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } - return httpAsyncClientBuilder; + // set the credentials provider for auth + if (properties.elasticSearchBasicAuthEnabled()) { + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials( + AuthScope.ANY, + new UsernamePasswordCredentials( + properties.elasticSearchAuthUser(), properties.elasticSearchAuthPassword())); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + return httpAsyncClientBuilder; }); - return new RestHighLevelClient(builder); - } - - @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper documentObjectMapper() { - val mapper = new ObjectMapper(); - mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return mapper; - } + return new RestHighLevelClient(builder); + } + @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + ObjectMapper documentObjectMapper() { + val mapper = new ObjectMapper(); + mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return mapper; + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java index 1563f07f..597b442c 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapter.java @@ -25,7 +25,6 @@ import bio.overture.maestro.app.infra.config.RootConfiguration; import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.message.IndexResult; -import bio.overture.maestro.domain.entities.indexing.FileCentricAnalysis; import bio.overture.maestro.domain.entities.indexing.FileCentricDocument; import bio.overture.maestro.domain.port.outbound.indexing.BatchIndexFilesCommand; import bio.overture.maestro.domain.port.outbound.indexing.FileCentricIndexAdapter; @@ -117,8 +116,7 @@ public Mono batchUpsertFileRepositories( this.indexName, this.elasticsearchRestClient, this::getAnalysisId, - this::mapFileToUpsertRepositoryQuery - ); + this::mapFileToUpsertRepositoryQuery); } private String getAnalysisId(FileCentricDocument d) { @@ -198,8 +196,7 @@ private void deleteByAnalysisIdRunnable(String analysisId) { QueryBuilders.boolQuery() .must( QueryBuilders.termQuery( - FileCentricDocument.Fields.analysis + "." + "analysis_id", - analysisId))); + FileCentricDocument.Fields.analysis + "." + "analysis_id", analysisId))); this.elasticsearchRestClient.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java index 077b3284..fcecd970 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SearchAdapterHelper.java @@ -6,6 +6,11 @@ import io.github.resilience4j.retry.Retry; import io.github.resilience4j.retry.RetryConfig; import io.vavr.control.Try; +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; @@ -24,179 +29,182 @@ import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import java.io.IOException; -import java.time.Duration; -import java.util.*; -import java.util.function.Function; -import java.util.stream.Collectors; - - @Slf4j @NoArgsConstructor public class SearchAdapterHelper { - private static final String ANALYSIS_ID = "analysisId"; - - public static Mono batchUpsertDocuments( - @NonNull List documents, - int documentsPerBulkRequest, - int maxRetriesAttempts, - long retriesWaitDuration, - String indexName, - RestHighLevelClient client, - Function documentAnalysisIdExtractor, - Function mapper - ) { - log.debug( - "in batchUpsertAnalysisRepositories, analyses count: {} ", - documents.size()); - return Mono.fromSupplier( - () -> bulkUpsertAnalysisRepositories(documents, + private static final String ANALYSIS_ID = "analysisId"; + + public static Mono batchUpsertDocuments( + @NonNull List documents, + int documentsPerBulkRequest, + int maxRetriesAttempts, + long retriesWaitDuration, + String indexName, + RestHighLevelClient client, + Function documentAnalysisIdExtractor, + Function mapper) { + log.debug("in batchUpsertAnalysisRepositories, analyses count: {} ", documents.size()); + return Mono.fromSupplier( + () -> + bulkUpsertAnalysisRepositories( + documents, + documentsPerBulkRequest, + maxRetriesAttempts, + retriesWaitDuration, + indexName, + client, + documentAnalysisIdExtractor, + mapper)) + .subscribeOn(Schedulers.elastic()); + } + + @SneakyThrows + private static IndexResult bulkUpsertAnalysisRepositories( + List analyses, + int documentsPerBulkRequest, + int maxRetriesAttempts, + long retriesWaitDuration, + String indexName, + RestHighLevelClient client, + Function documentAnalysisIdExtractor, + Function mapper) { + log.trace( + "in SearchAdapterHelper - bulkUpsertAnalysisRepositories, analyses count : {} ", + analyses.size()); + val failures = + Parallel.blockingScatterGather( + analyses, documentsPerBulkRequest, - maxRetriesAttempts, - retriesWaitDuration, - indexName, - client, - documentAnalysisIdExtractor, - mapper) - ).subscribeOn(Schedulers.elastic()); - } - - @SneakyThrows - private static IndexResult bulkUpsertAnalysisRepositories(List analyses, - int documentsPerBulkRequest, - int maxRetriesAttempts, - long retriesWaitDuration, - String indexName, - RestHighLevelClient client, - Function documentAnalysisIdExtractor, - Function mapper) { - log.trace("in AnalysisCentricElasticSearchAdapter - bulkUpsertAnalysisRepositories, analyses count : {} ", analyses.size()); - val failures = - Parallel.blockingScatterGather(analyses, - documentsPerBulkRequest, - (list) -> tryBulkUpsertRequestForPart(list, maxRetriesAttempts, retriesWaitDuration, mapper, documentAnalysisIdExtractor, client)) - .stream() - .flatMap(Set::stream) - .collect(Collectors.toUnmodifiableSet()); - return buildIndexResult(failures, indexName); - } - - @NotNull - private static Set tryBulkUpsertRequestForPart( - Map.Entry> entry, - int maxRetriesAttempts, - long retriesWaitDuration, - Function mapper, - Function documentAnalysisIdExtractor, - RestHighLevelClient client - ) { - val partNum = entry.getKey(); - val listPart = entry.getValue(); - val listPartHash = Objects.hashCode(listPart); - val retry = buildRetry(maxRetriesAttempts, retriesWaitDuration); - val decorated = - Retry.>decorateCheckedSupplier( - retry, - () -> { - log.trace( - "AnalysisCentricElasticSearchAdapter - tryBulkUpsertRequestForPart, sending part#: {}, hash: {} ", - partNum, - listPartHash); - doRequestForPart(listPart, mapper, client); - log.trace("AnalysisCentricElasticSearchAdapter - tryBulkUpsertRequestForPart: done bulk upsert all docs"); - return Set.of(); + (list) -> + tryBulkUpsertRequestForPart( + list, + maxRetriesAttempts, + retriesWaitDuration, + mapper, + documentAnalysisIdExtractor, + client)) + .stream() + .flatMap(Set::stream) + .collect(Collectors.toUnmodifiableSet()); + return buildIndexResult(failures, indexName); + } + + @NotNull + private static Set tryBulkUpsertRequestForPart( + Map.Entry> entry, + int maxRetriesAttempts, + long retriesWaitDuration, + Function mapper, + Function documentAnalysisIdExtractor, + RestHighLevelClient client) { + val partNum = entry.getKey(); + val listPart = entry.getValue(); + val listPartHash = Objects.hashCode(listPart); + val retry = buildRetry(maxRetriesAttempts, retriesWaitDuration); + val decorated = + Retry.>decorateCheckedSupplier( + retry, + () -> { + log.trace( + "SearchAdapterHelper - tryBulkUpsertRequestForPart, sending part#: {}, hash: {} ", + partNum, + listPartHash); + doRequestForPart(listPart, mapper, client); + log.trace( + "SearchAdapterHelper - tryBulkUpsertRequestForPart: done bulk upsert all docs"); + return Set.of(); + }); + val result = + Try.of(decorated) + .recover( + (t) -> { + log.error( + "failed sending request for: part#: {}, hash: {} to elastic search," + + " gathering failed Ids.", + partNum, + listPartHash, + t); + return listPart.stream() + .map(documentAnalysisIdExtractor) + .collect(Collectors.toUnmodifiableSet()); }); - val result = - Try.of(decorated) - .recover( - (t) -> { - log.error( - "failed sending request for: part#: {}, hash: {} to elastic search," - + " gathering failed Ids.", - partNum, - listPartHash, - t); - return listPart.stream() - .map(documentAnalysisIdExtractor) - .collect(Collectors.toUnmodifiableSet()); - }); - return result.get(); + return result.get(); + } + + private static void doRequestForPart( + List listPart, Function mapper, RestHighLevelClient client) + throws IOException { + bulkUpdateRequest(listPart.stream().map(mapper).collect(Collectors.toList()), client); + } + + private static void bulkUpdateRequest(List requests, RestHighLevelClient client) + throws IOException { + val bulkRequest = buildBulkUpdateRequest(requests); + checkForBulkUpdateFailure(client.bulk(bulkRequest, RequestOptions.DEFAULT)); + } + + public static IndexResult buildIndexResult( + @NonNull Set failures, @NonNull String indexName) { + val fails = + failures.isEmpty() + ? FailureData.builder().build() + : FailureData.builder().failingIds(Map.of(ANALYSIS_ID, failures)).build(); + return IndexResult.builder() + .indexName(indexName) + .failureData(fails) + .successful(failures.isEmpty()) + .build(); + } + + public static Retry buildRetry(int maxRetriesAttempts, long retriesWaitDuration) { + val retryConfig = + RetryConfig.custom() + .maxAttempts(maxRetriesAttempts) + .retryExceptions(IOException.class) + .waitDuration(Duration.ofMillis(retriesWaitDuration)) + .build(); + val retry = Retry.of("tryBulkUpsertRequestForPart", retryConfig); + return retry; + } + + public static BulkRequest buildBulkUpdateRequest(List requests) + throws IOException { + val bulkRequest = new BulkRequest(); + for (UpdateRequest query : requests) { + bulkRequest.add(prepareUpdate(query)); } - - private static void doRequestForPart(List listPart, - Function mapper, - RestHighLevelClient client) throws IOException { - bulkUpdateRequest( - listPart.stream() - .map(mapper) - .collect(Collectors.toList()), - client); + return bulkRequest; + } + + public static void checkForBulkUpdateFailure(BulkResponse bulkResponse) { + if (bulkResponse.hasFailures()) { + val failedDocuments = new HashMap(); + for (BulkItemResponse item : bulkResponse.getItems()) { + if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage()); + } + throw new RuntimeException( + "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" + + failedDocuments + + "]"); } - - private static void bulkUpdateRequest(List requests, RestHighLevelClient client) throws IOException { - val bulkRequest = buildBulkUpdateRequest(requests); - checkForBulkUpdateFailure(client.bulk(bulkRequest, RequestOptions.DEFAULT)); - } - - public static IndexResult buildIndexResult(@NonNull Set failures, @NonNull String indexName){ - val fails = - failures.isEmpty() - ? FailureData.builder().build() - : FailureData.builder().failingIds(Map.of(ANALYSIS_ID, failures)).build(); - return IndexResult.builder() - .indexName(indexName) - .failureData(fails).successful(failures.isEmpty()).build(); - } - - public static Retry buildRetry(int maxRetriesAttempts, long retriesWaitDuration){ - val retryConfig = - RetryConfig.custom() - .maxAttempts(maxRetriesAttempts) - .retryExceptions(IOException.class) - .waitDuration(Duration.ofMillis(retriesWaitDuration)) - .build(); - val retry = Retry.of("tryBulkUpsertRequestForPart", retryConfig); - return retry; - } - - public static BulkRequest buildBulkUpdateRequest(List requests) throws IOException { - val bulkRequest = new BulkRequest(); - for (UpdateRequest query : requests) { - bulkRequest.add(prepareUpdate(query)); - } - return bulkRequest; - } - - public static void checkForBulkUpdateFailure(BulkResponse bulkResponse) { - if (bulkResponse.hasFailures()) { - val failedDocuments = new HashMap(); - for (BulkItemResponse item : bulkResponse.getItems()) { - if (item.isFailed()) failedDocuments.put(item.getId(), item.getFailureMessage()); - } - throw new RuntimeException( - "Bulk indexing has failures. Use ElasticsearchException.getFailedDocuments() for detailed messages [" - + failedDocuments - + "]"); - } - } - - public static Script getInline(Map parameters) { - val inline = - new Script( - ScriptType.INLINE, - "painless", - "if (!ctx._source.repositories.contains(params.repository)) { ctx._source.repositories.add(params.repository) }", - parameters); - return inline; - } - - private static UpdateRequest prepareUpdate(UpdateRequest req) { - Assert.notNull(req, "No IndexRequest define for Query"); - String indexName = req.index(); - Assert.notNull(indexName, "No index defined for Query"); - Assert.notNull(req.id(), "No Id define for Query"); - return req; - } - -} \ No newline at end of file + } + + public static Script getInline(Map parameters) { + val inline = + new Script( + ScriptType.INLINE, + "painless", + "if (!ctx._source.repositories.contains(params.repository)) { ctx._source.repositories.add(params.repository) }", + parameters); + return inline; + } + + private static UpdateRequest prepareUpdate(UpdateRequest req) { + Assert.notNull(req, "No IndexRequest define for Query"); + String indexName = req.index(); + Assert.notNull(indexName, "No index defined for Query"); + Assert.notNull(req.id(), "No Id define for Query"); + return req; + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java index a51f6bc8..2a950aa0 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/SnakeCaseJacksonSearchResultMapper.java @@ -19,38 +19,37 @@ import bio.overture.maestro.app.infra.config.RootConfiguration; import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Arrays; +import java.util.LinkedList; import lombok.SneakyThrows; import lombok.val; import org.elasticsearch.action.get.MultiGetResponse; import org.springframework.beans.factory.annotation.Qualifier; -import java.util.Arrays; -import java.util.LinkedList; - - class SnakeCaseJacksonSearchResultMapper { - private ObjectMapper objectMapper; - - public SnakeCaseJacksonSearchResultMapper(@Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper objectMapper) { - this.objectMapper = objectMapper; - } - - @SneakyThrows - LinkedList mapResults(MultiGetResponse responses, Class clazz) { - val list = new LinkedList(); - Arrays.stream(responses.getResponses()) - .filter((response) -> !response.isFailed() && response.getResponse().isExists()) - .forEach((response) -> { - T result = convertSourceToObject(clazz, response.getResponse().getSourceAsString()); - list.add(result); + private ObjectMapper objectMapper; + + public SnakeCaseJacksonSearchResultMapper( + @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + @SneakyThrows + LinkedList mapResults(MultiGetResponse responses, Class clazz) { + val list = new LinkedList(); + Arrays.stream(responses.getResponses()) + .filter((response) -> !response.isFailed() && response.getResponse().isExists()) + .forEach( + (response) -> { + T result = convertSourceToObject(clazz, response.getResponse().getSourceAsString()); + list.add(result); }); - return list; - } - - @SneakyThrows - private T convertSourceToObject(Class clazz, String source) { - return objectMapper.readValue(source, clazz); - } -} \ No newline at end of file + return list; + } + + @SneakyThrows + private T convertSourceToObject(Class clazz, String source) { + return objectMapper.readValue(source, clazz); + } +} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java index 0811f51f..6f9f3cc5 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ApplicationPropertiesExclusionRulesDAO.java @@ -17,89 +17,88 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.rules; +import static java.text.MessageFormat.format; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; import bio.overture.maestro.domain.entities.indexing.rules.IDExclusionRule; import bio.overture.maestro.domain.entities.metadata.study.*; import bio.overture.maestro.domain.port.outbound.indexing.rules.ExclusionRulesDAO; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import lombok.*; -import lombok.extern.slf4j.Slf4j; -import org.springframework.core.io.Resource; -import reactor.core.publisher.Mono; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; -import java.io.FileNotFoundException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; - -import static java.text.MessageFormat.format; +import javax.inject.Inject; +import lombok.*; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Mono; /** - * Rules dao backed by application properties as source, it assumes a yaml - * structure matching the class: {@link RuleConfig} + * Rules dao backed by application properties as source, it assumes a yaml structure matching the + * class: {@link RuleConfig} */ @Slf4j public class ApplicationPropertiesExclusionRulesDAO implements ExclusionRulesDAO { - private Map, List> exclusionRules; + private Map, List> exclusionRules; - @Inject - public ApplicationPropertiesExclusionRulesDAO(ApplicationProperties properties) { - this.init(properties.idExclusionRules()); - } + @Inject + public ApplicationPropertiesExclusionRulesDAO(ApplicationProperties properties) { + this.init(properties.idExclusionRules()); + } - @SneakyThrows - public Mono, List>> getExclusionRules() { - return Mono.just(this.exclusionRules); - } + @SneakyThrows + public Mono, List>> getExclusionRules() { + return Mono.just(this.exclusionRules); + } - private void init(Map> idExclusionRules) { - try { - if (idExclusionRules == null || idExclusionRules.isEmpty()) { - this.exclusionRules = Map.of(); - return; - } + private void init(Map> idExclusionRules) { + try { + if (idExclusionRules == null || idExclusionRules.isEmpty()) { + this.exclusionRules = Map.of(); + return; + } - val rulesByEntity = new LinkedHashMap, List>(); - idExclusionRules.forEach((entity, ids) -> { - if (ids.isEmpty()) return; - val entityClass = getClassFor(entity); - rulesByEntity.put(entityClass, List.of(IDExclusionRule.builder() - .clazz(entityClass) - .ids(ids) - .build() - ) - ); - }); + val rulesByEntity = new LinkedHashMap, List>(); + idExclusionRules.forEach( + (entity, ids) -> { + if (ids.isEmpty()) return; + val entityClass = getClassFor(entity); + rulesByEntity.put( + entityClass, + List.of(IDExclusionRule.builder().clazz(entityClass).ids(ids).build())); + }); - this.exclusionRules = Map.copyOf(rulesByEntity); - log.info("loaded exclusionRules :{}", this.exclusionRules.size()); - } catch (Exception e) { - this.exclusionRules = Map.of(); - log.error("failed to read exclusion rules", e); - } + this.exclusionRules = Map.copyOf(rulesByEntity); + log.info("loaded exclusionRules :{}", this.exclusionRules.size()); + } catch (Exception e) { + this.exclusionRules = Map.of(); + log.error("failed to read exclusion rules", e); } + } - private Class getClassFor(String entity) { - switch (entity) { - case "studyId": return Study.class; - case "analysis": return Analysis.class; - case "files": return File.class; - case "donor": return Donor.class; - case "samples": return Sample.class; - case "specimen" : return Specimen.class; - } - throw new IllegalArgumentException(format("entity : {0} is not recognized for exclusion rules", entity)); + private Class getClassFor(String entity) { + switch (entity) { + case "studyId": + return Study.class; + case "analysis": + return Analysis.class; + case "files": + return File.class; + case "donor": + return Donor.class; + case "samples": + return Sample.class; + case "specimen": + return Specimen.class; } + throw new IllegalArgumentException( + format("entity : {0} is not recognized for exclusion rules", entity)); + } - @Getter - @NoArgsConstructor - @AllArgsConstructor - private static class RuleConfig { - private Map> byId; - } + @Getter + @NoArgsConstructor + @AllArgsConstructor + private static class RuleConfig { + private Map> byId; + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java index 147022d0..fa677306 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/rules/ExclusionRulesConfig.java @@ -22,10 +22,9 @@ /** * Elasticsearch related configuration this allows us to keep the beans package private to avoid - * other packages using them instead of the interface, and be more explicit about configuration scope. + * other packages using them instead of the interface, and be more explicit about configuration + * scope. */ @Configuration -@Import({ - ApplicationPropertiesExclusionRulesDAO.class -}) -public class ExclusionRulesConfig { } +@Import({ApplicationPropertiesExclusionRulesDAO.class}) +public class ExclusionRulesConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java index fdf783e9..2989ae3f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/PropertyFileStudyRepositoryDAO.java @@ -17,10 +17,14 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.repostiory; +import static bio.overture.maestro.domain.utility.Exceptions.notFound; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.app.infra.config.properties.PropertiesFileRepository; import bio.overture.maestro.domain.entities.metadata.repository.StudyRepository; import bio.overture.maestro.domain.port.outbound.metadata.repository.StudyRepositoryDAO; +import java.util.List; +import javax.inject.Inject; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.NonNull; @@ -28,59 +32,55 @@ import lombok.val; import reactor.core.publisher.Mono; -import javax.inject.Inject; -import java.util.List; - -import static bio.overture.maestro.domain.utility.Exceptions.notFound; - /** - * Properties files backed repository store, reads the information by binding - * to the application.config files property: maestro.repositories - * the configurable attributes can be found here: {@link PropertiesFileRepository} + * Properties files backed repository store, reads the information by binding to the + * application.config files property: maestro.repositories the configurable attributes can be found + * here: {@link PropertiesFileRepository} * - * this serves as a default in memory store, more sophisticated cases may create a DB backed store. + *

this serves as a default in memory store, more sophisticated cases may create a DB backed + * store. */ @Slf4j @Getter @NoArgsConstructor class PropertyFileStudyRepositoryDAO implements StudyRepositoryDAO { - private static final String MSG_REPO_NOT_FOUND = "Repository {0} not found"; - private List repositories; + private static final String MSG_REPO_NOT_FOUND = "Repository {0} not found"; + private List repositories; - @Inject - public PropertyFileStudyRepositoryDAO(ApplicationProperties properties) { - this.repositories = properties.repositories(); - } + @Inject + public PropertyFileStudyRepositoryDAO(ApplicationProperties properties) { + this.repositories = properties.repositories(); + } - @Override - @NonNull - public Mono getFilesRepository(@NonNull String code) { - val repository = repositories.stream() - .filter(propertiesFileRepository -> propertiesFileRepository.getCode().equalsIgnoreCase(code)) + @Override + @NonNull + public Mono getFilesRepository(@NonNull String code) { + val repository = + repositories.stream() + .filter( + propertiesFileRepository -> + propertiesFileRepository.getCode().equalsIgnoreCase(code)) .distinct() - .map(this :: toFilesRepository) + .map(this::toFilesRepository) .findFirst() .orElse(null); - if (repository == null) { - return Mono.error(notFound(MSG_REPO_NOT_FOUND, code)); - } - log.debug("loaded repository : {}", repository); - return Mono.just(repository); - } - - private StudyRepository toFilesRepository(PropertiesFileRepository propertiesFileRepository) { - log.trace("Converting : {} to StudyRepository ", propertiesFileRepository); - return StudyRepository.builder() - .code(propertiesFileRepository.getCode()) - .url(propertiesFileRepository.getUrl()) - .name(propertiesFileRepository.getName()) - .dataPath(propertiesFileRepository.getDataPath()) - .metadataPath(propertiesFileRepository.getMetadataPath()) - .organization(propertiesFileRepository.getOrganization()) - .country(propertiesFileRepository.getCountry()) - .storageType(propertiesFileRepository.getStorageType()) - .build(); + if (repository == null) { + return Mono.error(notFound(MSG_REPO_NOT_FOUND, code)); } + log.debug("loaded repository : {}", repository); + return Mono.just(repository); + } + private StudyRepository toFilesRepository(PropertiesFileRepository propertiesFileRepository) { + log.trace("Converting : {} to StudyRepository ", propertiesFileRepository); + return StudyRepository.builder() + .code(propertiesFileRepository.getCode()) + .url(propertiesFileRepository.getUrl()) + .name(propertiesFileRepository.getName()) + .organization(propertiesFileRepository.getOrganization()) + .country(propertiesFileRepository.getCountry()) + .storageType(propertiesFileRepository.getStorageType()) + .build(); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java index 7a95fd66..9e26ce63 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/repostiory/RepositoryConfig.java @@ -21,8 +21,5 @@ import org.springframework.context.annotation.Import; @Configuration -@Import({ - PropertyFileStudyRepositoryDAO.class -}) -public class RepositoryConfig { -} +@Import({PropertyFileStudyRepositoryDAO.class}) +public class RepositoryConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java index ce10142d..f58fa085 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongConfig.java @@ -21,8 +21,5 @@ import org.springframework.context.annotation.Import; @Configuration -@Import({ - SongStudyDAO.class -}) -public class SongConfig { -} +@Import({SongStudyDAO.class}) +public class SongConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java index 430e3517..7b3fe821 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAO.java @@ -17,6 +17,10 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.study.song; +import static bio.overture.maestro.domain.utility.Exceptions.notFound; +import static java.text.MessageFormat.format; +import static reactor.core.publisher.Mono.error; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.exception.NotFoundException; import bio.overture.maestro.domain.entities.metadata.study.Analysis; @@ -25,137 +29,167 @@ import bio.overture.maestro.domain.port.outbound.metadata.study.GetAnalysisCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.GetStudyAnalysesCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.StudyDAO; +import java.time.Duration; +import java.util.List; +import java.util.function.Function; +import javax.inject.Inject; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; -import org.springframework.web.reactive.function.client.ExchangeStrategies; import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.retry.Retry; -import javax.inject.Inject; -import java.time.Duration; -import java.util.List; -import java.util.function.Function; - -import static bio.overture.maestro.domain.utility.Exceptions.notFound; -import static java.text.MessageFormat.format; -import static reactor.core.publisher.Mono.error; @Slf4j class SongStudyDAO implements StudyDAO { - private static final String STUDY_ANALYSES_URL_TEMPLATE = "{0}/studies/{1}/analysis?analysisStates={2}"; - private static final String STUDY_ANALYSIS_URL_TEMPLATE = "{0}/studies/{1}/analysis/{2}"; - private static final String STUDIES_URL_TEMPLATE = "{0}/studies/all"; - private static final String MSG_STUDY_DOES_NOT_EXIST = "studyId {0} doesn't exist in the specified repository"; - private static final String MSG_ANALYSIS_DOES_NOT_EXIST = - "analysis {0} doesn't exist for studyId {1}, repository {2} (or not in a matching state)"; - private static final int FALLBACK_SONG_TIMEOUT = 60; - private static final int FALLBACK_SONG_ANALYSIS_TIMEOUT = 5; - private static final int FALLBACK_SONG_MAX_RETRY = 0; - private final WebClient webClient; - private final int songMaxRetries; - private final int minBackoffSec = 1; - private final int maxBackoffSec = 5; - private final String indexableStudyStatuses; - private final List indexableStudyStatusesList; - /** - * must be bigger than 0 or all calls will fail - */ - private final int studyCallTimeoutSeconds; - private final int analysisCallTimeoutSeconds; + private static final String STUDY_ANALYSES_URL_TEMPLATE = + "{0}/studies/{1}/analysis?analysisStates={2}"; + private static final String STUDY_ANALYSIS_URL_TEMPLATE = "{0}/studies/{1}/analysis/{2}"; + private static final String STUDIES_URL_TEMPLATE = "{0}/studies/all"; + private static final String MSG_STUDY_DOES_NOT_EXIST = + "studyId {0} doesn't exist in the specified repository"; + private static final String MSG_ANALYSIS_DOES_NOT_EXIST = + "analysis {0} doesn't exist for studyId {1}, repository {2} (or not in a matching state)"; + private static final int FALLBACK_SONG_TIMEOUT = 60; + private static final int FALLBACK_SONG_ANALYSIS_TIMEOUT = 5; + private static final int FALLBACK_SONG_MAX_RETRY = 0; + private final WebClient webClient; + private final int songMaxRetries; + private final int minBackoffSec = 1; + private final int maxBackoffSec = 5; + private final String indexableStudyStatuses; + private final List indexableStudyStatusesList; + /** must be bigger than 0 or all calls will fail */ + private final int studyCallTimeoutSeconds; + + private final int analysisCallTimeoutSeconds; - @Inject - public SongStudyDAO(@NonNull WebClient webClient, @NonNull ApplicationProperties applicationProperties) { - this.webClient = webClient; - this.indexableStudyStatuses = applicationProperties.indexableStudyStatuses(); - this.indexableStudyStatusesList = List.of(indexableStudyStatuses.split(",")); - this.songMaxRetries = applicationProperties.songMaxRetries() >= 0 ? applicationProperties.songMaxRetries() + @Inject + public SongStudyDAO( + @NonNull WebClient webClient, @NonNull ApplicationProperties applicationProperties) { + this.webClient = webClient; + this.indexableStudyStatuses = applicationProperties.indexableStudyStatuses(); + this.indexableStudyStatusesList = List.of(indexableStudyStatuses.split(",")); + this.songMaxRetries = + applicationProperties.songMaxRetries() >= 0 + ? applicationProperties.songMaxRetries() : FALLBACK_SONG_MAX_RETRY; - this.studyCallTimeoutSeconds = applicationProperties.songStudyCallTimeoutSeconds() > 0 ? applicationProperties.songStudyCallTimeoutSeconds() + this.studyCallTimeoutSeconds = + applicationProperties.songStudyCallTimeoutSeconds() > 0 + ? applicationProperties.songStudyCallTimeoutSeconds() : FALLBACK_SONG_TIMEOUT; - this.analysisCallTimeoutSeconds = applicationProperties.songAnalysisCallTimeoutSeconds() > 0 ? - applicationProperties.songAnalysisCallTimeoutSeconds(): FALLBACK_SONG_ANALYSIS_TIMEOUT; - } + this.analysisCallTimeoutSeconds = + applicationProperties.songAnalysisCallTimeoutSeconds() > 0 + ? applicationProperties.songAnalysisCallTimeoutSeconds() + : FALLBACK_SONG_ANALYSIS_TIMEOUT; + } - @Override - public Mono> getStudyAnalyses(GetStudyAnalysesCommand getStudyAnalysesCommand) { - log.trace("in getStudyAnalyses, args: {} ", getStudyAnalysesCommand); - val repoBaseUrl = getStudyAnalysesCommand.getFilesRepositoryBaseUrl(); - val studyId = getStudyAnalysesCommand.getStudyId(); - val analysisListType = new ParameterizedTypeReference>(){}; - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Mono> getStudyAnalyses(GetStudyAnalysesCommand getStudyAnalysesCommand) { + log.trace("in getStudyAnalyses, args: {} ", getStudyAnalysesCommand); + val repoBaseUrl = getStudyAnalysesCommand.getFilesRepositoryBaseUrl(); + val studyId = getStudyAnalysesCommand.getStudyId(); + val analysisListType = new ParameterizedTypeReference>() {}; + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("exception happened, retrying {}", getStudyAnalysesCommand, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> + log.error( + "exception happened, retrying {}", + getStudyAnalysesCommand, + retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDY_ANALYSES_URL_TEMPLATE, repoBaseUrl, studyId, this.indexableStudyStatuses)) - .retrieve() - .onStatus(HttpStatus.NOT_FOUND :: equals, - clientResponse -> error(notFound(MSG_STUDY_DOES_NOT_EXIST, studyId))) - .bodyToMono(analysisListType) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) - .doOnSuccess((list) -> log.trace("getStudyAnalyses out, analyses count {} args: {}", - list.size(), getStudyAnalysesCommand)); - } + return this.webClient + .get() + .uri(format(STUDY_ANALYSES_URL_TEMPLATE, repoBaseUrl, studyId, this.indexableStudyStatuses)) + .retrieve() + .onStatus( + HttpStatus.NOT_FOUND::equals, + clientResponse -> error(notFound(MSG_STUDY_DOES_NOT_EXIST, studyId))) + .bodyToMono(analysisListType) + .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) + .doOnSuccess( + (list) -> + log.trace( + "getStudyAnalyses out, analyses count {} args: {}", + list.size(), + getStudyAnalysesCommand)); + } - @Override - public Flux getStudies(@NonNull GetAllStudiesCommand getAllStudiesCommand) { - log.trace("in getStudyAnalyses, args: {} ", getAllStudiesCommand); - val repoBaseUrl = getAllStudiesCommand.getFilesRepositoryBaseUrl(); - val StringListType = new ParameterizedTypeReference>(){}; - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Flux getStudies(@NonNull GetAllStudiesCommand getAllStudiesCommand) { + log.trace("in getStudyAnalyses, args: {} ", getAllStudiesCommand); + val repoBaseUrl = getAllStudiesCommand.getFilesRepositoryBaseUrl(); + val StringListType = new ParameterizedTypeReference>() {}; + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("retrying {}", getAllStudiesCommand, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> log.error("retrying {}", getAllStudiesCommand, retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDIES_URL_TEMPLATE, repoBaseUrl)) - .accept(MediaType.APPLICATION_JSON) - .retrieve() - // we need to first parse as mono then stream it as flux because of this : - // https://github.com/spring-projects/spring-framework/issues/22662 - .bodyToMono(StringListType) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) - .flatMapMany(Flux::fromIterable) - .map(id -> Study.builder().studyId(id).build()); - } + return this.webClient + .get() + .uri(format(STUDIES_URL_TEMPLATE, repoBaseUrl)) + .accept(MediaType.APPLICATION_JSON) + .retrieve() + // we need to first parse as mono then stream it as flux because of this : + // https://github.com/spring-projects/spring-framework/issues/22662 + .bodyToMono(StringListType) + .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.studyCallTimeoutSeconds))) + .flatMapMany(Flux::fromIterable) + .map(id -> Study.builder().studyId(id).build()); + } - @Override - public Mono getAnalysis(@NonNull GetAnalysisCommand command) { - log.trace("in getAnalysis, args: {} ", command); - val repoBaseUrl = command.getFilesRepositoryBaseUrl(); - val analysisId = command.getAnalysisId(); - val studyId = command.getStudyId(); - val retryConfig = Retry.allBut(NotFoundException.class) + @Override + public Mono getAnalysis(@NonNull GetAnalysisCommand command) { + log.trace("in getAnalysis, args: {} ", command); + val repoBaseUrl = command.getFilesRepositoryBaseUrl(); + val analysisId = command.getAnalysisId(); + val studyId = command.getStudyId(); + val retryConfig = + Retry.allBut(NotFoundException.class) .retryMax(this.songMaxRetries) - .doOnRetry(retryCtx -> log.error("exception happened, retrying {}", command, retryCtx.exception())) - .exponentialBackoff(Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); + .doOnRetry( + retryCtx -> + log.error("exception happened, retrying {}", command, retryCtx.exception())) + .exponentialBackoff( + Duration.ofSeconds(minBackoffSec), Duration.ofSeconds(maxBackoffSec)); - return this.webClient.get() - .uri(format(STUDY_ANALYSIS_URL_TEMPLATE, repoBaseUrl, studyId, analysisId)) - .retrieve() - .onStatus(HttpStatus.NOT_FOUND::equals, - clientResponse -> error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl))) - .bodyToMono(Analysis.class) - .transform(retryAndTimeout(retryConfig, Duration.ofSeconds(this.analysisCallTimeoutSeconds))) - .doOnSuccess((analysis) -> log.trace("getAnalysis out, analysis {} args: {}", - analysis, command)) - .flatMap(analysis -> { - if (!indexableStudyStatusesList.contains(analysis.getAnalysisState())) { - return error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl)); - } - return Mono.just(analysis); + return this.webClient + .get() + .uri(format(STUDY_ANALYSIS_URL_TEMPLATE, repoBaseUrl, studyId, analysisId)) + .retrieve() + .onStatus( + HttpStatus.NOT_FOUND::equals, + clientResponse -> + error(notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl))) + .bodyToMono(Analysis.class) + .transform( + retryAndTimeout(retryConfig, Duration.ofSeconds(this.analysisCallTimeoutSeconds))) + .doOnSuccess( + (analysis) -> log.trace("getAnalysis out, analysis {} args: {}", analysis, command)) + .flatMap( + analysis -> { + if (!indexableStudyStatusesList.contains(analysis.getAnalysisState())) { + return error( + notFound(MSG_ANALYSIS_DOES_NOT_EXIST, analysisId, studyId, repoBaseUrl)); + } + return Mono.just(analysis); }); - } + } - private Function, Mono> retryAndTimeout(Retry retry, Duration timeout) { - // order is important here timeouts should be retried. - return (in) -> in.timeout(timeout).retryWhen(retry); - } + private Function, Mono> retryAndTimeout(Retry retry, Duration timeout) { + // order is important here timeouts should be retried. + return (in) -> in.timeout(timeout).retryWhen(retry); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java index 709a83ea..49971ebe 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/FileBasedFailuresLogger.java @@ -17,38 +17,34 @@ package bio.overture.maestro.app.infra.adapter.outbound.notification; +import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.FAILURE_LOG_PROP_KEY; + import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import reactor.core.publisher.Mono; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.FAILURE_LOG_PROP_KEY; - /** - * This channel will store any failure in an append only log files - * it uses logback loggers to do the write operation instead of manually - * writing to files. + * This channel will store any failure in an append only log files it uses logback loggers to do the + * write operation instead of manually writing to files. * - * the logs go to separate log files. see logback-spring.xml for the configs. + *

the logs go to separate log files. see logback-spring.xml for the configs. */ @Slf4j @ConditionalOnProperty(value = FAILURE_LOG_PROP_KEY, havingValue = "true") public class FileBasedFailuresLogger implements NotificationChannel { - @Override - public Mono send(IndexerNotification notification) { - log.error("{}", notification); - return Mono.just(true); - } + @Override + public Mono send(IndexerNotification notification) { + log.error("{}", notification); + return Mono.just(true); + } - @Override - public Set subscriptions() { - return Set.of( - NotificationName.ALL - ); - } + @Override + public Set subscriptions() { + return Set.of(NotificationName.ALL); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java index d008ff64..86d53207 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/LoggingNotificationChannel.java @@ -20,50 +20,45 @@ import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.springframework.core.annotation.Order; import reactor.core.publisher.Mono; -import java.util.Set; - - /** - * This channel is to log the failures to the standard logger (console). - * it's different than the FileBasedFailuresLogger in the fact that this is - * not persistent or for failures review. + * This channel is to log the failures to the standard logger (console). it's different than the + * FileBasedFailuresLogger in the fact that this is not persistent or for failures review. */ @Slf4j @Order(1) public class LoggingNotificationChannel implements NotificationChannel { - private static final int MAX_NOTIFICATION_STRING_LENGTH = 1024; - - @Override - public Mono send(IndexerNotification notification) { - val notificationString = notification.toString(); - val notificationStringTruncated = notificationString.substring(0, - Math.min(notificationString.length(), MAX_NOTIFICATION_STRING_LENGTH)); + private static final int MAX_NOTIFICATION_STRING_LENGTH = 1024; - switch (notification.getNotificationName().getCategory()) { - case ERROR: - log.error("{}", notificationStringTruncated); - break; - case WARN: - log.warn("{}", notificationStringTruncated); - break; - default: - log.info("{}", notificationStringTruncated); - } + @Override + public Mono send(IndexerNotification notification) { + val notificationString = notification.toString(); + val notificationStringTruncated = + notificationString.substring( + 0, Math.min(notificationString.length(), MAX_NOTIFICATION_STRING_LENGTH)); - return Mono.just(true); + switch (notification.getNotificationName().getCategory()) { + case ERROR: + log.error("{}", notificationStringTruncated); + break; + case WARN: + log.warn("{}", notificationStringTruncated); + break; + default: + log.info("{}", notificationStringTruncated); } - @Override - public Set subscriptions() { - return Set.of( - NotificationName.ALL - ); - } + return Mono.just(true); + } + @Override + public Set subscriptions() { + return Set.of(NotificationName.ALL); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java index 8ee98e81..8cda3b67 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/NotificationConfig.java @@ -22,8 +22,8 @@ @Configuration @Import({ - LoggingNotificationChannel.class, - FileBasedFailuresLogger.class, - Slack.class, + LoggingNotificationChannel.class, + FileBasedFailuresLogger.class, + Slack.class, }) public class NotificationConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java index f58ef45e..7f69f95c 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/Slack.java @@ -17,10 +17,16 @@ package bio.overture.maestro.app.infra.adapter.outbound.notification; +import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.NotificationChannel; import bio.overture.maestro.domain.api.NotificationName; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.time.Duration; +import java.util.Map; +import java.util.Set; +import javax.inject.Inject; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -30,79 +36,85 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; -import javax.inject.Inject; -import java.time.Duration; -import java.util.Map; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.properties.ApplicationProperties.MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY; - @Slf4j @ConditionalOnProperty(value = MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY, havingValue = "true") public class Slack implements NotificationChannel { - private static final String TYPE = "##TYPE##"; - private static final String DATA = "##DATA##"; - private final WebClient webClient; - private final SlackChannelInfo slackChannelInfo; + private static final String TYPE = "##TYPE##"; + private static final String DATA = "##DATA##"; + private final WebClient webClient; + private final SlackChannelInfo slackChannelInfo; - @Inject - public Slack(ApplicationProperties properties, WebClient webClient) { - this.webClient = webClient; - this.slackChannelInfo = properties.getSlackChannelInfo(); + @Inject + public Slack(ApplicationProperties properties, WebClient webClient) { + this.webClient = webClient; + this.slackChannelInfo = properties.getSlackChannelInfo(); + } + + @Override + public Mono send(@NonNull IndexerNotification notification) { + var template = this.slackChannelInfo.infoTemplate(); + switch (notification.getNotificationName().getCategory()) { + case ERROR: + template = this.slackChannelInfo.errorTemplate(); + break; + case WARN: + template = this.slackChannelInfo.warningTemplate(); + break; } - @Override - public Mono send(@NonNull IndexerNotification notification) { - var template = this.slackChannelInfo.infoTemplate(); - switch (notification.getNotificationName().getCategory()) { - case ERROR: - template = this.slackChannelInfo.errorTemplate(); break; - case WARN: - template = this.slackChannelInfo.warningTemplate(); break; - } - - val dataString = notification.getAttributes().toString(); - val dataStringTruncated = dataString.substring(0, - Math.min(dataString.length(), this.slackChannelInfo.maxDataLength())); - - val text = template. - replace(TYPE, notification.getNotificationName().name()) + val dataString = notification.getAttributes().toString(); + val dataStringTruncated = + dataString.substring( + 0, Math.min(dataString.length(), this.slackChannelInfo.maxDataLength())); + + val text = + template + .replace(TYPE, notification.getNotificationName().name()) .replace(DATA, dataStringTruncated); - val payload = Map.of( + val payload = + Map.of( "text", text, "channel", this.slackChannelInfo.channel(), - "username", this.slackChannelInfo.username() - ); - - return this.webClient.post() - .uri(this.slackChannelInfo.url()) - .contentType(MediaType.APPLICATION_JSON) - .body(BodyInserters.fromObject(payload)) - .retrieve() - .bodyToMono(String.class) - .timeout(Duration.ofSeconds(3L)) - .map((ignored) -> true) - .onErrorResume(e -> { - log.error("failed to send message to slack", e); - return Mono.just(false); + "username", this.slackChannelInfo.username()); + + return this.webClient + .post() + .uri(this.slackChannelInfo.url()) + .contentType(MediaType.APPLICATION_JSON) + .body(BodyInserters.fromObject(payload)) + .retrieve() + .bodyToMono(String.class) + .timeout(Duration.ofSeconds(3L)) + .map((ignored) -> true) + .onErrorResume( + e -> { + log.error("failed to send message to slack", e); + return Mono.just(false); }); - } + } - @Override - public Set subscriptions() { - return Set.copyOf(slackChannelInfo.subscriptions()); - } + @Override + public Set subscriptions() { + return Set.copyOf(slackChannelInfo.subscriptions()); + } - public interface SlackChannelInfo { - String url(); - String channel(); - String username(); - String errorTemplate(); - String warningTemplate(); - String infoTemplate(); - int maxDataLength(); - Set subscriptions(); - } + public interface SlackChannelInfo { + String url(); + + String channel(); + + String username(); + + String errorTemplate(); + + String warningTemplate(); + + String infoTemplate(); + + int maxDataLength(); + + Set subscriptions(); + } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java index 9cc5b66f..beaf83f9 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/adapter/outbound/notification/SlackInfo.java @@ -16,5 +16,3 @@ */ package bio.overture.maestro.app.infra.adapter.outbound.notification; - - diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java index 52983476..7b36b823 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/RootConfiguration.java @@ -35,70 +35,66 @@ import org.springframework.context.annotation.Primary; import org.springframework.web.reactive.function.client.WebClient; -/** - * Aggregates all configuration in one place - */ +/** Aggregates all configuration in one place */ @Configuration @Import({ - DomainApiConfig.class, - PortsConfig.class, - InfraConfig.class, + DomainApiConfig.class, + PortsConfig.class, + InfraConfig.class, }) public class RootConfiguration { - public final static String ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER = "documentObjectMapper"; + public static final String ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER = "documentObjectMapper"; } /** - * Configuration about domain related beans (ports implementations), delegated here to keep the domain module agnostic of injection framework . + * Configuration about domain related beans (ports implementations), delegated here to keep the + * domain module agnostic of injection framework . */ @Configuration @Import({ - ElasticSearchConfig.class, - ExclusionRulesConfig.class, - MessagingConfig.class, - WebConfig.class, - SongConfig.class, - RepositoryConfig.class, - NotificationConfig.class, + ElasticSearchConfig.class, + ExclusionRulesConfig.class, + MessagingConfig.class, + WebConfig.class, + SongConfig.class, + RepositoryConfig.class, + NotificationConfig.class, }) class PortsConfig {} /** - * Aggregator for all configurations related to - * the infrastructure beans (I/O & networking, properties, datasources, etc) + * Aggregator for all configurations related to the infrastructure beans (I/O & networking, + * properties, datasources, etc) */ @Configuration @Import({ - PropertiesConfig.class, + PropertiesConfig.class, }) class InfraConfig { - @Bean - WebClient webClient(ApplicationProperties properties) { - return WebClient.builder().codecs( - c -> c.defaultCodecs().maxInMemorySize(properties.webClientMaxInMemorySize())).build(); - } + @Bean + WebClient webClient(ApplicationProperties properties) { + return WebClient.builder() + .codecs(c -> c.defaultCodecs().maxInMemorySize(properties.webClientMaxInMemorySize())) + .build(); + } } -/** - * Configuration related to the indexer web api - */ +/** Configuration related to the indexer web api */ @Configuration @Import({ - GlobalWebExceptionHandler.class, - ManagementController.class, + GlobalWebExceptionHandler.class, + ManagementController.class, }) class WebConfig { - private static final String DEFAULT_DOCUMENT_JSON_MAPPER = "DEFAULT_DOCUMENT_JSON_MAPPER" ; + private static final String DEFAULT_DOCUMENT_JSON_MAPPER = "DEFAULT_DOCUMENT_JSON_MAPPER"; - /** - * This bean is needed for spring webflux to not use the ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER - * marked as primary so by default callers who don't specify which bean they need, will get this. - */ - @Primary - @Bean(name = DEFAULT_DOCUMENT_JSON_MAPPER) - public ObjectMapper objectMapper() { - return new ObjectMapper(); - } + /** + * This bean is needed for spring webflux to not use the ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER + * marked as primary so by default callers who don't specify which bean they need, will get this. + */ + @Primary + @Bean(name = DEFAULT_DOCUMENT_JSON_MAPPER) + public ObjectMapper objectMapper() { + return new ObjectMapper(); + } } - - diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java index 0ae609ff..0e9e397d 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/ApplicationProperties.java @@ -27,58 +27,58 @@ * the original properties */ public interface ApplicationProperties { - String FAILURE_LOG_PROP_KEY = "maestro.failureLog.enabled"; - String MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY = "maestro.notifications.slack.enabled"; + String FAILURE_LOG_PROP_KEY = "maestro.failureLog.enabled"; + String MAESTRO_NOTIFICATIONS_SLACK_ENABLED_PROP_KEY = "maestro.notifications.slack.enabled"; - int webClientMaxInMemorySize(); + int webClientMaxInMemorySize(); - List elasticSearchClusterNodes(); + List elasticSearchClusterNodes(); - String fileCentricAlias(); + String fileCentricAlias(); - String fileCentricIndexName(); + String fileCentricIndexName(); - boolean isFileCentricIndexEnabled(); + boolean isFileCentricIndexEnabled(); - String analysisCentricAlias(); + String analysisCentricAlias(); - String analysisCentricIndexName(); + String analysisCentricIndexName(); - boolean isAnalysisCentricIndexEnabled(); + boolean isAnalysisCentricIndexEnabled(); - int maxDocsPerBulkRequest(); + int maxDocsPerBulkRequest(); - int elasticSearchClientConnectionTimeoutMillis(); + int elasticSearchClientConnectionTimeoutMillis(); - int elasticSearchClientSocketTimeoutMillis(); + int elasticSearchClientSocketTimeoutMillis(); - boolean elasticSearchTlsTrustSelfSigned(); + boolean elasticSearchTlsTrustSelfSigned(); - boolean elasticSearchBasicAuthEnabled(); + boolean elasticSearchBasicAuthEnabled(); - String elasticSearchAuthUser(); + String elasticSearchAuthUser(); - String elasticSearchAuthPassword(); + String elasticSearchAuthPassword(); - List repositories(); + List repositories(); - Resource fileCentricIndex(); + Resource fileCentricIndex(); - Resource analysisCentricIndex(); + Resource analysisCentricIndex(); - Map> idExclusionRules(); + Map> idExclusionRules(); - int songMaxRetries(); + int songMaxRetries(); - int songStudyCallTimeoutSeconds(); + int songStudyCallTimeoutSeconds(); - long elasticSearchRetryWaitDurationMillis(); + long elasticSearchRetryWaitDurationMillis(); - int elasticSearchRetryMaxAttempts(); + int elasticSearchRetryMaxAttempts(); - String indexableStudyStatuses(); + String indexableStudyStatuses(); - int songAnalysisCallTimeoutSeconds(); + int songAnalysisCallTimeoutSeconds(); - Slack.SlackChannelInfo getSlackChannelInfo(); + Slack.SlackChannelInfo getSlackChannelInfo(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java index 4d34e43e..278f621f 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesConfig.java @@ -22,7 +22,6 @@ @Configuration @Import({ - DefaultApplicationProperties.class, + DefaultApplicationProperties.class, }) -public class PropertiesConfig { -} +public class PropertiesConfig {} diff --git a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java index b444ce1e..927b49ba 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java +++ b/maestro-app/src/main/java/bio/overture/maestro/app/infra/config/properties/PropertiesFileRepository.java @@ -18,19 +18,19 @@ package bio.overture.maestro.app.infra.config.properties; public interface PropertiesFileRepository { - String getName(); + String getName(); - String getCode(); + String getCode(); - String getUrl(); + String getUrl(); - String getDataPath(); + String getDataPath(); - String getMetadataPath(); + String getMetadataPath(); - String getOrganization(); + String getOrganization(); - String getCountry(); + String getCountry(); - bio.overture.maestro.domain.entities.indexing.StorageType getStorageType(); + bio.overture.maestro.domain.entities.indexing.StorageType getStorageType(); } diff --git a/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java b/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java index b5ff6a09..f17501d9 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java +++ b/maestro-app/src/main/java/bio/overture/maestro/domain/api/DomainApiConfig.java @@ -25,17 +25,16 @@ @Configuration @Import({ - DefaultIndexer.class, - Notifier.class, + DefaultIndexer.class, + Notifier.class, }) public class DomainApiConfig { @Bean IndexEnabled indexEnabled(@Autowired ApplicationProperties applicationProperties) { - return new IndexEnabled - .IndexEnabledBuilder() - .isAnalysisCentricEnabled(applicationProperties.isAnalysisCentricIndexEnabled()) - .isFileCentricEnabled(applicationProperties.isFileCentricIndexEnabled()) - .build(); + return new IndexEnabled.IndexEnabledBuilder() + .isAnalysisCentricEnabled(applicationProperties.isAnalysisCentricIndexEnabled()) + .isFileCentricEnabled(applicationProperties.isFileCentricIndexEnabled()) + .build(); } } diff --git a/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java b/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java index 225ad47b..b26de2c7 100644 --- a/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java +++ b/maestro-app/src/main/java/bio/overture/maestro/domain/api/package-info.java @@ -19,4 +19,4 @@ /* * This package is here to define the package hidden beans from maestro domain module * as spring beans without making the classes public to avoid misusing them. - */ \ No newline at end of file + */ diff --git a/maestro-app/src/main/resources/analysis_centric.json b/maestro-app/src/main/resources/analysis_centric.json index ac90b27b..b81b5fad 100644 --- a/maestro-app/src/main/resources/analysis_centric.json +++ b/maestro-app/src/main/resources/analysis_centric.json @@ -88,7 +88,7 @@ "donors": { "type": "nested", "properties": { - "id": { + "donor_id": { "type": "keyword", "copy_to": [ "analysis_autocomplete" @@ -106,7 +106,7 @@ "specimens": { "type": "nested", "properties": { - "id": { + "specimen_id": { "type": "keyword", "copy_to": [ "analysis_autocomplete" @@ -130,7 +130,7 @@ "samples": { "type": "nested", "properties": { - "id": { + "sample_id": { "type": "keyword", "copy_to": [ "analysis_autocomplete" @@ -160,7 +160,7 @@ "files": { "type": "nested", "properties": { - "id": { + "object_id": { "type": "keyword", "copy_to": [ "analysis_autocomplete" diff --git a/maestro-app/src/main/resources/config/application.yml b/maestro-app/src/main/resources/config/application.yml index 1bedb20b..e696faa2 100644 --- a/maestro-app/src/main/resources/config/application.yml +++ b/maestro-app/src/main/resources/config/application.yml @@ -6,6 +6,7 @@ maestro: # -1 = unlimited memory size maxInMemorySize: -1 song: + indexableStudyStatus: PUBLISHED maxRetries: 3 timeoutSec: study: 100 # some studies take really long, +30 secs, to be downloaded @@ -54,8 +55,6 @@ maestro: - code: song.overture # must be unique & must match song.serverId if using kafka integration with song url: https://song.domain.com # Change this to a valid domain where the song exists in your setup name: local song - dataPath: /oicr.icgc/data - metadataPath: /oicr.icgc.meta/metadata # optional storageType: S3 organization: ICGC @@ -64,7 +63,6 @@ maestro: - code: song.overture1 url: http://localhost:8080 name: local song - metadataPath: /oicr.icgc.meta/metadata # optional storageType: S3 organization: overture diff --git a/maestro-app/src/main/resources/file_centric.json b/maestro-app/src/main/resources/file_centric.json index 0acf9628..618afcf1 100644 --- a/maestro-app/src/main/resources/file_centric.json +++ b/maestro-app/src/main/resources/file_centric.json @@ -3,8 +3,8 @@ "file_centric": {} }, "settings": { - "index.number_of_shards" : 11, - "index.max_result_window":300000, + "index.number_of_shards": 11, + "index.max_result_window": 300000, "analysis": { "analyzer": { "autocomplete_analyzed": { @@ -40,7 +40,7 @@ }, "mappings": { "dynamic": false, - "date_detection":false, + "date_detection": false, "properties": { "file_autocomplete": { "fields": { @@ -67,11 +67,9 @@ "file_autocomplete" ] }, - "dataset": { + "data_type": { "type": "keyword", - "copy_to": [ - "file_autocomplete" - ] + "copy_to": ["file_autocomplete"] }, "object_id": { "type": "keyword", @@ -85,14 +83,17 @@ "file_access": { "type": "keyword" }, - "analysis" : { - "properties" : { - "analysis_id" : { + "analysis": { + "properties": { + "analysis_id": { "type": "keyword", "copy_to": [ "file_autocomplete" ] }, + "analysis_state": { + "type": "keyword" + }, "analysis_type": { "type": "keyword" }, @@ -104,17 +105,8 @@ } } }, - "data_type": { - "type": "keyword" - }, - "files": { + "file": { "properties": { - "id": { - "type": "keyword" - }, - "format": { - "type": "keyword" - }, "md5sum": { "type": "keyword" }, @@ -127,34 +119,12 @@ "size": { "type": "long" }, - "last_modified": { - "type": "long" - }, - "auxiliary_file": { - "properties": { - "id": { - "type": "keyword" - }, - "format": { - "type": "keyword" - }, - "md5sum": { - "type": "keyword" - }, - "name": { - "type": "keyword" - }, - "size": { - "type": "long" - } - } - }, "index_file": { "properties": { "object_id": { "type": "keyword" }, - "format": { + "file_type": { "type": "keyword" }, "md5sum": { @@ -173,7 +143,7 @@ "donors": { "type": "nested", "properties": { - "id": { + "donor_id": { "type": "keyword", "copy_to": [ "file_autocomplete" @@ -191,7 +161,7 @@ "specimens": { "type": "nested", "properties": { - "id": { + "specimen_id": { "type": "keyword", "copy_to": [ "file_autocomplete" @@ -215,7 +185,7 @@ "samples": { "type": "nested", "properties": { - "id": { + "sample_id": { "type": "keyword", "copy_to": [ "file_autocomplete" diff --git a/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java b/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java index 046f62ea..9a9449ff 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/MaestroIntegrationTest.java @@ -34,41 +34,39 @@ import org.springframework.cloud.contract.wiremock.AutoConfigureWireMock; import org.springframework.test.context.ContextConfiguration; - /** - * Base class for full integration tests. - * it will create an elasticsearch container (using test containers). + * Base class for full integration tests. it will create an elasticsearch container (using test + * containers). * - * if you need a more light weight faster tests, avoid using this class, this is meant for full end to end. + *

if you need a more light weight faster tests, avoid using this class, this is meant for full + * end to end. */ @Slf4j @Tag(TestCategory.INT_TEST) @ContextConfiguration(classes = {Maestro.class}) @AutoConfigureWireMock(port = 0) // we bind the port in the application.yml ${wiremock.server.port} -@SpringBootTest(classes = {Maestro.class}, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { - "embedded.elasticsearch.enabled=true" -}) +@SpringBootTest( + classes = {Maestro.class}, + webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, + properties = {"embedded.elasticsearch.enabled=true"}) public abstract class MaestroIntegrationTest { - /** wait time for elastic search to update */ - @Value("${maestro.test.elasticsearch.sleep_millis:2500}") - protected int sleepMillis; - - @Autowired - private ApplicationProperties properties; + /** wait time for elastic search to update */ + @Value("${maestro.test.elasticsearch.sleep_millis:2500}") + protected int sleepMillis; - @Autowired - private RestHighLevelClient client; + @Autowired private ApplicationProperties properties; - @AfterEach - @SneakyThrows - void tearDown() { - // clean indexes after each test to keep tests isolated - DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(); - deleteByQueryRequest.indices(properties.fileCentricAlias()); - deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); - client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); - Thread.sleep(sleepMillis); - } + @Autowired private RestHighLevelClient client; -} \ No newline at end of file + @AfterEach + @SneakyThrows + void tearDown() { + // clean indexes after each test to keep tests isolated + DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(); + deleteByQueryRequest.indices(properties.fileCentricAlias()); + deleteByQueryRequest.setQuery(QueryBuilders.matchAllQuery()); + client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); + Thread.sleep(sleepMillis); + } +} diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java index 5d6aa58d..d6af94ee 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/inbound/messaging/song/SongAnalysisStreamListenerTest.java @@ -17,7 +17,13 @@ package bio.overture.maestro.app.infra.adapter.inbound.messaging.song; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.*; + import bio.overture.maestro.app.infra.adapter.inbound.messaging.MessagingConfig; +import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.api.Indexer; import bio.overture.maestro.domain.api.message.AnalysisIdentifier; import bio.overture.maestro.domain.api.message.IndexAnalysisCommand; @@ -29,16 +35,12 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.messaging.support.GenericMessage; +import org.springframework.test.context.ContextConfiguration; import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.when; /* * This test is based on : https://docs.spring.io/spring-cloud-stream/docs/current/reference/htmlsingle/#_testing @@ -47,56 +49,96 @@ * However this test does serve the purpose of testing the listener */ @SpringBootTest(classes = {SongAnalysisStreamListenerTest.MockApplication.class}) +@ContextConfiguration(classes = {SongAnalysisStreamListenerTest.Config.class}) class SongAnalysisStreamListenerTest { - @MockBean - private Indexer indexer; + @Autowired SongAnalysisSink sink; + @MockBean private Indexer indexer; - @Autowired - SongAnalysisSink sink; + @Test + void shouldIndexOnAnalysisPublishedMessage() throws Exception { + val analysisPublishedMessage = + "{ \"analysisId\" : \"EGAZ00001254368\", \"studyId\" : \"PEME-CA\", " + + "\"songServerId\": \"collab\", \"state\": \"PUBLISHED\" }"; + when(indexer.indexAnalysis(any())) + .thenReturn(Flux.just(IndexResult.builder().successful(true).build())); + sink.songInput().send(new GenericMessage<>(analysisPublishedMessage)); + Thread.sleep(2000); + then(indexer) + .should(times(1)) + .indexAnalysis( + eq( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId("PEME-CA") + .analysisId("EGAZ00001254368") + .repositoryCode("collab") + .build()) + .build())); + } - @Test - void shouldIndexOnAnalysisPublishedMessage() throws Exception { - val analysisPublishedMessage = "{ \"analysisId\" : \"EGAZ00001254368\", \"studyId\" : \"PEME-CA\", " + - "\"songServerId\": \"collab\", \"state\": \"PUBLISHED\" }"; - when(indexer.indexAnalysis(any())).thenReturn(Flux.just(IndexResult.builder().successful(true).build())); - sink.songInput().send(new GenericMessage<>(analysisPublishedMessage)); - Thread.sleep(2000); - then(indexer).should(times(1)) - .indexAnalysis(eq(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId("PEME-CA") - .analysisId("EGAZ00001254368") - .repositoryCode("collab") - .build()) - .build() - ) - ); - } + @Test + void shouldAddOnAnalysisSuppressedMessage() throws Exception { + val analysisSuppressedMessage = + "{ \"analysisId\" : \"EGAZ00001254368\", \"studyId\" : \"PEME-CA\", " + + "\"songServerId\": \"collab\", \"state\": \"SUPPRESSED\" }"; + when(indexer.indexAnalysis(any())) + .thenReturn(Flux.just(IndexResult.builder().successful(true).build())); + sink.songInput().send(new GenericMessage<>(analysisSuppressedMessage)); + Thread.sleep(2000); + then(indexer) + .should(times(1)) + .indexAnalysis( + eq( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId("PEME-CA") + .analysisId("EGAZ00001254368") + .repositoryCode("collab") + .build()) + .build())); + } - @Test - void shouldRemoveOnAnalysisSuppressedMessage() throws Exception { - val analysisPublishedMessage = "{ \"analysisId\" : \"EGAZ00001254368\", \"studyId\" : \"PEME-CA\", " + - "\"songServerId\": \"collab\", \"state\": \"SUPPRESSED\" }"; - when(indexer.removeAnalysis(any())).thenReturn(Mono.just(IndexResult.builder().successful(true).build())); - sink.songInput().send(new GenericMessage<>(analysisPublishedMessage)); - Thread.sleep(2000); - then(indexer).should(times(1)).removeAnalysis(eq(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId("PEME-CA") - .analysisId("EGAZ00001254368") - .repositoryCode("collab") - .build()) - .build() - ) - ); - } + @Test + void shouldRemoveOnAnalysisUnpublishedMessage() throws Exception { + val analysisPublishedMessage = + "{ \"analysisId\" : \"EGAZ00001254368\", \"studyId\" : \"PEME-CA\", " + + "\"songServerId\": \"collab\", \"state\": \"UNPUBLISHED\" }"; + when(indexer.removeAnalysis(any())) + .thenReturn(Flux.just(IndexResult.builder().successful(true).build())); + sink.songInput().send(new GenericMessage<>(analysisPublishedMessage)); + Thread.sleep(2000); + then(indexer) + .should(times(1)) + .removeAnalysis( + eq( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId("PEME-CA") + .analysisId("EGAZ00001254368") + .repositoryCode("collab") + .build()) + .build())); + } + + /* + * This is needed or you will get : + * org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers + */ + @SpringBootApplication + @Import({MessagingConfig.class}) + static class MockApplication {} - /* - * This is needed or you will get : - * org.springframework.integration.MessageDispatchingException: Dispatcher has no subscribers - */ - @SpringBootApplication - @Import({MessagingConfig.class}) - static class MockApplication { } -} \ No newline at end of file + @Configuration + static class Config { + @Bean + ApplicationProperties properties() { + ApplicationProperties properties = mock(ApplicationProperties.class); + when(properties.indexableStudyStatuses()).thenReturn("PUBLISHED,SUPPRESSED"); + return properties; + } + } +} diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java index 29120e2f..8ec2568c 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/indexing/elasticsearch/FileCentricElasticSearchAdapterUnavaibilityTest.java @@ -17,6 +17,11 @@ package bio.overture.maestro.app.infra.adapter.outbound.indexing.elasticsearch; +import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + import bio.overture.maestro.app.infra.config.properties.PropertiesConfig; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.message.IndexResult; @@ -26,6 +31,10 @@ import bio.overture.masestro.test.TestCategory; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.http.HttpHost; @@ -45,86 +54,71 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.test.StepVerifier; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; -import java.util.Set; - -import static bio.overture.maestro.app.infra.config.RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - @Slf4j @Tag(TestCategory.INT_TEST) -@SpringBootTest(properties = { - "embedded.elasticsearch.enabled=false" -}) +@SpringBootTest(properties = {"embedded.elasticsearch.enabled=false"}) @ContextConfiguration(classes = {FileCentricElasticSearchAdapterUnavaibilityTest.Config.class}) class FileCentricElasticSearchAdapterUnavaibilityTest { - @SpyBean - private RestHighLevelClient client; + @SpyBean private RestHighLevelClient client; - @Autowired - private FileCentricElasticSearchAdapter adapter; + @Autowired private FileCentricElasticSearchAdapter adapter; - @Test - void shouldRetryUpsertOnIOException() throws IOException { - // given - val files = Arrays.asList(Fixture.loadJsonFixtureSnakeCase( - this.getClass(), "PEME-CA.files.json", FileCentricDocument[].class)); + @Test + void shouldRetryUpsertOnIOException() throws IOException { + // given + val files = + Arrays.asList( + Fixture.loadJsonFixtureSnakeCase( + this.getClass(), "PEME-CA.files.json", FileCentricDocument[].class)); - val expectedResult = IndexResult.builder() - .indexName("file_centric") - .failureData( - FailureData.builder() + val expectedResult = + IndexResult.builder() + .indexName("file_centric") + .failureData( + FailureData.builder() .failingIds(Map.of("analysisId", Set.of("EGAZ00001254368"))) - .build() - ).successful(false) - .build(); - - // when - val result = adapter.batchUpsertFileRepositories(BatchIndexFilesCommand.builder() - .files(files) - .build()); - - //then - StepVerifier.create(result) - .expectNext(expectedResult) - .verifyComplete(); - - // since this is a final method I had to add the mockito-extensions directory to test resources - // see why.md there for more info. - verify(client, times(3)).bulk(any(BulkRequest.class), any(RequestOptions.class)); + .build()) + .successful(false) + .build(); + + // when + val result = + adapter.batchUpsertFileRepositories(BatchIndexFilesCommand.builder().files(files).build()); + + // then + StepVerifier.create(result).expectNext(expectedResult).verifyComplete(); + + // since this is a final method I had to add the mockito-extensions directory to test resources + // see why.md there for more info. + verify(client, times(3)).bulk(any(BulkRequest.class), any(RequestOptions.class)); + } + + @Import({ + FileCentricElasticSearchAdapter.class, + SnakeCaseJacksonSearchResultMapper.class, + PropertiesConfig.class + }) + @Configuration + static class Config { + + @Bean + WebClient webClient() { + return WebClient.builder().build(); } - @Import({ - FileCentricElasticSearchAdapter.class, - SnakeCaseJacksonSearchResultMapper.class, - PropertiesConfig.class - }) - @Configuration - static class Config { - - @Bean - WebClient webClient() { - return WebClient.builder().build(); - } - - @Bean - RestHighLevelClient mockClient() { - //this will trigger an IO exception - val restClient = RestClient.builder(new HttpHost("nonexisting", 9201, "http")); - return new RestHighLevelClient(restClient); - } - - @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - ObjectMapper documentObjectMapper() { - val mapper = new ObjectMapper(); - mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return mapper; - } + @Bean + RestHighLevelClient mockClient() { + // this will trigger an IO exception + val restClient = RestClient.builder(new HttpHost("nonexisting", 9201, "http")); + return new RestHighLevelClient(restClient); + } + @Bean(name = ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + ObjectMapper documentObjectMapper() { + val mapper = new ObjectMapper(); + mapper.setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return mapper; } -} \ No newline at end of file + } +} diff --git a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java index c53f2c79..38f854f6 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/app/infra/adapter/outbound/metadata/study/song/SongStudyDAOTest.java @@ -17,6 +17,13 @@ package bio.overture.maestro.app.infra.adapter.outbound.metadata.study.song; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonString; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + import bio.overture.maestro.app.infra.config.properties.ApplicationProperties; import bio.overture.maestro.domain.entities.metadata.study.Analysis; import bio.overture.maestro.domain.port.outbound.metadata.study.GetAnalysisCommand; @@ -25,6 +32,7 @@ import bio.overture.masestro.test.TestCategory; import com.fasterxml.jackson.core.type.TypeReference; import com.github.tomakehurst.wiremock.stubbing.Scenario; +import java.util.List; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -42,189 +50,164 @@ import reactor.retry.RetryExhaustedException; import reactor.test.StepVerifier; -import java.util.List; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonString; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - @Slf4j @Tag(TestCategory.INT_TEST) -@SpringBootTest(properties = { - "embedded.elasticsearch.enabled=false" -}) +@SpringBootTest(properties = {"embedded.elasticsearch.enabled=false"}) @ContextConfiguration(classes = {SongStudyDAOTest.Config.class}) @AutoConfigureWireMock(port = 0) class SongStudyDAOTest { - @Value("${wiremock.server.port}") - private int wiremockPort; + @Value("${wiremock.server.port}") + private int wiremockPort; - @Autowired - private StudyDAO songStudyDAO; + @Autowired private StudyDAO songStudyDAO; - @Test - void fetchingAnalysisShouldReturnMonoErrorIfRetriesExhausted() { + @Test + void fetchingAnalysisShouldReturnMonoErrorIfRetriesExhausted() { - //given - val analysisId = "EGAZ00001254247"; - val command = GetAnalysisCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) + // given + val analysisId = "EGAZ00001254247"; + val command = + GetAnalysisCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) .studyId("PEME-CA") .analysisId(analysisId) .build(); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .willReturn(aResponse() + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .willReturn( + aResponse() .withStatus(400) .withBody("

Some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - ); - - //when - val analysesMono = songStudyDAO.getAnalysis(command); - - //then - StepVerifier.create(analysesMono) - .expectErrorSatisfies((e) -> { - assertTrue(e instanceof RetryExhaustedException); - }).verify(); - } - - @Test - @SneakyThrows - void shouldRetryFetchingAnalysisOnFailure() { - val analysis = loadJsonString(this.getClass(), - "PEME-CA.analysis.json"); - val analysisObj = loadJsonFixture(this.getClass(), - "PEME-CA.analysis.json", Analysis.class); - val analysisId = "EGAZ00001254247"; - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse() + .withHeader("content-type", "text/html"))); + + // when + val analysesMono = songStudyDAO.getAnalysis(command); + + // then + StepVerifier.create(analysesMono) + .expectErrorSatisfies( + (e) -> { + assertTrue(e instanceof RetryExhaustedException); + }) + .verify(); + } + + @Test + @SneakyThrows + void shouldRetryFetchingAnalysisOnFailure() { + val analysis = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); + val analysisObj = loadJsonFixture(this.getClass(), "PEME-CA.analysis.json", Analysis.class); + val analysisId = "EGAZ00001254247"; + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() .withStatus(400) .withBody("

some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - .willSetStateTo("WORKING") - ); - - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs("WORKING") - .willReturn(aResponse() + .withHeader("content-type", "text/html")) + .willSetStateTo("WORKING")); + + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis/" + analysisId)) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs("WORKING") + .willReturn( + aResponse() .withBody(analysis) .withStatus(200) - .withHeader("content-type", "application/json") - ) - ); - - val analysesMono = songStudyDAO.getAnalysis(GetAnalysisCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) - .studyId("PEME-CA") - .analysisId(analysisId) - .build() - ); - - StepVerifier.create(analysesMono) - .expectNext(analysisObj) - .verifyComplete(); - - } - - @Test - @SneakyThrows - void shouldRetryFetchingStudyAnalysesOnFailure() { - val analyses = loadJsonString(this.getClass(), - "PEME-CA.study.json"); - val analysesList = loadJsonFixture(this.getClass(), - "PEME-CA.study.json", new TypeReference>() {}); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs(Scenario.STARTED) - .willReturn(aResponse() + .withHeader("content-type", "application/json"))); + + val analysesMono = + songStudyDAO.getAnalysis( + GetAnalysisCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) + .studyId("PEME-CA") + .analysisId(analysisId) + .build()); + + StepVerifier.create(analysesMono).expectNext(analysisObj).verifyComplete(); + } + + @Test + @SneakyThrows + void shouldRetryFetchingStudyAnalysesOnFailure() { + val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + val analysesList = + loadJsonFixture( + this.getClass(), "PEME-CA.study.json", new TypeReference>() {}); + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs(Scenario.STARTED) + .willReturn( + aResponse() .withStatus(400) .withBody("

some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - .willSetStateTo("WORKING") - ); - - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .inScenario("RANDOM_FAILURE") - .whenScenarioStateIs("WORKING") - .willReturn(aResponse() + .withHeader("content-type", "text/html")) + .willSetStateTo("WORKING")); + + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .inScenario("RANDOM_FAILURE") + .whenScenarioStateIs("WORKING") + .willReturn( + aResponse() .withBody(analyses) .withStatus(200) - .withHeader("content-type", "application/json") - ) - ); - - val analysesMono = songStudyDAO.getStudyAnalyses(GetStudyAnalysesCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) - .studyId("PEME-CA") - .build() - ); - - StepVerifier.create(analysesMono) - .expectNext(analysesList) - .verifyComplete(); - - } - - @Test - void fetchingStudyAnalysesShouldReturnRetryExhaustedException() { - //given - val command = GetStudyAnalysesCommand.builder() - .filesRepositoryBaseUrl("http://localhost:"+ wiremockPort) + .withHeader("content-type", "application/json"))); + + val analysesMono = + songStudyDAO.getStudyAnalyses( + GetStudyAnalysesCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) + .studyId("PEME-CA") + .build()); + + StepVerifier.create(analysesMono).expectNext(analysesList).verifyComplete(); + } + + @Test + void fetchingStudyAnalysesShouldReturnRetryExhaustedException() { + // given + val command = + GetStudyAnalysesCommand.builder() + .filesRepositoryBaseUrl("http://localhost:" + wiremockPort) .studyId("PEME-CA") .build(); - stubFor( - request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() + stubFor( + request("GET", urlEqualTo("/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() .withStatus(400) .withBody("

Some wierd unexpected text

") - .withHeader("content-type", "text/html") - ) - ); + .withHeader("content-type", "text/html"))); - //when - val analysesMono = songStudyDAO.getStudyAnalyses(command); + // when + val analysesMono = songStudyDAO.getStudyAnalyses(command); - //then - StepVerifier.create(analysesMono) - .expectError(RetryExhaustedException.class) - .verify(); - } + // then + StepVerifier.create(analysesMono).expectError(RetryExhaustedException.class).verify(); + } - @Import({ - SongConfig.class - }) - @Configuration - static class Config { - @Bean - WebClient webClient() { - return WebClient.builder().build(); - } - - @Bean - ApplicationProperties properties() { - ApplicationProperties properties = mock(ApplicationProperties.class); - when(properties.songMaxRetries()).thenReturn(3); - when(properties.songStudyCallTimeoutSeconds()).thenReturn(20); - when(properties.indexableStudyStatuses()).thenReturn("PUBLISHED"); - return properties; - } + @Import({SongConfig.class}) + @Configuration + static class Config { + @Bean + WebClient webClient() { + return WebClient.builder().build(); } + @Bean + ApplicationProperties properties() { + ApplicationProperties properties = mock(ApplicationProperties.class); + when(properties.songMaxRetries()).thenReturn(3); + when(properties.songStudyCallTimeoutSeconds()).thenReturn(20); + when(properties.indexableStudyStatuses()).thenReturn("PUBLISHED"); + return properties; + } + } } - diff --git a/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java b/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java index 3a627b31..5d71b1b6 100644 --- a/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java +++ b/maestro-app/src/test/java/bio/overture/maestro/domain/api/IndexerIntegrationTest.java @@ -17,6 +17,17 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.DefaultIndexer.*; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonString; +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static java.text.MessageFormat.format; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.times; + import bio.overture.maestro.MaestroIntegrationTest; import bio.overture.maestro.app.infra.adapter.outbound.notification.Slack; import bio.overture.maestro.app.infra.config.RootConfiguration; @@ -32,6 +43,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.client.ResponseDefinitionBuilder; +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.val; import org.elasticsearch.action.search.SearchRequest; @@ -51,506 +65,582 @@ import org.springframework.boot.test.mock.mockito.SpyBean; import reactor.test.StepVerifier; -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.DefaultIndexer.*; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonString; -import static com.github.tomakehurst.wiremock.client.WireMock.*; -import static java.text.MessageFormat.format; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.times; - @Tag(TestCategory.INT_TEST) class IndexerIntegrationTest extends MaestroIntegrationTest { - @Autowired - private RestHighLevelClient restHighLevelClient; + @Autowired private RestHighLevelClient restHighLevelClient; - @Autowired - private ApplicationProperties applicationProperties; + @Autowired private ApplicationProperties applicationProperties; - @Autowired - @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) - private ObjectMapper elasticSearchJsonMapper; + @Autowired + @Qualifier(RootConfiguration.ELASTIC_SEARCH_DOCUMENT_JSON_MAPPER) + private ObjectMapper elasticSearchJsonMapper; - @SpyBean - private Notifier notifier; + @SpyBean private Notifier notifier; - @SpyBean - private Slack slackChannel; + @SpyBean private Slack slackChannel; - private String alias; + private String alias; - @Autowired - private DefaultIndexer indexer; + @Autowired private DefaultIndexer indexer; - @BeforeEach - void setUp() { - alias = applicationProperties.fileCentricAlias(); - } + @BeforeEach + void setUp() { + alias = applicationProperties.fileCentricAlias(); + } - @Test - void shouldHandleErrorsIfStudyDaoThrowException() { + @Test + void shouldHandleErrorsIfStudyDaoThrowException() { - val analysisId = "EGAZ00001254368"; - val repoId = "collab"; - val studyId = "PEME-CA"; + val analysisId = "EGAZ00001254368"; + val repoId = "collab"; + val studyId = "PEME-CA"; - // Given - stubFor( - request("GET", urlEqualTo(format("/{0}/studies/{1}/analysis/{2}", repoId, studyId, analysisId))) - .willReturn(aResponse() + // Given + stubFor( + request( + "GET", + urlEqualTo(format("/{0}/studies/{1}/analysis/{2}", repoId, studyId, analysisId))) + .willReturn( + aResponse() .withStatus(500) .withBody("

Some weird unexpected text

") - .withHeader("content-type", "text/html") - ) - ); - - // checks the notification request - stubFor(request("POST", urlEqualTo("/slack")) - .withRequestBody(equalToJson("{\"username\":\"maestro\"," + - "\"text\":\":bangbang: Error : " + NotificationName.FAILED_TO_FETCH_ANALYSIS.name() - + ", Error Info: ```{analysisId=" + analysisId + ", " + - "repoCode=" + repoId + ", studyId=" + studyId + ", " + - "err=org.springframework.web.reactive.function.UnsupportedMediaTypeException: Content type 'text/html' " + - "not supported for bodyType=bio.overture.maestro.domain.entities.metadata.studyId.Analysis}```\"," + - "\"channel\":\"maestro-test\"}") - ) - .willReturn(aResponse() - .withStatus(200) - .withBody("ok") - ) - ); - - // test - val result = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId(analysisId) - .studyId(studyId) - .repositoryCode(repoId) - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder() + .withHeader("content-type", "text/html"))); + + // checks the notification request + stubFor( + request("POST", urlEqualTo("/slack")) + .withRequestBody( + equalToJson( + "{\"username\":\"maestro\"," + + "\"text\":\":bangbang: Error : " + + NotificationName.FAILED_TO_FETCH_ANALYSIS.name() + + ", Error Info: ```{analysisId=" + + analysisId + + ", " + + "repoCode=" + + repoId + + ", studyId=" + + studyId + + ", " + + "err=org.springframework.web.reactive.function.UnsupportedMediaTypeException: Content type 'text/html' " + + "not supported for bodyType=bio.overture.maestro.domain.entities.metadata.studyId.Analysis}```\"," + + "\"channel\":\"maestro-test\"}")) + .willReturn(aResponse().withStatus(200).withBody("ok"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId(analysisId) + .studyId(studyId) + .repositoryCode(repoId) + .build()) + .build()); + + StepVerifier.create(result) + .expectNext( + IndexResult.builder() .failureData( FailureData.builder() - .failingIds(Map.of(ANALYSIS_ID, Set.of(analysisId))).build() - ).successful(false).build() - ).verifyComplete(); - - ArgumentMatcher matcher = (arg) -> - arg.getNotificationName().equals(NotificationName.FAILED_TO_FETCH_ANALYSIS) + .failingIds(Map.of(ANALYSIS_ID, Set.of(analysisId))) + .build()) + .successful(false) + .build()) + .verifyComplete(); + + ArgumentMatcher matcher = + (arg) -> + arg.getNotificationName().equals(NotificationName.FAILED_TO_FETCH_ANALYSIS) && arg.getAttributes().get(ANALYSIS_ID).equals(analysisId) && arg.getAttributes().get(REPO_CODE).equals(repoId) && arg.getAttributes().get(STUDY_ID).equals(studyId) && arg.getAttributes().containsKey(ERR); - then(notifier).should(times(1)).notify(Mockito.argThat(matcher)); - then(slackChannel).should(times(1)).send(Mockito.argThat(matcher)); - - } - - @Test - void shouldIndexAnalysis() throws InterruptedException, IOException { - // Given - val analyses = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + then(notifier).should(times(1)).notify(Mockito.argThat(matcher)); + then(slackChannel).should(times(1)).send(Mockito.argThat(matcher)); + } + + @Test + void shouldIndexAnalysis() throws InterruptedException, IOException { + // Given + val analyses = loadJsonString(this.getClass(), "PEME-CA.analysis.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis/EGAZ00001254368")) - .willReturn(aResponse() - .withStatus(200) - .withBody(analyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - val result = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .analysisId("EGAZ00001254368") - .studyId("PEME-CA") - .repositoryCode("collab") - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(1L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - } + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis/EGAZ00001254368")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254368") + .studyId("PEME-CA") + .repositoryCode("collab") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldIndexCRAMAnalysis() throws InterruptedException, IOException { + // Given + val analyses = loadJsonString(this.getClass(), "CRAM-TEST.analysis.json"); + val expectedDoc = + loadJsonFixture( + this.getClass(), + "cram.doc0.json", + FileCentricDocument.class, + elasticSearchJsonMapper, + Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - @Test - void shouldIndexStudyRepositoryWithExclusionsApplied() throws InterruptedException, IOException { - // Given - val studiesArray = loadJsonFixture(this.getClass(), "studies.json", String[].class); - val studies = Arrays.stream(studiesArray) - .map(s-> Study.builder().studyId(s).build()) + stubFor( + request("GET", urlEqualTo("/collab/studies/CRAM-TEST/analysis/EGAZ00001254369")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254369") + .studyId("CRAM-TEST") + .repositoryCode("collab") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc, docs.get(0)); + } + + @Test + void shouldIndexStudyRepositoryWithExclusionsApplied() throws InterruptedException, IOException { + // Given + val studiesArray = loadJsonFixture(this.getClass(), "studies.json", String[].class); + val studies = + Arrays.stream(studiesArray) + .map(s -> Study.builder().studyId(s).build()) .collect(Collectors.toList()); - val expectedDocs = Arrays.asList( - loadJsonFixture(this.getClass(), + val expectedDocs = + Arrays.asList( + loadJsonFixture( + this.getClass(), "docs.json", - FileCentricDocument[].class, elasticSearchJsonMapper, - Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl()) - ) - ); - - for (Study study: studies) { - val studyId = study.getStudyId(); - val studyAnalyses = getStudyAnalysesAsString(studyId); - stubFor(request("GET", urlEqualTo("/collab/studies/" + studyId + "/analysis?analysisStates=PUBLISHED")) - .willReturn( - aResponse() - .withStatus(200) - .withBody(studyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - } - - stubFor(request("GET", urlEqualTo("/collab/studies/all")) + FileCentricDocument[].class, + elasticSearchJsonMapper, + Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl()))); + + for (Study study : studies) { + val studyId = study.getStudyId(); + val studyAnalyses = getStudyAnalysesAsString(studyId); + stubFor( + request( + "GET", + urlEqualTo("/collab/studies/" + studyId + "/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(studyAnalyses) + .withHeader("Content-Type", "application/json"))); + } + + stubFor( + request("GET", urlEqualTo("/collab/studies/all")) .willReturn(ResponseDefinitionBuilder.okForJson(studiesArray))); - // test - val result = indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode("collab") - .build()); + // test + val result = + indexer.indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode("collab").build()); - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); - Thread.sleep(sleepMillis); + Thread.sleep(sleepMillis); - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(32L, docs.size()); - val sortedExpected = expectedDocs.stream() + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(32L, docs.size()); + val sortedExpected = + expectedDocs.stream() .sorted(Comparator.comparing(FileCentricDocument::getObjectId)) .collect(Collectors.toList()); - val sortedDocs = docs.stream() + val sortedDocs = + docs.stream() .sorted(Comparator.comparing(FileCentricDocument::getObjectId)) .collect(Collectors.toList()); - assertEquals(sortedExpected, sortedDocs); - - } - - @Test - void shouldIndexStudyWithExclusionsApplied() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + assertEquals(sortedExpected, sortedDocs); + } + + @Test + void shouldIndexStudyWithExclusionsApplied() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val analyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), + val expectedDoc1 = + loadJsonFixture( + this.getClass(), "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(analyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - val result = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("collab") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc1, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldDeleteSingleAnalysis() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); - val expectedDoc0 = loadJsonFixture(this.getClass(), + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(analyses) + .withHeader("Content-Type", "application/json"))); + + // test + val result = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("collab").studyId("PEME-CA").build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc1, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldDeleteSingleAnalysis() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - .withStatus(200) - ) - ); - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - .withStatus(200) - ) - ); - - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // test - val result = indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier( - AnalysisIdentifier.builder() - .analysisId("EGAZ00001254247") - .studyId("PEME-CA") - .repositoryCode("aws") - .build() - ).build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(1L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldUpdateExistingFileDocRepository() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); - - val expectedDoc0 = loadJsonFixture(this.getClass(), + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json") + .withStatus(200))); + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json") + .withStatus(200))); + + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // test + val result = + indexer.removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .analysisId("EGAZ00001254247") + .studyId("PEME-CA") + .repositoryCode("aws") + .build()) + .build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(1L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldUpdateExistingFileDocRepository() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.study.json"); + + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val multiRepoDoc = loadJsonFixture(this.getClass(), + val multiRepoDoc = + loadJsonFixture( + this.getClass(), "doc2.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of( "COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl(), - "AWS_REPO_URL", applicationProperties.repositories().get(1).getUrl() - ) - ); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - // step 1 - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // step 2 index the same files from another repository: - val secondResult = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("aws") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(secondResult) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(multiRepoDoc, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @Test - void shouldDetectAndNotifyConflictingDocuments() throws InterruptedException, IOException { - // Given - @SuppressWarnings("all") - val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); - - // this has a different analysis id than the one in previous files - @SuppressWarnings("all") - val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.conflicting.study.json"); - val awsStudyAnalysesList = loadJsonFixture(this.getClass(), "PEME-CA.aws.conflicting.study.json", new TypeReference>() {}); - val expectedDoc0 = loadJsonFixture(this.getClass(), + "AWS_REPO_URL", applicationProperties.repositories().get(1).getUrl())); + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json"))); + + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json"))); + + // test + // step 1 + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // step 2 index the same files from another repository: + val secondResult = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("aws").studyId("PEME-CA").build()); + + StepVerifier.create(secondResult) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(multiRepoDoc, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @Test + void shouldDetectAndNotifyConflictingDocuments() throws InterruptedException, IOException { + // Given + @SuppressWarnings("all") + val collabAnalyses = loadJsonString(this.getClass(), "PEME-CA.study.json"); + + // this has a different analysis id than the one in previous files + @SuppressWarnings("all") + val awsStudyAnalyses = loadJsonString(this.getClass(), "PEME-CA.aws.conflicting.study.json"); + val awsStudyAnalysesList = + loadJsonFixture( + this.getClass(), + "PEME-CA.aws.conflicting.study.json", + new TypeReference>() {}); + val expectedDoc0 = + loadJsonFixture( + this.getClass(), "doc0.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedDoc1 = loadJsonFixture(this.getClass(), "doc1.json", + val expectedDoc1 = + loadJsonFixture( + this.getClass(), + "doc1.json", FileCentricDocument.class, elasticSearchJsonMapper, Map.of("COLLAB_REPO_URL", applicationProperties.repositories().get(0).getUrl())); - val expectedNotification = new IndexerNotification(NotificationName.INDEX_FILE_CONFLICT, + val expectedNotification = + new IndexerNotification( + NotificationName.INDEX_FILE_CONFLICT, getConflicts(expectedDoc1, awsStudyAnalysesList.get(0).getAnalysisId())); - stubFor(request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(collabAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - stubFor(request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) - .willReturn(aResponse() - .withStatus(200) - .withBody(awsStudyAnalyses) - .withHeader("Content-Type", "application/json") - ) - ); - - // test - populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); - - // index the same files from another repository: - // test - val secondResult = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("aws") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(secondResult) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - Thread.sleep(sleepMillis); - - // assertions - val docs = getFileCentricDocuments(); - assertNotNull(docs); - then(notifier).should(times(1)).notify(eq(expectedNotification)); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc0, docs.get(0)); - assertEquals(expectedDoc1, docs.get(1)); - } - - @NotNull - private Map getConflicts(FileCentricDocument document, String differentAnalysisId) { - return Map.of("conflicts", List.of(DefaultIndexer.FileConflict.builder() - .indexedFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(document.getStudyId()) - .analysisId(document.getAnalysis().getAnalysisId()) - .objectId(document.getObjectId()) - .repoCode(document - .getRepositories() - .stream().map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet())) - .build() - ).newFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(document.getStudyId()) - .analysisId(differentAnalysisId) - .objectId(document.getObjectId()) - .repoCode(Set.of("aws")) - .build() - ).build())); - } - - @SneakyThrows - private void populateIndexWithCollabStudy(FileCentricDocument expectedDoc0, FileCentricDocument expectedDoc1) throws InterruptedException { - val result = indexer.indexStudy(IndexStudyCommand.builder() - .repositoryCode("COLLAB") - .studyId("PEME-CA") - .build()); - - StepVerifier.create(result) - .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) - .verifyComplete(); - - Thread.sleep(sleepMillis); - - // assertions - List docs = getFileCentricDocuments(); - - assertNotNull(docs); - assertEquals(2L, docs.size()); - assertEquals(expectedDoc1, docs.get(1)); - assertEquals(expectedDoc0, docs.get(0)); - } - - @NotNull - private List getFileCentricDocuments() throws IOException { - val searchRequest = new SearchRequest(alias); - val searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(QueryBuilders.matchAllQuery()); - searchSourceBuilder.from(0); - searchSourceBuilder.size(100); - searchRequest.source(searchSourceBuilder); - val response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); - val docs = new ArrayList(); - - for(SearchHit hit : response.getHits().getHits()) { - String source = hit.getSourceAsString(); - val doc = elasticSearchJsonMapper.readValue(source, FileCentricDocument.class); - docs.add(doc); - } - return docs; - } - - @SneakyThrows - private String getStudyAnalysesAsString(String studyId) { - return loadJsonString(getClass(), studyId +".analysis.json"); + stubFor( + request("GET", urlEqualTo("/collab/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(collabAnalyses) + .withHeader("Content-Type", "application/json"))); + stubFor( + request("GET", urlEqualTo("/aws/studies/PEME-CA/analysis?analysisStates=PUBLISHED")) + .willReturn( + aResponse() + .withStatus(200) + .withBody(awsStudyAnalyses) + .withHeader("Content-Type", "application/json"))); + + // test + populateIndexWithCollabStudy(expectedDoc0, expectedDoc1); + + // index the same files from another repository: + // test + val secondResult = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("aws").studyId("PEME-CA").build()); + + StepVerifier.create(secondResult) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + Thread.sleep(sleepMillis); + + // assertions + val docs = getFileCentricDocuments(); + assertNotNull(docs); + then(notifier).should(times(1)).notify(eq(expectedNotification)); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc0, docs.get(0)); + assertEquals(expectedDoc1, docs.get(1)); + } + + @NotNull + private Map getConflicts( + FileCentricDocument document, String differentAnalysisId) { + return Map.of( + "conflicts", + List.of( + DefaultIndexer.FileConflict.builder() + .indexedFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(document.getStudyId()) + .analysisId(document.getAnalysis().getAnalysisId()) + .objectId(document.getObjectId()) + .repoCode( + document.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .newFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(document.getStudyId()) + .analysisId(differentAnalysisId) + .objectId(document.getObjectId()) + .repoCode(Set.of("aws")) + .build()) + .build())); + } + + @SneakyThrows + private void populateIndexWithCollabStudy( + FileCentricDocument expectedDoc0, FileCentricDocument expectedDoc1) + throws InterruptedException { + val result = + indexer.indexStudy( + IndexStudyCommand.builder().repositoryCode("COLLAB").studyId("PEME-CA").build()); + + StepVerifier.create(result) + .expectNext(IndexResult.builder().indexName("file_centric_1.0").successful(true).build()) + .verifyComplete(); + + Thread.sleep(sleepMillis); + + // assertions + List docs = getFileCentricDocuments(); + + assertNotNull(docs); + assertEquals(2L, docs.size()); + assertEquals(expectedDoc1, docs.get(1)); + assertEquals(expectedDoc0, docs.get(0)); + } + + @NotNull + private List getFileCentricDocuments() throws IOException { + val searchRequest = new SearchRequest(alias); + val searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); + searchSourceBuilder.from(0); + searchSourceBuilder.size(100); + searchRequest.source(searchSourceBuilder); + val response = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + val docs = new ArrayList(); + + for (SearchHit hit : response.getHits().getHits()) { + String source = hit.getSourceAsString(); + val doc = elasticSearchJsonMapper.readValue(source, FileCentricDocument.class); + docs.add(doc); } - -} \ No newline at end of file + return docs; + } + + @SneakyThrows + private String getStudyAnalysesAsString(String studyId) { + return loadJsonString(getClass(), studyId + ".analysis.json"); + } +} diff --git a/maestro-app/src/test/resources/config/application.yml b/maestro-app/src/test/resources/config/application.yml index d5146c1d..cb204dd9 100644 --- a/maestro-app/src/test/resources/config/application.yml +++ b/maestro-app/src/test/resources/config/application.yml @@ -9,6 +9,7 @@ maestro: donor: - DO52693 song: + indexableStudyStatus: PUBLISHED maxRetries: 3 timeoutSec: study: 5 diff --git a/maestro-app/src/test/resources/fixtures/FileCentricElasticSearchAdapterUnavaibilityTest/PEME-CA.files.json b/maestro-app/src/test/resources/fixtures/FileCentricElasticSearchAdapterUnavaibilityTest/PEME-CA.files.json index b76ed748..7ee762d9 100644 --- a/maestro-app/src/test/resources/fixtures/FileCentricElasticSearchAdapterUnavaibilityTest/PEME-CA.files.json +++ b/maestro-app/src/test/resources/fixtures/FileCentricElasticSearchAdapterUnavaibilityTest/PEME-CA.files.json @@ -3,29 +3,27 @@ "object_id": "43d29864-6756-51f0-bb2a-ba7cff860778", "file_access": "controlled", "file_type": "BAM", + "data_type": "AlignedReads", "study_id": "PEME-CA", "analysis": { "analysis_id": "EGAZ00001254368", "analysis_type": "sequencingRead", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PEME-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "EGAZ00001254368", "aligned": true, "libraryStrategy": "WGS" } }, - "files": { + "file": { "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam", - "format": "BAM", "md5sum": "29ff9df727803d20834b9997bc17e970", "size": 88906416144, - "last_modified": null, "index_file": { "object_id": "d819f154-2292-56f7-94f3-bbe03fb45bc5", "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam.bai", - "format": "BAI", + "file_type": "BAI", "md5sum": "e2a83668aa4cfe4f4048e75212d44f19", "size": 8939816 } @@ -37,26 +35,24 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/41ba4fb3-9428-50b5-af6c-d779cd59b04d" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO232978", + "donor_id": "DO232978", "gender": "Female", "submitter_donor_id": "MDT-AP-0749", - "specimens": { - "id": "SP201301", + "specimens": [{ + "specimen_id": "SP201301", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "MDT-AP-0749_tumor_specimen", - "samples": { - "id": "SA604924", + "samples": [{ + "sample_id": "SA604924", "submitter_sample_id": "MDT-AP-0749_tumor", "sample_type": "DNA" - } - } + }] + }] } ] } diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/CRAM-TEST.analysis.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/CRAM-TEST.analysis.json new file mode 100644 index 00000000..bd28dee6 --- /dev/null +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/CRAM-TEST.analysis.json @@ -0,0 +1,74 @@ +{ + "analysisType": { + "name": "sequencingRead", + "version": 1 + }, + "info": { + "dcc_project_code": "CRAM-TEST", + "isPcawg": false + }, + "analysisId": "EGAZ00001254369", + "studyId": "CRAM-TEST", + "analysisState": "PUBLISHED", + "samples": [ + { + "sampleId": "SA604924", + "specimenId": "SP201301", + "submitterSampleId": "MDT-AP-0749_tumor", + "sampleType": "DNA", + "specimen": { + "specimenId": "SP201301", + "donorId": "DO232978", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO232978", + "submitterDonorId": "MDT-AP-0749", + "studyId": "CRAM-TEST", + "gender": "Female" + } + } + ], + "files": [ + { + "objectId": "3286edf3-31a9-42a3-8141-36c3ad713488", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "bundle.EGAZ00001254369.xml", + "fileSize": 6342, + "fileType": "XML", + "fileMd5sum": "fb157ece007dc31b3d34add273efedcb", + "fileAccess": "open", + "dataType": "AlignedReads" + }, + { + "objectId": "fbf3e6da-3e1e-45f7-994c-970530365d40", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "f5b822cfd81b4b9bbadfbd9b67f4ac4d.tumor_MDT-AP-0749_merged.mdup.cram.crai", + "fileSize": 8939816, + "fileType": "CRAI", + "fileMd5sum": "e2a83668aa4cfe4f4048e75212d44f19", + "fileAccess": "controlled", + "dataType": "AlignedReads" + }, + { + "objectId": "1e576567-9b70-4038-8580-88047f2e157f", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "f5b822cfd81b4b9bbadfbd9b67f4ac4d.tumor_MDT-AP-0749_merged.mdup.cram", + "fileSize": 88906416144, + "fileType": "CRAM", + "fileMd5sum": "f5b822cfd81b4b9bbadfbd9b67f4ac4d", + "fileAccess": "controlled", + "dataType": "AlignedReads" + } + ], + "experiment": { + "analysisId": "EGAZ00001254369", + "aligned": true, + "libraryStrategy": "WGS", + "info": {} + } +} \ No newline at end of file diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.analysis.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.analysis.json index 85ced908..149e0583 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.analysis.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.analysis.json @@ -15,18 +15,28 @@ "submitterSampleId": "MDT-AP-0749_tumor", "sampleType": "DNA", "specimen": { - "specimenId": "SP201301", "donorId": "DO232978", "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", - - "specimenType": "Primary tumour - solid tissue" + "specimenType": "Primary tumour - solid tissue", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } }, "donor": { "donorId": "DO232978", "submitterDonorId": "MDT-AP-0749", "studyId": "PEME-CA", - "gender": "Female" + "gender": "Female", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } + }, + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false } } ], @@ -40,10 +50,13 @@ "fileType": "XML", "fileMd5sum": "fb157ece007dc31b3d34add273efedcb", "fileAccess": "open", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } }, { - "objectId": "d819f154-2292-56f7-94f3-bbe03fb45bc5", "studyId": "PEME-CA", "analysisId": "EGAZ00001254368", @@ -52,7 +65,16 @@ "fileType": "BAI", "fileMd5sum": "e2a83668aa4cfe4f4048e75212d44f19", "fileAccess": "controlled", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false, + "nested": { + "nestedAgain": [ + {"theeNested": "word"} + ] + } + } }, { @@ -64,7 +86,11 @@ "fileType": "BAM", "fileMd5sum": "29ff9df727803d20834b9997bc17e970", "fileAccess": "controlled", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } } ], "experiment": { diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.study.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.study.json index 14da88b2..4ee02135 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.study.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/PEME-CA.study.json @@ -1,9 +1,6 @@ [ { - "analysisType": { - "name": "sequencingRead", - "version": 1 - }, + "analysisType": { "name": "sequencingRead", "version": 1 }, "info": { "dcc_project_code": "PEME-CA", "isPcawg": false @@ -13,30 +10,38 @@ "analysisState": "PUBLISHED", "samples": [ { - "info": {}, "sampleId": "SA604924", "specimenId": "SP201301", "submitterSampleId": "MDT-AP-0749_tumor", "sampleType": "DNA", "specimen": { - "info": {}, "specimenId": "SP201301", "donorId": "DO232978", "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", - "specimenType": "Primary tumour - solid tissue" + "specimenType": "Primary tumour - solid tissue", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } }, "donor": { "donorId": "DO232978", "submitterDonorId": "MDT-AP-0749", "studyId": "PEME-CA", "gender": "Female", - "info": {} + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } + }, + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false } } ], "files": [ { - "info": {}, "objectId": "41ba4fb3-9428-50b5-af6c-d779cd59b04d", "studyId": "PEME-CA", "analysisId": "EGAZ00001254368", @@ -45,10 +50,13 @@ "fileType": "XML", "fileMd5sum": "fb157ece007dc31b3d34add273efedcb", "fileAccess": "open", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } }, { - "info": {}, "objectId": "d819f154-2292-56f7-94f3-bbe03fb45bc5", "studyId": "PEME-CA", "analysisId": "EGAZ00001254368", @@ -57,10 +65,18 @@ "fileType": "BAI", "fileMd5sum": "e2a83668aa4cfe4f4048e75212d44f19", "fileAccess": "controlled", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false, + "nested": { + "nestedAgain": [ + {"theeNested": "word"} + ] + } + } }, { - "info": {}, "objectId": "43d29864-6756-51f0-bb2a-ba7cff860778", "studyId": "PEME-CA", "analysisId": "EGAZ00001254368", @@ -69,7 +85,11 @@ "fileType": "BAM", "fileMd5sum": "29ff9df727803d20834b9997bc17e970", "fileAccess": "controlled", - "dataType": "AlignedReads" + "dataType": "AlignedReads", + "info": { + "dcc_project_code": "PEME-CA", + "isPcawg": false + } } ], "experiment": { diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/cram.doc0.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/cram.doc0.json new file mode 100644 index 00000000..e68e3184 --- /dev/null +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/cram.doc0.json @@ -0,0 +1,64 @@ +{ + "object_id": "1e576567-9b70-4038-8580-88047f2e157f", + "file_access": "controlled", + "file_type": "CRAM", + "data_type": "AlignedReads", + "study_id": "CRAM-TEST", + "analysis": { + "analysis_id": "EGAZ00001254369", + "analysis_state": "PUBLISHED", + "analysis_type": "sequencingRead", + "analysis_version": "1", + "info": { + "dcc_project_code": "CRAM-TEST", + "isPcawg": false + }, + "experiment": { + "analysisId": "EGAZ00001254369", + "aligned": true, + "libraryStrategy": "WGS", + "info": {} + } + }, + "file": { + "name": "f5b822cfd81b4b9bbadfbd9b67f4ac4d.tumor_MDT-AP-0749_merged.mdup.cram", + "md5sum": "f5b822cfd81b4b9bbadfbd9b67f4ac4d", + "size": 88906416144, + "data_type": "AlignedReads", + "index_file": { + "object_id": "fbf3e6da-3e1e-45f7-994c-970530365d40", + "name": "f5b822cfd81b4b9bbadfbd9b67f4ac4d.tumor_MDT-AP-0749_merged.mdup.cram.crai", + "file_type": "CRAI", + "data_type": "AlignedReads", + "md5sum": "e2a83668aa4cfe4f4048e75212d44f19", + "size": 8939816 + } + }, + "repositories": [ + { + "code": "collab", + "organization": "ICGC", + "name": "collaboratory", + "type": "S3", + "country": "CA", + "url": "##COLLAB_REPO_URL##" + } + ], + "donors": [ + { + "donor_id": "DO232978", + "gender": "Female", + "submitter_donor_id": "MDT-AP-0749", + "specimens": [{ + "specimen_id": "SP201301", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "MDT-AP-0749_tumor_specimen", + "samples": [{ + "sample_id": "SA604924", + "submitter_sample_id": "MDT-AP-0749_tumor", + "sample_type": "DNA" + }] + }] + } + ] +} \ No newline at end of file diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc0.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc0.json index 5cd85307..8f40d719 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc0.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc0.json @@ -1,15 +1,16 @@ { "object_id": "43d29864-6756-51f0-bb2a-ba7cff860778", - "file_type" : "BAM", "file_access": "controlled", "file_type": "BAM", + "data_type": "AlignedReads", "study_id": "PEME-CA", + "dcc_project_code": "PEME-CA", + "isPcawg": false, "analysis": { "analysis_id": "EGAZ00001254368", + "analysis_state": "PUBLISHED", "analysis_type": "sequencingRead", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PEME-CA", "info": { "dcc_project_code": "PEME-CA", "isPcawg": false @@ -21,20 +22,27 @@ "info": {} } }, - "files": { + "file": { "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam", - "format": "BAM", "md5sum": "29ff9df727803d20834b9997bc17e970", "size": 88906416144, + "dcc_project_code": "PEME-CA", + "isPcawg": false, "data_type": "AlignedReads", - "last_modified": null, "index_file": { "object_id": "d819f154-2292-56f7-94f3-bbe03fb45bc5", "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam.bai", - "format": "BAI", + "file_type": "BAI", "data_type": "AlignedReads", "md5sum": "e2a83668aa4cfe4f4048e75212d44f19", - "size": 8939816 + "size": 8939816, + "dcc_project_code": "PEME-CA", + "isPcawg": false, + "nested": { + "nestedAgain": [ + {"theeNested": "word"} + ] + } } }, "repositories": [ @@ -44,26 +52,30 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/41ba4fb3-9428-50b5-af6c-d779cd59b04d" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO232978", - "gender" : "Female", + "donor_id": "DO232978", + "gender": "Female", "submitter_donor_id": "MDT-AP-0749", - "specimens": { - "id": "SP201301", + "dcc_project_code": "PEME-CA", + "isPcawg": false, + "specimens": [{ + "specimen_id": "SP201301", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "MDT-AP-0749_tumor_specimen", - "samples": { - "id": "SA604924", + "dcc_project_code": "PEME-CA", + "isPcawg": false, + "samples": [{ + "sample_id": "SA604924", "submitter_sample_id": "MDT-AP-0749_tumor", - "sample_type": "DNA" - } - } + "sample_type": "DNA", + "dcc_project_code": "PEME-CA", + "isPcawg": false + }] + }] } ] } \ No newline at end of file diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc1.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc1.json index 7c7b91d2..a45599ab 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc1.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc1.json @@ -2,17 +2,17 @@ "object_id": "9ce9358d-c93a-5f83-8032-4addcb84b51a", "file_access": "controlled", "file_type": "BAM", + "data_type": "AlignedReads", "study_id": "PEME-CA", "analysis": { "analysis_id": "EGAZ00001254247", "analysis_type": "sequencingRead", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": { "dcc_project_code": "PEME-CA", "isPcawg": false }, - "study_id": "PEME-CA", "experiment": { "analysisId": "EGAZ00001254247", "aligned": true, @@ -20,17 +20,15 @@ "info": {} } }, - "files": { + "file": { "name": "b2e40ebb719e1754e99be2e752239639.control_MDT-AP-0432_merged.mdup.bam", - "format": "BAM", "md5sum": "b2e40ebb719e1754e99be2e752239639", "size": 77675501639, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "6aa8d318-e168-520e-9d1e-cef127ee6b65", "name": "b2e40ebb719e1754e99be2e752239639.control_MDT-AP-0432_merged.mdup.bam.bai", - "format": "BAI", + "file_type": "BAI", "data_type": "AlignedReads", "md5sum": "227a2a11d660a8a53a7375c6623034e2", "size": 8982928 @@ -43,26 +41,24 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/464116e4-afc9-5879-b567-6f54513a32dc" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO232959", + "donor_id": "DO232959", "gender" : "Female", "submitter_donor_id": "MDT-AP-0432", - "specimens": { - "id": "SP200947", + "specimens": [{ + "specimen_id": "SP200947", "specimen_type": "Normal - blood derived", "submitter_specimen_id": "MDT-AP-0432_control_specimen", - "samples": { - "id": "SA604905", + "samples": [{ + "sample_id": "SA604905", "submitter_sample_id": "MDT-AP-0432_control", "sample_type": "DNA" - } - } + }] + }] } ] } \ No newline at end of file diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc2.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc2.json index 3017099b..0f45909a 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc2.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/doc2.json @@ -2,17 +2,17 @@ "object_id": "9ce9358d-c93a-5f83-8032-4addcb84b51a", "file_access": "controlled", "file_type": "BAM", + "data_type": "AlignedReads", "study_id": "PEME-CA", "analysis": { "analysis_id": "EGAZ00001254247", "analysis_type": "sequencingRead", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": { "dcc_project_code": "PEME-CA", "isPcawg": false }, - "study_id": "PEME-CA", "experiment": { "info": {}, "analysisId": "EGAZ00001254247", @@ -20,17 +20,15 @@ "libraryStrategy": "WGS" } }, - "files": { + "file": { "name": "b2e40ebb719e1754e99be2e752239639.control_MDT-AP-0432_merged.mdup.bam", - "format": "BAM", "md5sum": "b2e40ebb719e1754e99be2e752239639", "size": 77675501639, "data_type": "AlignedReads", - "last_modified": null, "index_file": { "object_id": "6aa8d318-e168-520e-9d1e-cef127ee6b65", "name": "b2e40ebb719e1754e99be2e752239639.control_MDT-AP-0432_merged.mdup.bam.bai", - "format": "BAI", + "file_type": "BAI", "data_type": "AlignedReads", "md5sum": "227a2a11d660a8a53a7375c6623034e2", "size": 8982928 @@ -43,9 +41,7 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/464116e4-afc9-5879-b567-6f54513a32dc" + "url": "##COLLAB_REPO_URL##" }, { "code": "aws", @@ -53,26 +49,24 @@ "name": "aws repo", "type": "S3", "country": "US", - "url": "##AWS_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/464116e4-afc9-5879-b567-6f54513a32dc" + "url": "##AWS_REPO_URL##" } ], "donors": [ { - "id": "DO232959", + "donor_id": "DO232959", "gender" : "Female", "submitter_donor_id": "MDT-AP-0432", - "specimens": { - "id": "SP200947", + "specimens": [{ + "specimen_id": "SP200947", "specimen_type": "Normal - blood derived", "submitter_specimen_id": "MDT-AP-0432_control_specimen", - "samples": { - "id": "SA604905", + "samples": [{ + "sample_id": "SA604905", "submitter_sample_id": "MDT-AP-0432_control", "sample_type": "DNA" - } - } + }] + }] } ] } \ No newline at end of file diff --git a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/docs.json b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/docs.json index 630621a4..392ba31f 100644 --- a/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/docs.json +++ b/maestro-app/src/test/resources/fixtures/IndexerIntegrationTest/docs.json @@ -3,13 +3,13 @@ "object_id": "b63dff38-e239-5a5d-9c36-ec851049fd12", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -18,16 +18,14 @@ "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "93354882b0fc014b49be1b24df8f435e", "size": 44440835, - "last_modified": null, "index_file": { "object_id": "eb2832eb-ad54-574b-bb60-1abf74489ed8", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "ade04a5fd0e9d4052b0ec0b280a9be60", "size": 1458418, "data_type": "AlignedReads" @@ -41,26 +39,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", - "submitter_sample_id": "RK202_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP107109", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK202_C01", + "samples": [ + { + "sample_id": "SA515389", + "submitter_sample_id": "RK202_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -68,13 +68,13 @@ "object_id": "feecfb7d-2aa9-5434-a0a0-56a6c5ee4a70", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -83,16 +83,14 @@ "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svfix2_4-0-12.20160209.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "ae2e0a8eb02414569fa06cd56a3e809e", "size": 11431, - "last_modified": null, "index_file": { "object_id": "e14e73a5-a869-5b9a-a89c-be77fa54135e", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svfix2_4-0-12.20160209.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "9ce7cdedb7450a05c03e419235ed56a6", "size": 1756, "data_type": "AlignedReads" @@ -106,26 +104,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", - "submitter_sample_id": "RK202_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP107109", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK202_C01", + "samples": [ + { + "sample_id": "SA515389", + "submitter_sample_id": "RK202_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -133,13 +133,13 @@ "object_id": "3106300b-3666-52b7-a3fa-a5096a51814b", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -148,16 +148,14 @@ "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "492989e81699bd5a56b7956e0be7416d", "size": 3648, - "last_modified": null, "index_file": { "object_id": "ad4e821d-1908-5e37-8b09-f2bcbe841a6b", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "acdf137ad635f95ee93412fa9153d066", "size": 5539, "data_type": "AlignedReads" @@ -171,26 +169,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", - "submitter_sample_id": "RK202_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP107109", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK202_C01", + "samples": [ + { + "sample_id": "SA515389", + "submitter_sample_id": "RK202_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -198,13 +198,13 @@ "object_id": "bed33bd4-eec1-5905-8480-5def36d7f6de", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -213,16 +213,14 @@ "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "1f3cfd1e1ad462cff9871eb2373935ac", "size": 1418099, - "last_modified": null, "index_file": { "object_id": "deab15a3-6820-5bcc-926a-bca2ab5ddaa0", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "2233fb90872d759d279395b1a7ad3461", "size": 236926, "data_type": "AlignedReads" @@ -236,26 +234,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", - "submitter_sample_id": "RK202_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP107109", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK202_C01", + "samples": [ + { + "sample_id": "SA515389", + "submitter_sample_id": "RK202_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -263,13 +263,13 @@ "object_id": "24329157-e218-5854-8f45-7fa6b9e046c6", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -278,16 +278,14 @@ "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "7d0676c6a85696e1cb2919836b0ab71e", "size": 10536170, - "last_modified": null, "index_file": { "object_id": "ad8a45f3-ec8d-5391-ae93-b19dc982698a", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "8269ee64b80e6ec66e590767f697d37b", "size": 1031078, "data_type": "AlignedReads" @@ -301,26 +299,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -328,13 +328,13 @@ "object_id": "4d070189-bd33-5391-b7af-ea32c6782875", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -343,16 +343,14 @@ "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "1d0a8bcf2abf8373c6f8cc63cb748aa8", "size": 59633, - "last_modified": null, "index_file": { "object_id": "14e94abe-1fd1-5a85-8d26-a7291edd80ef", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "fdb961feac82a7912bf657450e02c44d", "size": 11325, "data_type": "AlignedReads" @@ -366,26 +364,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -393,13 +393,13 @@ "object_id": "107c83c7-595e-5b36-957b-ae706611d5b9", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "info": {}, @@ -408,16 +408,14 @@ "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "fc246f5e1f670b1f756b8e4294f59504", "size": 51548, - "last_modified": null, "index_file": { "object_id": "c5496778-0d72-5cc6-aac4-98777cfbc0c2", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "5aeb4749609bd0a4d81ecffd673a36e1", "size": 10447, "data_type": "AlignedReads" @@ -431,26 +429,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -458,13 +458,13 @@ "object_id": "da524891-37af-5a21-9867-c6812018aa22", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", @@ -473,16 +473,14 @@ "info": {} } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "1f22fff9e578244ba16762a4c82792e8", "size": 1710383, - "last_modified": null, "index_file": { "object_id": "d87c4599-bba4-5c23-b7a2-57bbeed7eacb", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "ade5449b588c81496e4e7d2b0b49e3d0", "size": 61514, "data_type": "AlignedReads" @@ -496,26 +494,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -523,13 +523,13 @@ "object_id": "46194bd5-b131-55f7-b7b1-99293ba1d6b9", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", @@ -538,16 +538,14 @@ "info": {} } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "e75eef7af227c5c80d0b499ffd39c43b", "size": 31242, - "last_modified": null, "index_file": { "object_id": "55283d00-b1a2-521c-99e1-e9d8d88c660d", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "8cae7bbcd28b6a78ad5c53642ae9695c", "size": 14606, "data_type": "AlignedReads" @@ -561,26 +559,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -588,13 +588,13 @@ "object_id": "ae5b1daf-05d5-5e41-be2a-9f1d3369d517", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", @@ -603,16 +603,14 @@ "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "f199c6714f7a6880d30bf7aa56e00625", "size": 10191, - "last_modified": null, "index_file": { "object_id": "c4d10f9f-7e9e-5cb3-a30a-0ff8865952ce", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "139ce5f29fc0e877d5ae5a5d5e03c3f0", "size": 4212, "data_type": "AlignedReads" @@ -626,26 +624,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -653,13 +653,13 @@ "object_id": "8b28796a-1630-5d12-92a7-d8264ee57bc6", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", @@ -668,16 +668,14 @@ "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "d641e6e373d21d2fc882f2e4550b7b2c", "size": 382815, - "last_modified": null, "index_file": { "object_id": "a99d2a9c-96c4-5788-9149-79a146f0f925", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "797e4ef5c348d4a23e9e83790812d857", "size": 151104, "data_type": "AlignedReads" @@ -691,26 +689,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", - "submitter_sample_id": "RK156_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP50179", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK156_C01", + "samples": [ + { + "sample_id": "SA270462", + "submitter_sample_id": "RK156_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -718,13 +718,13 @@ "object_id": "316177db-022d-5121-aba8-ce72a92ee778", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", @@ -733,16 +733,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "33b96c263be31852de8d5d15775561c9", "size": 404406, - "last_modified": null, "index_file": { "object_id": "97c3147f-dd1d-507f-a73d-8646f7ce2a0b", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "3ed262dbc3b1c427177bdd5af5ef5633", "size": 27885, "data_type": "AlignedReads" @@ -756,26 +754,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -783,14 +783,14 @@ "object_id": "18917df0-2cc6-5ac5-88d8-f48acf24c92e", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", @@ -798,16 +798,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "d3227038bd37e74ff88b07e5408c76ed", "size": 1853889, - "last_modified": null, "index_file": { "object_id": "9abe4aad-9d35-5d4f-955c-8bceb559827e", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "a7cf0d572f1649c8791cda7892c7be55", "size": 67312, "data_type": "AlignedReads" @@ -821,26 +819,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -848,14 +848,14 @@ "object_id": "5c222a32-a79d-55e0-a490-440217c63393", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "info": {}, @@ -863,16 +863,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "64ef1b8daf5b6eec161555f5cdad3420", "size": 10429379, - "last_modified": null, "index_file": { "object_id": "ad68b7cb-e91c-506e-acae-812ae2ea7678", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "78b2dd0696a1f2ea8ac8e25a77c4e239", "size": 1032907, "data_type": "AlignedReads" @@ -886,26 +884,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -913,14 +913,14 @@ "object_id": "2af2e398-c27b-5645-b60b-b932a916184d", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", @@ -928,16 +928,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "7e7f3e23aa8869bfd82ce16382b7bc7d", "size": 400597, - "last_modified": null, "index_file": { "object_id": "bbb31cb3-f175-52c4-ba66-569e1a1d2ed7", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "130e37274d3c7bebcb41d027eea9148d", "size": 27877, "data_type": "AlignedReads" @@ -951,26 +949,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -978,14 +978,14 @@ "object_id": "d9388ed4-1603-584f-9b18-5ff53b9bffeb", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", @@ -993,16 +993,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "ee313d9cbe077063174d92050dde03e0", "size": 25270, - "last_modified": null, "index_file": { "object_id": "95e7563e-e8d1-5377-8cfc-1c45a72448da", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "941a7d48a3250de6edcbbde1cbb4b7e1", "size": 12044, "data_type": "AlignedReads" @@ -1016,26 +1014,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1043,14 +1043,14 @@ "object_id": "60307755-185e-536e-aa3b-190fb9934844", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "info": {}, @@ -1058,16 +1058,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "cbafecb01dea96496176bf7497cb9e80", "size": 2540, - "last_modified": null, "index_file": { "object_id": "221e532c-a0c8-5573-8399-81f785423e57", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "de502cd9a25efe6f868f3156e8fd302e", "size": 1336, "data_type": "AlignedReads" @@ -1081,26 +1079,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1108,14 +1108,14 @@ "object_id": "2664e052-ba85-5be4-89e0-4f2f06a998c0", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "LIRI-JP", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", @@ -1123,16 +1123,14 @@ "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "ca10e3bd248498c9427b2b5e324f0242", "size": 422502, - "last_modified": null, "index_file": { "object_id": "87d9f1f2-3d13-5dd2-8ed5-d175952e2586", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "9029f92d2d1ea6d76b3a88795756aacc", "size": 160972, "data_type": "AlignedReads" @@ -1146,26 +1144,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", - "specimen_type": "Primary tumour - solid tissue", - "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", - "submitter_sample_id": "RK091_C01", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP99145", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "RK091_C01", + "samples": [ + { + "sample_id": "SA501653", + "submitter_sample_id": "RK091_C01", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1173,13 +1173,13 @@ "object_id": "e86158ec-0eb6-50e2-bbb5-7bfed4745db0", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", @@ -1188,16 +1188,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "14ded2a7342f9124421ace5aa920510d", "size": 543337, - "last_modified": null, "index_file": { "object_id": "bb949d15-334f-5712-9f04-41225a7beb8d", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "99f0b87f5113494e2fc2284c91dc15e0", "size": 224729, "data_type": "AlignedReads" @@ -1211,26 +1209,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1238,13 +1238,13 @@ "object_id": "0474e5f1-3b49-5ec5-8186-a1ffc6952f37", "file_type": "VCF", "file_access": "controlled", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", @@ -1253,16 +1253,14 @@ "info": {} } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "0e1e3c924b37d90c99bb04f719a56ce4", "size": 5565355, - "last_modified": null, "index_file": { "object_id": "2910e07c-268b-57a5-877a-0cec35298319", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1abed463bc92e0d8486699b37af73737", "size": 546153, "data_type": "AlignedReads" @@ -1276,26 +1274,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1303,14 +1303,14 @@ "object_id": "54abbcb5-485f-5acd-88e9-50261e0c8e17", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", @@ -1318,16 +1318,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "19b56e546bdcdb7f68dfd603b1b7a8d7", "size": 101384873, - "last_modified": null, "index_file": { "object_id": "425898fd-35d3-5ee2-ac00-50dfd724f4c6", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "a6113c54193fd7539dffd06f50f04779", "size": 1541330, "data_type": "AlignedReads" @@ -1341,26 +1339,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1368,14 +1368,14 @@ "object_id": "0d7ee358-01fe-59ce-b62b-0663bec8596d", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "info": {}, @@ -1383,16 +1383,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "c102abc0bfa0b63b2e1ea2ae62e18b80", "size": 72850649, - "last_modified": null, "index_file": { "object_id": "04c24136-c19d-5042-a3cd-36f63c7b4d6b", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "156880f52b94b3119afa3ac6dd7b8ced", "size": 1502241, "data_type": "AlignedReads" @@ -1406,26 +1404,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1433,14 +1433,14 @@ "object_id": "432779b5-95db-5caf-a529-85a36a707c04", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", @@ -1448,16 +1448,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "93b219c94687e12b5fda564399fa3319", "size": 23763, - "last_modified": null, "index_file": { "object_id": "51306186-092d-5e3f-bb3c-028873052eb2", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "68d76af7c7b9b7c00fa0438101739798", "size": 10691, "data_type": "AlignedReads" @@ -1471,26 +1469,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1498,14 +1498,14 @@ "object_id": "4f2bee69-e66e-5af5-856a-a0f3c34b12ff", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "info": {}, @@ -1513,16 +1513,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "6ebd4aa8a821da40955d7fcb879be82a", "size": 18226, - "last_modified": null, "index_file": { "object_id": "e5d387a1-3b51-5526-ae22-270df2e1fd2f", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "96b05a083d371ae8317651ceffb0f48b", "size": 6134, "data_type": "AlignedReads" @@ -1536,26 +1534,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1563,14 +1563,14 @@ "object_id": "3f571505-3bea-5065-956a-4b5a793ad1ed", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "info": {}, @@ -1578,16 +1578,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "f3a99bd737b3b416aa34e131b939b2a0", "size": 161511, - "last_modified": null, "index_file": { "object_id": "e8569e7c-6ae4-5aeb-ae50-b94366aae5df", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "cb856f2d5837c5f065c7c598dcd795b5", "size": 19769, "data_type": "AlignedReads" @@ -1601,26 +1599,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", - "submitter_sample_id": "PCSI_0233_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125730", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0233_Pa_P_526", + "samples": [ + { + "sample_id": "SA533729", + "submitter_sample_id": "PCSI_0233_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1628,14 +1628,14 @@ "object_id": "e122b153-c98b-530d-879a-11871a427d1b", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "info": {}, @@ -1643,16 +1643,14 @@ "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "4824d68baf7c2b06f5a652bb5462b9b4", "size": 58091383, - "last_modified": null, "index_file": { "object_id": "b26a535c-3531-55ad-8725-17aa6afb22c2", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "894fa70742c16ed7f845abd68609f20c", "size": 1524609, "data_type": "AlignedReads" @@ -1666,26 +1664,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", - "submitter_sample_id": "ASHPC_0022_Pa_P", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125791", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "ASHPC_0022_Pa_P", + "samples": [ + { + "sample_id": "SA533758", + "submitter_sample_id": "ASHPC_0022_Pa_P", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1693,14 +1693,14 @@ "object_id": "36ede5a5-fcde-5b6a-95c1-af28321fbd39", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "info": {}, @@ -1708,16 +1708,14 @@ "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "0502184e75c3a3fb83e2aa6ab536e78a", "size": 59407, - "last_modified": null, "index_file": { "object_id": "b9800f14-13d5-55cf-9383-a65b5446116a", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1ba234eb0c457dba2942215d3fc4741a", "size": 2961, "data_type": "AlignedReads" @@ -1731,26 +1729,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", - "submitter_sample_id": "ASHPC_0022_Pa_P", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125791", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "ASHPC_0022_Pa_P", + "samples": [ + { + "sample_id": "SA533758", + "submitter_sample_id": "ASHPC_0022_Pa_P", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1758,14 +1758,14 @@ "object_id": "5c1fcacb-7c66-53a6-8fb6-93d1281e55b8", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", @@ -1773,16 +1773,14 @@ "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "bafebb1c613c348c696dc8f940ce1039", "size": 2964, - "last_modified": null, "index_file": { "object_id": "52db8e51-5b3e-56ce-b939-0a96b4dfbce3", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "725becdd11e63c371e57a987420846bd", "size": 4783, "data_type": "AlignedReads" @@ -1796,26 +1794,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", - "submitter_sample_id": "ASHPC_0022_Pa_P", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125791", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "ASHPC_0022_Pa_P", + "samples": [ + { + "sample_id": "SA533758", + "submitter_sample_id": "ASHPC_0022_Pa_P", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1823,14 +1823,14 @@ "object_id": "e704d3f5-1274-5cdd-924f-609159e121cc", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", @@ -1838,16 +1838,14 @@ "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "37ee70b282788eb0660f7d965f355898", "size": 1704966, - "last_modified": null, "index_file": { "object_id": "6b32cfec-e82c-5528-b908-8e00ae927578", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "91dece67c383a67123caae382a130cb9", "size": 303747, "data_type": "AlignedReads" @@ -1861,26 +1859,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", - "submitter_sample_id": "ASHPC_0022_Pa_P", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125791", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "ASHPC_0022_Pa_P", + "samples": [ + { + "sample_id": "SA533758", + "submitter_sample_id": "ASHPC_0022_Pa_P", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1888,14 +1888,14 @@ "object_id": "9727c61d-3871-54f1-a65a-2d1d91acd242", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "b8e09611-225a-4de6-87dc-104753072252", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "PACA-CA", "experiment": { "analysisId": "b8e09611-225a-4de6-87dc-104753072252", "info": {}, @@ -1903,16 +1903,14 @@ "matchedNormalSampleSubmitterId": "PCSI_0083_Du_R" } }, - "files": { + "file": { "name": "536dedba-46c4-4a21-b112-13c030b13069.consensus.20161006.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "01e54034c70b85812e9aac58f2525594", "size": 329863, - "last_modified": null, "index_file": { "object_id": "08519ab5-8665-58dc-8476-1c40cae5b789", "name": "536dedba-46c4-4a21-b112-13c030b13069.consensus.20161006.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "5b83369174136b88f16a08ab87bb67f7", "size": 148925, "data_type": "AlignedReads" @@ -1926,26 +1924,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/ca8479dc-8f00-5757-b433-c3cf5b76f255" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO35442", + "donor_id": "DO35442", "gender": "Female", "submitter_donor_id": "PCSI_0083", - "specimens": { - "id": "SP125732", - "specimen_type": "Primary tumour - other", - "submitter_specimen_id": "PCSI_0083_Pa_P_526", - "samples": { - "id": "SA520221", - "submitter_sample_id": "PCSI_0083_Pa_P_526", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP125732", + "specimen_type": "Primary tumour - other", + "submitter_specimen_id": "PCSI_0083_Pa_P_526", + "samples": [ + { + "sample_id": "SA520221", + "submitter_sample_id": "PCSI_0083_Pa_P_526", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -1953,14 +1953,14 @@ "object_id": "316612a4-4c23-5434-976a-810f8cde06b7", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "MALY-DE", "analysis": { "analysis_id": "0a1df2a2-029d-48cc-839c-0a7c89ff972f", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "info": {}, - "study_id": "MALY-DE", "experiment": { "analysisId": "0a1df2a2-029d-48cc-839c-0a7c89ff972f", "variantCallingTool": "MUSE variant call pipeline", @@ -1968,16 +1968,14 @@ "matchedNormalSampleSubmitterId": "control_4128852" } }, - "files": { + "file": { "name": "e2fa7251-507e-4d76-95a3-a228adc3885a.MUSE_1-0rc-vcf.20151107.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "f83b0f9099e8594b589440819ab119cd", "size": 273787, - "last_modified": null, "index_file": { "object_id": "a9c47630-491b-5587-a58d-6736734d6163", "name": "e2fa7251-507e-4d76-95a3-a228adc3885a.MUSE_1-0rc-vcf.20151107.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "f5629a05fe5b8dd6a63381de259aa935", "size": 146924, "data_type": "AlignedReads" @@ -1991,26 +1989,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/95cbd8c1-1cb1-5c18-8c2d-6f54867d7b72" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO221129", + "donor_id": "DO221129", "gender": "Female", "submitter_donor_id": "4128852", - "specimens": { - "id": "SP124981", - "specimen_type": "Primary tumour - lymph node", - "submitter_specimen_id": "tumor_4128852", - "samples": { - "id": "SA557689", - "submitter_sample_id": "tumor_4128852", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP124981", + "specimen_type": "Primary tumour - lymph node", + "submitter_specimen_id": "tumor_4128852", + "samples": [ + { + "sample_id": "SA557689", + "submitter_sample_id": "tumor_4128852", + "sample_type": "DNA" + } + ] } - } + ] } ] }, @@ -2018,13 +2018,13 @@ "object_id": "7b32b455-d8e2-5e51-bcad-dad7945c8b15", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "MALY-DE", "analysis": { "analysis_id": "327c74cd-ee0b-4885-bafd-741e17b8b163", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "MALY-DE", + "analysis_state": "PUBLISHED", "info": {}, "experiment": { "analysisId": "327c74cd-ee0b-4885-bafd-741e17b8b163", @@ -2033,16 +2033,14 @@ "matchedNormalSampleSubmitterId": "control_4166503" } }, - "files": { + "file": { "name": "866ecfe7-caa6-4565-9418-6b9d6c8a3b43.consensus.20161006.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "020a444fb10ec02b40e141aa9cd0de87", "size": 38671, - "last_modified": null, "index_file": { "object_id": "4ba753ee-bbb7-5f57-9cda-6e15c519f834", "name": "866ecfe7-caa6-4565-9418-6b9d6c8a3b43.consensus.20161006.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "7c606af5d37d3788850da1a5f1705feb", "size": 19306, "data_type": "AlignedReads" @@ -2056,26 +2054,28 @@ "name": "collaboratory", "type": "S3", "country": "CA", - "url": "##COLLAB_REPO_URL##", - "data_path": "/oicr.icgc/data", - "metadata_path": "/oicr.icgc.meta/metadata/46c2fec8-fd4a-5b62-8819-647bf23ac15c" + "url": "##COLLAB_REPO_URL##" } ], "donors": [ { - "id": "DO52662", + "donor_id": "DO52662", "gender": "Female", "submitter_donor_id": "4166503", - "specimens": { - "id": "SP116638", - "specimen_type": "Primary tumour - lymph node", - "submitter_specimen_id": "tumor_4166503", - "samples": { - "id": "SA542043", - "submitter_sample_id": "tumor_4166503", - "sample_type": "DNA" + "specimens": [ + { + "specimen_id": "SP116638", + "specimen_type": "Primary tumour - lymph node", + "submitter_specimen_id": "tumor_4166503", + "samples": [ + { + "sample_id": "SA542043", + "submitter_sample_id": "tumor_4166503", + "sample_type": "DNA" + } + ] } - } + ] } ] } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java index add6d816..41c8da93 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverter.java @@ -1,23 +1,18 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.DocumentConverterHelper.getDonors; import bio.overture.maestro.domain.entities.indexing.Repository; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDocument; -import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDonor; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricFile; import bio.overture.maestro.domain.entities.metadata.repository.StudyRepository; import bio.overture.maestro.domain.entities.metadata.study.Analysis; import bio.overture.maestro.domain.entities.metadata.study.File; -import bio.overture.maestro.domain.entities.metadata.study.Sample; -import bio.overture.maestro.domain.entities.metadata.study.Specimen; +import java.util.List; +import java.util.stream.Collectors; import lombok.NonNull; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.List; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.FileCentricDocumentConverter.getMetadataFileId; -import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; @Slf4j @UtilityClass @@ -27,25 +22,25 @@ static List fromAnalysis(Analysis analysis, StudyReposi return List.of(convertAnalysis(analysis, repository)); } - static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepository repository){ - val metadataFileId = getMetadataFileId(analysis); - val doc = AnalysisCentricDocument.builder() + static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepository repository) { + val doc = + AnalysisCentricDocument.builder() .analysisId(analysis.getAnalysisId()) .analysisState(analysis.getAnalysisState()) .analysisType(analysis.getAnalysisType().getName()) .analysisVersion(analysis.getAnalysisType().getVersion()) .studyId(analysis.getStudyId()) .donors(getDonors(analysis)) - .repositories(List.of(Repository.builder() - .type(repository.getStorageType().name().toUpperCase()) - .organization(repository.getOrganization()) - .name(repository.getName()) - .code(repository.getCode()) - .country(repository.getCountry()) - .url(repository.getUrl()) - .dataPath(repository.getDataPath()) - .metadataPath(repository.getMetadataPath() + "/" + metadataFileId) - .build())) + .repositories( + List.of( + Repository.builder() + .type(repository.getStorageType().name().toUpperCase()) + .organization(repository.getOrganization()) + .name(repository.getName()) + .code(repository.getCode()) + .country(repository.getCountry()) + .url(repository.getUrl()) + .build())) .experiment(analysis.getExperiment()) .files(buildAnalysisCentricFiles(analysis.getFiles())) .build(); @@ -53,98 +48,22 @@ static AnalysisCentricDocument convertAnalysis(Analysis analysis, StudyRepositor return doc; } - public static List getDonors(@NonNull Analysis analysis){ - val groupedByDonorMap = analysis.getSamples() - .stream() - .map(sample -> extractDonor(sample)) - .collect(Collectors.groupingBy(AnalysisCentricDonor :: getId, Collectors.toList())); - return groupedByDonorMap.values() - .stream() - .collect(Collectors.toList()) - .stream() - .map(donorList -> mergeDonorBySpecimen(donorList)) - .collect(Collectors.toList()); + private static List buildAnalysisCentricFiles(@NonNull List files) { + return files.stream().map(AnalysisCentricDocumentConverter::fromFile).collect(Collectors.toList()); } - /** - * Groups specimens belonging to a AnalysisCentricDonor - * @param list a grouped list with each AnalysisCentricDonor element having exactly one Specimen - * @return a fully assembled AnalysisCentticDonor object with a list of specimens that belongs to the current donor - */ - private static AnalysisCentricDonor mergeDonorBySpecimen(@NonNull List list){ - - checkNotFound(list.size() > 0, - "Failed to merge AnalysisCentricDonor by specimen: donor list is empty."); - - // Every element in list has the same donor, so just use the first donor - val anyDonor = list.get(0); - - checkNotFound(anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, - "Failed to merge AnalysisCentricDonor by specimen: donor doesn't have specimen,"); - - val specimenList = list.stream() - // One donor only has one specimen in list - .map(analysisCentricDonor -> analysisCentricDonor.getSpecimens().get(0)) - .collect(Collectors.toList()); - - return AnalysisCentricDonor.builder() - .id(anyDonor.getId()) - .submitterDonorId(anyDonor.getSubmitterDonorId()) - .gender(anyDonor.getGender()) - .specimens(specimenList) - .build(); - } - - private static AnalysisCentricDonor extractDonor(@NonNull Sample sample){ - val donor = sample.getDonor(); - val specimen = sample.getSpecimen(); - return AnalysisCentricDonor.builder() - .id(donor.getDonorId()) - .gender(donor.getGender()) - .submitterDonorId(donor.getSubmitterDonorId()) - .specimens(buildSpecimen(specimen, sample)) - .build(); - } - - /** - * Converts metadata Specimen pojo and metadata Sample pojo to an indexing Specimen pojo. - * @param specimen - * @param sample - * @return - */ - private static List buildSpecimen(@NonNull Specimen specimen, - @NonNull Sample sample){ - return List.of(bio.overture.maestro.domain.entities.indexing.Specimen.builder() - .id(specimen.getSpecimenId()) - .submitterSpecimenId(specimen.getSubmitterSpecimenId()) - .specimenType(specimen.getSpecimenType()) - .specimenTissueSource(specimen.getSpecimenTissueSource()) - .tumourNormalDesignation(specimen.getTumourNormalDesignation()) - .samples(bio.overture.maestro.domain.entities.indexing.Sample.builder() - .id(sample.getSampleId()) - .matchedNormalSubmitterSampleId(sample.getMatchedNormalSubmitterSampleId()) - .submitterSampleId(sample.getSubmitterSampleId()) - .sampleType(sample.getSampleType()) - .build()) - .build()); - } - - private static List buildAnalysisCentricFiles(@NonNull List files){ - return files.stream() - .map(file -> fromFile(file)) - .collect(Collectors.toList()); - } - - private static AnalysisCentricFile fromFile(@NonNull File file){ - return AnalysisCentricFile.builder() - .id(file.getObjectId()) - .fileAccess(file.getFileAccess()) - .dataType(file.getDataType()) - .md5Sum(file.getFileMd5sum()) - .name(file.getFileName()) - .size(file.getFileSize()) - .fileType(file.getFileType()) - .build(); + private static AnalysisCentricFile fromFile(@NonNull File file) { + val fileDoc = AnalysisCentricFile.builder() + .objectId(file.getObjectId()) + .fileAccess(file.getFileAccess()) + .dataType(file.getDataType()) + .md5Sum(file.getFileMd5sum()) + .name(file.getFileName()) + .size(file.getFileSize()) + .fileType(file.getFileType()) + .build(); + fileDoc.replaceInfo(file.getInfo()); + return fileDoc; } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java index 4ddb7a85..bece16c7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DefaultIndexer.java @@ -17,6 +17,11 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.fromAnalysis; +import static bio.overture.maestro.domain.api.ExclusionRulesEvaluator.shouldExcludeAnalysis; +import static bio.overture.maestro.domain.utility.Exceptions.wrapWithIndexerException; +import static java.text.MessageFormat.format; + import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.message.*; @@ -41,744 +46,833 @@ import io.vavr.Tuple2; import io.vavr.control.Either; import io.vavr.control.Try; +import java.util.*; +import java.util.stream.Collectors; +import javax.inject.Inject; import lombok.*; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; -import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; -import static bio.overture.maestro.domain.api.AnalysisCentricDocumentConverter.fromAnalysis; -import static bio.overture.maestro.domain.api.ExclusionRulesEvaluator.shouldExcludeAnalysis; -import static bio.overture.maestro.domain.utility.Exceptions.wrapWithIndexerException; -import static java.text.MessageFormat.format; @Slf4j class DefaultIndexer implements Indexer { - static final String STUDY_ID = "studyId"; - static final String REPO_CODE = "repoCode"; - static final String ANALYSIS_ID = "analysisId"; - static final String ERR = "err"; - - private static final String REPO_URL = "repoUrl"; - private static final String FAILURE_DATA = "failureData"; - private static final String CONFLICTS = "conflicts"; - - private boolean isFileCentricEnabled; - private boolean isAnalysisCentricEnabled; - - private final FileCentricIndexAdapter fileCentricIndexAdapter; - private final AnalysisCentricIndexAdapter analysisCentricIndexAdapter; - private final StudyDAO studyDAO; - private final StudyRepositoryDAO studyRepositoryDao; - private final ExclusionRulesDAO exclusionRulesDAO; - private final Notifier notifier; - - @Inject - DefaultIndexer(FileCentricIndexAdapter fileCentricIndexAdapter, - AnalysisCentricIndexAdapter analysisCentricIndexAdapter, - StudyDAO studyDAO, - StudyRepositoryDAO studyRepositoryDao, - ExclusionRulesDAO exclusionRulesDAO, - Notifier notifier, - IndexEnabledProperties indexEnabled - ) { - this.fileCentricIndexAdapter = fileCentricIndexAdapter; - this.analysisCentricIndexAdapter = analysisCentricIndexAdapter; - this.studyDAO = studyDAO; - this.studyRepositoryDao = studyRepositoryDao; - this.exclusionRulesDAO = exclusionRulesDAO; - this.notifier = notifier; - this.isAnalysisCentricEnabled = indexEnabled.isAnalysisCentricEnabled(); - this.isFileCentricEnabled = indexEnabled.isFileCentricEnabled(); - } - - @Override - public Flux indexAnalysis(@NonNull IndexAnalysisCommand command) { - List > monos = new ArrayList<>(); - if (isFileCentricEnabled) { - monos.add(indexAnalysisToFileCentric(command)); - } - if(isAnalysisCentricEnabled){ - monos.add(indexAnalysisToAnalysisCentric(command)); - } - return Flux.merge(monos); - } - - public Mono indexAnalysisToAnalysisCentric(@NonNull IndexAnalysisCommand indexAnalysisCommand){ - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - - return prepareTuple(indexAnalysisCommand) - .flatMap(this :: getAnalysisCentricDocuments) - .flatMap(this :: batchUpsertAnalysesAndCollectFailtures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier)); - } - - public Mono indexAnalysisToFileCentric(@NonNull IndexAnalysisCommand indexAnalysisCommand) { - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - - return prepareTuple(indexAnalysisCommand) - .flatMap(this :: getFileCentricDocuments) - .flatMap(this :: batchUpsertFilesAndCollectFailures) - // this handles exceptions that were handled already and avoids them getting to the generic handler - // because we know if we get this exception it was already logged and notified so we don't want that again. - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - // this handler handles uncaught exceptions - .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier)); - } - - @Override - public Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand) { - val analysisIdentifier = removeAnalysisCommand.getAnalysisIdentifier(); - return this.fileCentricIndexAdapter.removeAnalysisFiles(analysisIdentifier.getAnalysisId()) - .thenReturn(IndexResult.builder().successful(true).build()) - .onErrorResume((e) -> handleRemoveAnalysisError(analysisIdentifier)); - } - - @Override - public Flux indexStudy(@NonNull IndexStudyCommand command) { - List > monos = new ArrayList<>(); - - Mono, StudyAndRepository>> mono = prepareStudyAndRepo(command) - .flatMap(studyAndRepository -> getFilteredAnalyses - (studyAndRepository.getStudyRepository().getUrl(), studyAndRepository.getStudy().getStudyId()) - .map(analyses -> - new Tuple2<>(analyses, studyAndRepository) - )); - - if (isFileCentricEnabled) { - monos.add(indexStudyToFileCentric(command, mono)); - } - if(isAnalysisCentricEnabled){ - monos.add(indexStudyToAnalysisCentric(command, mono)); - } - return Flux.merge(monos); - } - - private Mono indexStudyToFileCentric(@NonNull IndexStudyCommand command, - @NonNull Mono, StudyAndRepository>> tuple2) { - log.trace("in indexStudyToFileCentric, args: {} ", command); - return tuple2 - .map(t -> buildFileCentricDocuments(t._2().getStudyRepository(), t._1())) - .flatMap(this :: batchUpsertFilesAndCollectFailures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexStudyError(e, command.getStudyId(), - command.getRepositoryCode())); - } - - private Mono indexStudyToAnalysisCentric(@NonNull IndexStudyCommand command, - @NonNull Mono, StudyAndRepository>> tuple2) { - log.trace("in indexStudyToAnalysisCentric, args: {} ", command); - return tuple2 - .map( t -> buildAnalysisCentricDocuments(t._2().getStudyRepository(), t._1())) - .flatMap(this :: batchUpsertAnalysesAndCollectFailtures) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexStudyError(e, command.getStudyId(), - command.getRepositoryCode())); - } - - @Override - public Mono indexRepository(@NonNull IndexStudyRepositoryCommand command) { - log.trace("in indexRepository, args: {} ", command); - return tryGetStudyRepository(command.getRepositoryCode()) - .flatMapMany(this :: getAllStudies) - .flatMap(studyAndRepository -> - this.indexStudy(IndexStudyCommand.builder() - .studyId(studyAndRepository.getStudy().getStudyId()) - .repositoryCode(studyAndRepository.studyRepository.getCode()) - .build()) - // I had to put this block inside this flatMap to allow these operations to bubble up their exceptions - // to this onErrorResume handler without interrupting the main flux, and terminating it with error signals. - // for example if fetchAnalyses down stream throws error for a studyId the studies flux will - // continue emitting studies + static final String STUDY_ID = "studyId"; + static final String REPO_CODE = "repoCode"; + static final String ANALYSIS_ID = "analysisId"; + static final String ERR = "err"; + + private static final String REPO_URL = "repoUrl"; + private static final String FAILURE_DATA = "failureData"; + private static final String CONFLICTS = "conflicts"; + private static final String FILE_CENTRIC_INDEX = "file_centric_1.0"; + private static final String ANALYSIS_CENTRIC_INDEX = "analysis_centric_1.0"; + + private boolean isFileCentricEnabled; + private boolean isAnalysisCentricEnabled; + + private final FileCentricIndexAdapter fileCentricIndexAdapter; + private final AnalysisCentricIndexAdapter analysisCentricIndexAdapter; + private final StudyDAO studyDAO; + private final StudyRepositoryDAO studyRepositoryDao; + private final ExclusionRulesDAO exclusionRulesDAO; + private final Notifier notifier; + + @Inject + DefaultIndexer( + FileCentricIndexAdapter fileCentricIndexAdapter, + AnalysisCentricIndexAdapter analysisCentricIndexAdapter, + StudyDAO studyDAO, + StudyRepositoryDAO studyRepositoryDao, + ExclusionRulesDAO exclusionRulesDAO, + Notifier notifier, + IndexEnabledProperties indexEnabled) { + this.fileCentricIndexAdapter = fileCentricIndexAdapter; + this.analysisCentricIndexAdapter = analysisCentricIndexAdapter; + this.studyDAO = studyDAO; + this.studyRepositoryDao = studyRepositoryDao; + this.exclusionRulesDAO = exclusionRulesDAO; + this.notifier = notifier; + this.isAnalysisCentricEnabled = indexEnabled.isAnalysisCentricEnabled(); + this.isFileCentricEnabled = indexEnabled.isFileCentricEnabled(); + } + + @Override + public Flux indexAnalysis(@NonNull IndexAnalysisCommand command) { + List> monos = new ArrayList<>(); + if (isFileCentricEnabled) { + monos.add(indexAnalysisToFileCentric(command)); + } + if (isAnalysisCentricEnabled) { + monos.add(indexAnalysisToAnalysisCentric(command)); + } + return Flux.merge(monos); + } + + public Mono indexAnalysisToAnalysisCentric( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + + return prepareTuple(indexAnalysisCommand) + .flatMap(this::getAnalysisCentricDocuments) + .flatMap(this::batchUpsertAnalysesAndCollectFailtures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> handleIndexAnalysisError(e, analysisIdentifier, ANALYSIS_CENTRIC_INDEX)); + } + + public Mono indexAnalysisToFileCentric( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + + return prepareTuple(indexAnalysisCommand) + .flatMap(this::getFileCentricDocuments) + .flatMap(this::batchUpsertFilesAndCollectFailures) + // this handles exceptions that were handled already and avoids them getting to the generic + // handler + // because we know if we get this exception it was already logged and notified so we don't + // want that again. + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + // this handler handles uncaught exceptions + .onErrorResume((e) -> handleIndexAnalysisError(e, analysisIdentifier, FILE_CENTRIC_INDEX)); + } + + @Override + public Flux removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand) { + val analysisIdentifier = removeAnalysisCommand.getAnalysisIdentifier(); + List> monos = new ArrayList<>(); + + if (isFileCentricEnabled) { + val mono = + this.fileCentricIndexAdapter + .removeAnalysisFiles(analysisIdentifier.getAnalysisId()) + .thenReturn( + IndexResult.builder().indexName(FILE_CENTRIC_INDEX).successful(true).build()) + .onErrorResume( + (e) -> handleRemoveAnalysisError(FILE_CENTRIC_INDEX, analysisIdentifier)); + monos.add(mono); + } + + if (isAnalysisCentricEnabled) { + val mono = + this.analysisCentricIndexAdapter + .removeAnalysisDocs(analysisIdentifier.getAnalysisId()) + .thenReturn( + IndexResult.builder().indexName(ANALYSIS_CENTRIC_INDEX).successful(true).build()) + .onErrorResume( + (e) -> handleRemoveAnalysisError(ANALYSIS_CENTRIC_INDEX, analysisIdentifier)); + monos.add(mono); + } + return Flux.merge(monos); + } + + @Override + public Flux indexStudy(@NonNull IndexStudyCommand command) { + List> monos = new ArrayList<>(); + + Mono, StudyAndRepository>> mono = + prepareStudyAndRepo(command) + .flatMap( + studyAndRepository -> + getFilteredAnalyses( + studyAndRepository.getStudyRepository().getUrl(), + studyAndRepository.getStudy().getStudyId()) + .map(analyses -> new Tuple2<>(analyses, studyAndRepository))); + + if (isFileCentricEnabled) { + monos.add(indexStudyToFileCentric(command, mono)); + } + if (isAnalysisCentricEnabled) { + monos.add(indexStudyToAnalysisCentric(command, mono)); + } + return Flux.merge(monos); + } + + private Mono indexStudyToFileCentric( + @NonNull IndexStudyCommand command, + @NonNull Mono, StudyAndRepository>> tuple2) { + log.trace("in indexStudyToFileCentric, args: {} ", command); + return tuple2 + .map(t -> buildFileCentricDocuments(t._2().getStudyRepository(), t._1())) + .flatMap(this::batchUpsertFilesAndCollectFailures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> + handleIndexStudyError( + e, command.getStudyId(), command.getRepositoryCode(), FILE_CENTRIC_INDEX)); + } + + private Mono indexStudyToAnalysisCentric( + @NonNull IndexStudyCommand command, + @NonNull Mono, StudyAndRepository>> tuple2) { + log.trace("in indexStudyToAnalysisCentric, args: {} ", command); + return tuple2 + .map(t -> buildAnalysisCentricDocuments(t._2().getStudyRepository(), t._1())) + .flatMap(this::batchUpsertAnalysesAndCollectFailtures) + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume( + (e) -> + handleIndexStudyError( + e, command.getStudyId(), command.getRepositoryCode(), ANALYSIS_CENTRIC_INDEX)); + } + + @Override + public Mono indexRepository(@NonNull IndexStudyRepositoryCommand command) { + log.trace("in indexRepository, args: {} ", command); + return tryGetStudyRepository(command.getRepositoryCode()) + .flatMapMany(this::getAllStudies) + .flatMap( + studyAndRepository -> + this.indexStudy( + IndexStudyCommand.builder() + .studyId(studyAndRepository.getStudy().getStudyId()) + .repositoryCode(studyAndRepository.studyRepository.getCode()) + .build()) + // I had to put this block inside this flatMap to allow these operations to bubble up + // their exceptions + // to this onErrorResume handler without interrupting the main flux, and terminating it + // with error signals. + // for example if fetchAnalyses down stream throws error for a studyId the studies flux + // will + // continue emitting studies ) - .onErrorResume(IndexerException.class, (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) - .onErrorResume((e) -> handleIndexRepositoryError(e, command.getRepositoryCode())) - .reduce(this :: reduceIndexResult); - } - - @Override - public void addRule(AddRuleCommand addRuleCommand) { - throw new IndexerException("not implemented yet"); - } - - @Override - public void deleteRule(DeleteRuleCommand deleteRuleCommand) { - throw new IndexerException("not implemented yet"); - } - - @Override - public List getAllRules() { - throw new IndexerException("not implemented yet"); - } - - /* **************** * - * Private Methods * - * **************** */ - private Mono prepareStudyAndRepo(@NonNull IndexStudyCommand command){ - return tryGetStudyRepository(command.getRepositoryCode()) - .map(filesRepository -> toStudyAndRepositoryTuple(command, filesRepository)); - } - - private Mono prepareTuple(@NonNull IndexAnalysisCommand indexAnalysisCommand){ - val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); - return tryGetStudyRepository(analysisIdentifier.getRepositoryCode()) - .map(studyRepository -> - buildStudyAnalysisRepoTuple(analysisIdentifier, studyRepository)); - } - - private Mono handleRemoveAnalysisError(@NonNull AnalysisIdentifier analysisIdentifier) { - val failureInfo = Map.of(ANALYSIS_ID, Set.of(analysisIdentifier.getAnalysisId())); - this.notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_REMOVE_ANALYSIS, failureInfo)); - return Mono.just(IndexResult.builder() + .onErrorResume( + IndexerException.class, + (ex) -> Mono.just(this.convertIndexerExceptionToIndexResult(ex))) + .onErrorResume((e) -> handleIndexRepositoryError(e, command.getRepositoryCode())) + .reduce(this::reduceIndexResult); + } + + @Override + public void addRule(AddRuleCommand addRuleCommand) { + throw new IndexerException("not implemented yet"); + } + + @Override + public void deleteRule(DeleteRuleCommand deleteRuleCommand) { + throw new IndexerException("not implemented yet"); + } + + @Override + public List getAllRules() { + throw new IndexerException("not implemented yet"); + } + + /* **************** * + * Private Methods * + * **************** */ + private Mono prepareStudyAndRepo(@NonNull IndexStudyCommand command) { + return tryGetStudyRepository(command.getRepositoryCode()) + .map(filesRepository -> toStudyAndRepositoryTuple(command, filesRepository)); + } + + private Mono prepareTuple( + @NonNull IndexAnalysisCommand indexAnalysisCommand) { + val analysisIdentifier = indexAnalysisCommand.getAnalysisIdentifier(); + return tryGetStudyRepository(analysisIdentifier.getRepositoryCode()) + .map(studyRepository -> buildStudyAnalysisRepoTuple(analysisIdentifier, studyRepository)); + } + + private Mono handleRemoveAnalysisError( + String indexName, @NonNull AnalysisIdentifier analysisIdentifier) { + val failureInfo = Map.of(ANALYSIS_ID, Set.of(analysisIdentifier.getAnalysisId())); + this.notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_REMOVE_ANALYSIS, failureInfo)); + return Mono.just( + IndexResult.builder() + .indexName(indexName) .failureData(FailureData.builder().failingIds(failureInfo).build()) - .build() - ); - } - - private Mono tryGetStudyRepository(@NonNull String repoCode) { - return this.studyRepositoryDao.getFilesRepository(repoCode) - .onErrorMap((e) -> handleGetStudyRepoError(repoCode, e)); - } - - private Throwable handleGetStudyRepoError(@NonNull String repoCode, Throwable e) { - val failure = Map.of(REPO_CODE, Set.of(repoCode)); - this.notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_REPOSITORY, failure)); - return wrapWithIndexerException(e, "failed getting repository", FailureData.builder() - .failingIds(failure).build()); - } - - private Flux getAllStudies(StudyRepository studyRepository) { - return this.studyDAO.getStudies(GetAllStudiesCommand.builder() - .filesRepositoryBaseUrl(studyRepository.getUrl()) - .build() - ).onErrorMap((e) -> handleGetStudiesError(studyRepository, e)) + .build()); + } + + private Mono tryGetStudyRepository(@NonNull String repoCode) { + return this.studyRepositoryDao + .getFilesRepository(repoCode) + .onErrorMap((e) -> handleGetStudyRepoError(repoCode, e)); + } + + private Throwable handleGetStudyRepoError(@NonNull String repoCode, Throwable e) { + val failure = Map.of(REPO_CODE, Set.of(repoCode)); + this.notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_REPOSITORY, failure)); + return wrapWithIndexerException( + e, "failed getting repository", FailureData.builder().failingIds(failure).build()); + } + + private Flux getAllStudies(StudyRepository studyRepository) { + return this.studyDAO + .getStudies( + GetAllStudiesCommand.builder().filesRepositoryBaseUrl(studyRepository.getUrl()).build()) + .onErrorMap((e) -> handleGetStudiesError(studyRepository, e)) .map(study -> toStudyAndRepository(studyRepository, study)); - } - - @NotNull - private Throwable handleGetStudiesError(StudyRepository studyRepository, Throwable e) { - val errMsg = getErrorMessageOrType(e); - notifyFailedToFetchStudies(studyRepository.getCode(), errMsg); - return wrapWithIndexerException(e, "fetch studies failed", FailureData.builder() + } + + @NotNull + private Throwable handleGetStudiesError(StudyRepository studyRepository, Throwable e) { + val errMsg = getErrorMessageOrType(e); + notifyFailedToFetchStudies(studyRepository.getCode(), errMsg); + return wrapWithIndexerException( + e, + "fetch studies failed", + FailureData.builder() .failingIds(Map.of(REPO_CODE, Set.of(studyRepository.getCode()))) .build()); - } - - private void notifyFailedToFetchStudies(String code, String message) { - val attrs = Map.of(REPO_CODE, code, ERR, message); - val notification = - new IndexerNotification(NotificationName.FETCH_REPO_STUDIES_FAILED, attrs); - notifier.notify(notification); - } - - private StudyAndRepository toStudyAndRepository(StudyRepository studyRepository, Study studyEither) { - return StudyAndRepository.builder() - .study(studyEither) - .studyRepository(studyRepository) - .build(); - } - - private StudyAndRepository toStudyAndRepositoryTuple(@NonNull IndexStudyCommand indexStudyCommand, - @NonNull StudyRepository filesRepository) { - return StudyAndRepository.builder() - .study(Study.builder().studyId(indexStudyCommand.getStudyId()).build()) - .studyRepository(filesRepository).build(); - } - - private Mono> getFilteredAnalyses(@NonNull String repoBaseUrl, @NonNull String studyId) { - return fetchAnalyses(repoBaseUrl, studyId).flatMap(this :: getExclusionRulesAndFilter); - } - - private Mono> fetchAnalyses(@NonNull String studyRepositoryBaseUrl, @NonNull String studyId) { - val command = GetStudyAnalysesCommand.builder() + } + + private void notifyFailedToFetchStudies(String code, String message) { + val attrs = Map.of(REPO_CODE, code, ERR, message); + val notification = new IndexerNotification(NotificationName.FETCH_REPO_STUDIES_FAILED, attrs); + notifier.notify(notification); + } + + private StudyAndRepository toStudyAndRepository( + StudyRepository studyRepository, Study studyEither) { + return StudyAndRepository.builder().study(studyEither).studyRepository(studyRepository).build(); + } + + private StudyAndRepository toStudyAndRepositoryTuple( + @NonNull IndexStudyCommand indexStudyCommand, @NonNull StudyRepository filesRepository) { + return StudyAndRepository.builder() + .study(Study.builder().studyId(indexStudyCommand.getStudyId()).build()) + .studyRepository(filesRepository) + .build(); + } + + private Mono> getFilteredAnalyses( + @NonNull String repoBaseUrl, @NonNull String studyId) { + return fetchAnalyses(repoBaseUrl, studyId).flatMap(this::getExclusionRulesAndFilter); + } + + private Mono> fetchAnalyses( + @NonNull String studyRepositoryBaseUrl, @NonNull String studyId) { + val command = + GetStudyAnalysesCommand.builder() .filesRepositoryBaseUrl(studyRepositoryBaseUrl) .studyId(studyId) .build(); - return this.studyDAO - .getStudyAnalyses(command) - .onErrorMap(e -> handleFetchAnalysesError(studyRepositoryBaseUrl, studyId, command, e)); - } - - private Throwable handleFetchAnalysesError(String studyRepositoryBaseUrl, String studyId, - GetStudyAnalysesCommand command, Throwable e) { - notifyStudyFetchingError(studyId, studyRepositoryBaseUrl, e.getMessage()); - return wrapWithIndexerException(e, - format("failed fetching studyId analysis, command: {0}, retries exhausted", command), - FailureData.builder() - .failingIds(Map.of(STUDY_ID, Set.of(studyId))) - .build() - ); - } - - private void notifyStudyFetchingError(String studyId, String repoUrl, String excMsg) { - val attrs = Map.of(STUDY_ID, studyId, REPO_URL, repoUrl, ERR, excMsg); - val notification = - new IndexerNotification(NotificationName.STUDY_ANALYSES_FETCH_FAILED, attrs); - notifier.notify(notification); - } - - private StudyAnalysisRepositoryTuple buildStudyAnalysisRepoTuple(@NonNull AnalysisIdentifier indexAnalysisCommand, - StudyRepository filesRepository) { - return StudyAnalysisRepositoryTuple.builder() - .analysisId(indexAnalysisCommand.getAnalysisId()) - .study(Study.builder().studyId(indexAnalysisCommand.getStudyId()).build()) - .studyRepository(filesRepository) - .build(); - } - - private Mono> getAnalysisFromStudyRepository(StudyAnalysisRepositoryTuple tuple) { - return tryFetchAnalysis(tuple).flatMap(this :: getExclusionRulesAndFilter); - } - - private Mono>> - getFileCentricDocuments(StudyAnalysisRepositoryTuple tuple) { - return getAnalysisFromStudyRepository(tuple) - .map((analyses) -> buildFileCentricDocuments(tuple.studyRepository, analyses)); - } - - private Mono>> - getAnalysisCentricDocuments(StudyAnalysisRepositoryTuple tuple){ - return getAnalysisFromStudyRepository(tuple) - .map((analyses -> buildAnalysisCentricDocuments(tuple.studyRepository, analyses))); - } - - private Mono> tryFetchAnalysis(StudyAnalysisRepositoryTuple tuple) { - return this.studyDAO.getAnalysis(GetAnalysisCommand.builder() - .analysisId(tuple.getAnalysisId()) - .filesRepositoryBaseUrl(tuple.getStudyRepository().getUrl()) - .studyId(tuple.getStudy().getStudyId()) - .build() - ).map(List::of) + return this.studyDAO + .getStudyAnalyses(command) + .onErrorMap(e -> handleFetchAnalysesError(studyRepositoryBaseUrl, studyId, command, e)); + } + + private Throwable handleFetchAnalysesError( + String studyRepositoryBaseUrl, String studyId, GetStudyAnalysesCommand command, Throwable e) { + notifyStudyFetchingError(studyId, studyRepositoryBaseUrl, e.getMessage()); + return wrapWithIndexerException( + e, + format("failed fetching studyId analysis, command: {0}, retries exhausted", command), + FailureData.builder().failingIds(Map.of(STUDY_ID, Set.of(studyId))).build()); + } + + private void notifyStudyFetchingError(String studyId, String repoUrl, String excMsg) { + val attrs = Map.of(STUDY_ID, studyId, REPO_URL, repoUrl, ERR, excMsg); + val notification = new IndexerNotification(NotificationName.STUDY_ANALYSES_FETCH_FAILED, attrs); + notifier.notify(notification); + } + + private StudyAnalysisRepositoryTuple buildStudyAnalysisRepoTuple( + @NonNull AnalysisIdentifier indexAnalysisCommand, StudyRepository filesRepository) { + return StudyAnalysisRepositoryTuple.builder() + .analysisId(indexAnalysisCommand.getAnalysisId()) + .study(Study.builder().studyId(indexAnalysisCommand.getStudyId()).build()) + .studyRepository(filesRepository) + .build(); + } + + private Mono> getAnalysisFromStudyRepository(StudyAnalysisRepositoryTuple tuple) { + return tryFetchAnalysis(tuple).flatMap(this::getExclusionRulesAndFilter); + } + + private Mono>> getFileCentricDocuments( + StudyAnalysisRepositoryTuple tuple) { + return getAnalysisFromStudyRepository(tuple) + .map((analyses) -> buildFileCentricDocuments(tuple.studyRepository, analyses)); + } + + private Mono>> getAnalysisCentricDocuments( + StudyAnalysisRepositoryTuple tuple) { + return getAnalysisFromStudyRepository(tuple) + .map((analyses -> buildAnalysisCentricDocuments(tuple.studyRepository, analyses))); + } + + private Mono> tryFetchAnalysis(StudyAnalysisRepositoryTuple tuple) { + return this.studyDAO + .getAnalysis( + GetAnalysisCommand.builder() + .analysisId(tuple.getAnalysisId()) + .filesRepositoryBaseUrl(tuple.getStudyRepository().getUrl()) + .studyId(tuple.getStudy().getStudyId()) + .build()) + .map(List::of) .onErrorMap((e) -> handleFetchAnalysisError(tuple, e)); - } + } - private IndexResult convertIndexerExceptionToIndexResult(IndexerException e) { - return IndexResult.builder() - .failureData(e.getFailureData()) - .successful(false) - .build(); - } + private IndexResult convertIndexerExceptionToIndexResult(IndexerException e) { + return IndexResult.builder().failureData(e.getFailureData()).successful(false).build(); + } - private Mono handleIndexStudyError(Throwable e, String studyId, String repoCode) { - val context = Map.of( + private Mono handleIndexStudyError( + Throwable e, String studyId, String repoCode, String indexName) { + val context = + Map.of( STUDY_ID, studyId, REPO_CODE, repoCode, - ERR, getErrorMessageOrType(e) - ); - val failingId = Map.of(STUDY_ID, Set.of(studyId)); - return notifyAndReturnFallback(failingId, context); - } - - private Mono handleIndexAnalysisError(Throwable e, @NonNull AnalysisIdentifier indexAnalysisCommand) { - log.error("unhandled exception while indexing analysis", e); - val failureContext = Map.of( + ERR, getErrorMessageOrType(e)); + val failingId = Map.of(STUDY_ID, Set.of(studyId)); + return notifyAndReturnFallback(failingId, context, indexName); + } + + private Mono handleIndexAnalysisError( + Throwable e, @NonNull AnalysisIdentifier indexAnalysisCommand, @NonNull String indexName) { + log.error("unhandled exception while indexing analysis", e); + val failureContext = + Map.of( ANALYSIS_ID, indexAnalysisCommand.getAnalysisId(), STUDY_ID, indexAnalysisCommand.getStudyId(), REPO_CODE, indexAnalysisCommand.getRepositoryCode(), - ERR, getErrorMessageOrType(e) - ); - val failedAnalysisId = Map.of(ANALYSIS_ID, Set.of(indexAnalysisCommand.getAnalysisId())); - return notifyAndReturnFallback(failedAnalysisId, failureContext); - } - - @NotNull - private Mono notifyAndReturnFallback(Map> failingIds, - Map contextInfo) { - this.notifier.notify(new IndexerNotification(NotificationName.UNHANDLED_ERROR, contextInfo)); - return Mono.just(IndexResult.builder() + ERR, getErrorMessageOrType(e)); + val failedAnalysisId = Map.of(ANALYSIS_ID, Set.of(indexAnalysisCommand.getAnalysisId())); + return notifyAndReturnFallback(failedAnalysisId, failureContext, indexName); + } + + @NotNull + private Mono notifyAndReturnFallback( + Map> failingIds, + Map contextInfo, + String indexName) { + this.notifier.notify(new IndexerNotification(NotificationName.UNHANDLED_ERROR, contextInfo)); + return Mono.just( + IndexResult.builder() + .indexName(indexName) .failureData(FailureData.builder().failingIds(failingIds).build()) .successful(false) - .build() - ); - } - - private Mono> getExclusionRulesAndFilter(List analyses) { - return this.exclusionRulesDAO.getExclusionRules() - .defaultIfEmpty(Map.of()) - .map(ruleMap -> AnalysisAndExclusions.builder() - .analyses(analyses) - .exclusionRulesMap(ruleMap) - .build() - ) - .map(analysisAndExclusions -> filterExcludedAnalyses(analyses, analysisAndExclusions)) - .onErrorMap((e) -> handleExclusionStepError(analyses, e)); - } - - private Throwable handleExclusionStepError(List analyses, Throwable e) { - val failureInfo = Map.of(ANALYSIS_ID, analyses.stream() - .map(Analysis::getAnalysisId) - .collect(Collectors.toUnmodifiableSet()) - ); - notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, failureInfo)); - return wrapWithIndexerException(e, - "failed filtering analysis", - FailureData.builder().failingIds(failureInfo).build() - ); - } - - private Tuple2> - buildFileCentricDocuments(StudyRepository repo, List analyses) { - - return analyses.stream() - .map(analysis -> buildFileDocuments(analysis, repo)) - .map(newEither -> newEither.fold( + .build()); + } + + private Mono> getExclusionRulesAndFilter(List analyses) { + return this.exclusionRulesDAO + .getExclusionRules() + .defaultIfEmpty(Map.of()) + .map( + ruleMap -> + AnalysisAndExclusions.builder() + .analyses(analyses) + .exclusionRulesMap(ruleMap) + .build()) + .map(analysisAndExclusions -> filterExcludedAnalyses(analyses, analysisAndExclusions)) + .onErrorMap((e) -> handleExclusionStepError(analyses, e)); + } + + private Throwable handleExclusionStepError(List analyses, Throwable e) { + val failureInfo = + Map.of( + ANALYSIS_ID, + analyses.stream().map(Analysis::getAnalysisId).collect(Collectors.toUnmodifiableSet())); + notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, failureInfo)); + return wrapWithIndexerException( + e, "failed filtering analysis", FailureData.builder().failingIds(failureInfo).build()); + } + + private Tuple2> buildFileCentricDocuments( + StudyRepository repo, List analyses) { + + return analyses.stream() + .map(analysis -> buildFileDocuments(analysis, repo)) + .map( + newEither -> + newEither.fold( (left) -> new Tuple2<>(left.getFailureData(), List.of()), - (right) -> new Tuple2<>(FailureData.builder().build(), right) - ) - ).reduce((accumulated, current) -> { - accumulated._1().addFailures(current._1()); - val combined = new ArrayList<>(accumulated._2()); - combined.addAll(current._2()); - return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); - }).orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); - } - - private Tuple2> - buildAnalysisCentricDocuments(StudyRepository repo, List analyses){ - return analyses.stream() - .map(analysis -> buildAnalysisDocuments(analysis, repo)) - .map(newEither -> newEither.fold( - (left) -> new Tuple2<>(left.getFailureData(), List.of()), - (right) -> new Tuple2<>(FailureData.builder().build(), right) - ) - ).reduce((accumulated, current) -> { - accumulated._1().addFailures(current._1()); - val combined = new ArrayList<>(accumulated._2()); - combined.addAll(current._2()); - return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); - }).orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); - } - - private Mono batchUpsert(List files) { - return getAlreadyIndexed(files) - .map(storedFilesList -> findConflicts(files, storedFilesList)) - .flatMap(conflictsCheckResult -> { - handleConflicts(conflictsCheckResult); - return Mono.just(conflictsCheckResult); + (right) -> new Tuple2<>(FailureData.builder().build(), right))) + .reduce( + (accumulated, current) -> { + accumulated._1().addFailures(current._1()); + val combined = new ArrayList<>(accumulated._2()); + combined.addAll(current._2()); + return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); }) - .map(conflictsCheckResult -> removeConflictingFromInputFilesList(files, conflictsCheckResult)) - .flatMap(this :: callBatchUpsert) - .doOnNext(this :: notifyIndexRequestFailures) - .onErrorResume( - (ex) -> ex instanceof IndexerException, - (ex) -> Mono.just(IndexResult.builder() - .successful(false) - .failureData(((IndexerException) ex).getFailureData()) - .build()) - ).doOnSuccess(indexResult -> log.trace("finished batchUpsert, list size {}, hashcode {}", files.size(), - Objects.hashCode(files)) - ); - } - - private Mono batchUpsertAnalysis(List analyses) { - return Mono.fromSupplier(() -> analyses).subscribeOn(Schedulers.elastic()) - .flatMap(this :: callBatchUpsertAnalysis) - .doOnNext(this :: notifyIndexRequestFailures) - .onErrorResume( - (ex) -> ex instanceof IndexerException, - (ex) -> Mono.just(IndexResult.builder() - .successful(false) - .failureData(((IndexerException) ex).getFailureData()) - .build()) - ).doOnSuccess(indexResult -> log.trace("finished batch upsert analysis, list size {}, hashcode {}", analyses.size(), - Objects.hashCode(analyses))); - } - - private List filterExcludedAnalyses(List analyses, AnalysisAndExclusions analysisAndExclusions) { - return analyses.stream() - .filter(analysis -> !shouldExcludeAnalysis(analysis, analysisAndExclusions.getExclusionRulesMap())) - .collect(Collectors.toList()); - } - - private Either> - buildFileDocuments(Analysis analysis, StudyRepository repository) { - - return Try.of(() -> FileCentricDocumentConverter.fromAnalysis(analysis, repository)) - .onFailure((e) -> notifyBuildDocumentFailure(NotificationName.CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED, analysis, repository, e)) - .toEither() - .left() - .map((t) -> wrapBuildDocumentException(analysis, t)) - .toEither(); - } - - private Either> - buildAnalysisDocuments(Analysis analysis, StudyRepository repository){ - - return Try.of(() -> fromAnalysis(analysis, repository)) - .onFailure( (e) -> - notifyBuildDocumentFailure(NotificationName.CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED, analysis, repository, e)) - .toEither() - .left() - .map( (t) -> wrapBuildDocumentException(analysis, t)) - .toEither(); - } - - // if there is already a record in another song - private Mono> getAlreadyIndexed(List files) { - return fileCentricIndexAdapter.fetchByIds(files.stream() - .map(FileCentricDocument :: getObjectId) - .collect(Collectors.toList()) - ).map((fetchResult) -> { - // we convert this list to a hash map to optimize performance for large lists when we lookup files by Ids - val idToFileMap = new HashMap(); - fetchResult.forEach(item -> idToFileMap.put(item.getObjectId(), item)); - return Collections.unmodifiableMap(idToFileMap); - } - ); - } - - private ConflictsCheckResult findConflicts(List filesToIndex, - Map storedFiles) { - val conflictingPairs = filesToIndex.stream() + .orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); + } + + private Tuple2> buildAnalysisCentricDocuments( + StudyRepository repo, List analyses) { + return analyses.stream() + .map(analysis -> buildAnalysisDocuments(analysis, repo)) + .map( + newEither -> + newEither.fold( + (left) -> + new Tuple2<>(left.getFailureData(), List.of()), + (right) -> new Tuple2<>(FailureData.builder().build(), right))) + .reduce( + (accumulated, current) -> { + accumulated._1().addFailures(current._1()); + val combined = new ArrayList<>(accumulated._2()); + combined.addAll(current._2()); + return new Tuple2<>(accumulated._1(), Collections.unmodifiableList(combined)); + }) + .orElseGet(() -> new Tuple2<>(FailureData.builder().build(), List.of())); + } + + private Mono batchUpsert(List files) { + return getAlreadyIndexed(files) + .map(storedFilesList -> findConflicts(files, storedFilesList)) + .flatMap( + conflictsCheckResult -> { + handleConflicts(conflictsCheckResult); + return Mono.just(conflictsCheckResult); + }) + .map( + conflictsCheckResult -> + removeConflictingFromInputFilesList(files, conflictsCheckResult)) + .flatMap(this::callBatchUpsert) + .doOnNext(this::notifyIndexRequestFailures) + .onErrorResume( + (ex) -> ex instanceof IndexerException, + (ex) -> + Mono.just( + IndexResult.builder() + .indexName(FILE_CENTRIC_INDEX) + .successful(false) + .failureData(((IndexerException) ex).getFailureData()) + .build())) + .doOnSuccess( + indexResult -> + log.trace( + "finished batchUpsert, list size {}, hashcode {}", + files.size(), + Objects.hashCode(files))); + } + + private Mono batchUpsertAnalysis(List analyses) { + return Mono.fromSupplier(() -> analyses) + .subscribeOn(Schedulers.elastic()) + .flatMap(this::callBatchUpsertAnalysis) + .doOnNext(this::notifyIndexRequestFailures) + .onErrorResume( + (ex) -> ex instanceof IndexerException, + (ex) -> + Mono.just( + IndexResult.builder() + .successful(false) + .failureData(((IndexerException) ex).getFailureData()) + .build())) + .doOnSuccess( + indexResult -> + log.trace( + "finished batch upsert analysis, list size {}, hashcode {}", + analyses.size(), + Objects.hashCode(analyses))); + } + + private List filterExcludedAnalyses( + List analyses, AnalysisAndExclusions analysisAndExclusions) { + return analyses.stream() + .filter( + analysis -> + !shouldExcludeAnalysis(analysis, analysisAndExclusions.getExclusionRulesMap())) + .collect(Collectors.toList()); + } + + private Either> buildFileDocuments( + Analysis analysis, StudyRepository repository) { + + return Try.of(() -> FileCentricDocumentConverter.fromAnalysis(analysis, repository)) + .onFailure( + (e) -> + notifyBuildDocumentFailure( + NotificationName.CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED, analysis, repository, e)) + .toEither() + .left() + .map((t) -> wrapBuildDocumentException(analysis, t)) + .toEither(); + } + + private Either> buildAnalysisDocuments( + Analysis analysis, StudyRepository repository) { + + return Try.of(() -> fromAnalysis(analysis, repository)) + .onFailure( + (e) -> + notifyBuildDocumentFailure( + NotificationName.CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED, + analysis, + repository, + e)) + .toEither() + .left() + .map((t) -> wrapBuildDocumentException(analysis, t)) + .toEither(); + } + + // if there is already a record in another song + private Mono> getAlreadyIndexed( + List files) { + return fileCentricIndexAdapter + .fetchByIds( + files.stream().map(FileCentricDocument::getObjectId).collect(Collectors.toList())) + .map( + (fetchResult) -> { + // we convert this list to a hash map to optimize performance for large lists when we + // lookup files by Ids + val idToFileMap = new HashMap(); + fetchResult.forEach(item -> idToFileMap.put(item.getObjectId(), item)); + return Collections.unmodifiableMap(idToFileMap); + }); + } + + private ConflictsCheckResult findConflicts( + List filesToIndex, Map storedFiles) { + val conflictingPairs = + filesToIndex.stream() .map(fileToIndex -> findIfAnyStoredFileConflicts(storedFiles, fileToIndex)) .filter(Objects::nonNull) .collect(Collectors.toList()); - return ConflictsCheckResult.builder() - .conflictingFiles(conflictingPairs) - .build(); - } + return ConflictsCheckResult.builder().conflictingFiles(conflictingPairs).build(); + } - private Throwable handleFetchAnalysisError(StudyAnalysisRepositoryTuple tuple, Throwable e) { - log.error("failed to fetch analysis", e); - val notificationInfo = Map.of( + private Throwable handleFetchAnalysisError(StudyAnalysisRepositoryTuple tuple, Throwable e) { + log.error("failed to fetch analysis", e); + val notificationInfo = + Map.of( ANALYSIS_ID, tuple.getAnalysisId(), REPO_CODE, tuple.getStudyRepository().getCode(), STUDY_ID, tuple.getStudy().getStudyId(), - ERR, e.getMessage() - ); - val failureInfo = Map.of( - ANALYSIS_ID, Set.of(tuple.getAnalysisId()) - ); - notifier.notify(new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, notificationInfo)); - return wrapWithIndexerException(e, "failed getting analysis", FailureData.builder() - .failingIds( - failureInfo - ).build() - ); - } - - private void handleConflicts(ConflictsCheckResult conflictingFiles) { - if (conflictingFiles == null || conflictingFiles.getConflictingFiles().isEmpty()) return; - this.notifyConflicts(conflictingFiles); - } - - private List removeConflictingFromInputFilesList(List files, - ConflictsCheckResult conflictsCheckResult) { - return files.stream() - .filter(fileCentricDocument -> !isInConflictsList(conflictsCheckResult, fileCentricDocument)) - .collect(Collectors.toUnmodifiableList()); - } - - private Mono callBatchUpsert(List conflictFreeFilesList) { - return this.fileCentricIndexAdapter.batchUpsertFileRepositories( - BatchIndexFilesCommand.builder().files(conflictFreeFilesList).build() - ); - } - - private Mono callBatchUpsertAnalysis(List analyses){ - return this.analysisCentricIndexAdapter.batchUpsertAnalysisRepositories( BatchIndexAnalysisCommand. - builder().analyses(analyses).build()); - } - - private void notifyIndexRequestFailures(IndexResult indexResult) { - if (!indexResult.isSuccessful()) { - notifier.notify( - new IndexerNotification( - NotificationName.INDEX_REQ_FAILED, - Map.of( - FAILURE_DATA, indexResult.getFailureData() - ) - ) - ); - } - } - - private void notifyBuildDocumentFailure(NotificationName notificationName, - Analysis analysis, StudyRepository repository, Throwable e) { - notifier.notify( - new IndexerNotification( - notificationName, - Map.of( - ANALYSIS_ID, analysis.getAnalysisId(), - STUDY_ID, analysis.getStudyId(), - REPO_CODE, repository.getCode(), - ERR, e.getMessage() - ) - ) - ); - } - - private IndexerException wrapBuildDocumentException(Analysis analysis, Throwable throwable) { - return wrapWithIndexerException(throwable, - format("buildFileDocuments failed for analysis : {0}, studyId: {1}", - analysis.getAnalysisId(), analysis.getStudyId()), - FailureData.builder() - .failingIds(Map.of(ANALYSIS_ID, Set.of(analysis.getAnalysisId()))) - .build()); - } - - private Tuple2 - findIfAnyStoredFileConflicts(Map storedFiles, FileCentricDocument fileToIndex) { - if (storedFiles.containsKey(fileToIndex.getObjectId())) { - val storedFile = storedFiles.get(fileToIndex.getObjectId()); - if (fileToIndex.isValidReplica(storedFile)) { - return null; - } - return new Tuple2<>(fileToIndex, storedFiles.get(fileToIndex.getObjectId())); - } else { - return null; - } - } - - private void notifyConflicts(ConflictsCheckResult conflictsCheckResult) { - val conflictingFileList = conflictsCheckResult.getConflictingFiles() - .stream() + ERR, e.getMessage()); + val failureInfo = Map.of(ANALYSIS_ID, Set.of(tuple.getAnalysisId())); + notifier.notify( + new IndexerNotification(NotificationName.FAILED_TO_FETCH_ANALYSIS, notificationInfo)); + return wrapWithIndexerException( + e, "failed getting analysis", FailureData.builder().failingIds(failureInfo).build()); + } + + private void handleConflicts(ConflictsCheckResult conflictingFiles) { + if (conflictingFiles == null || conflictingFiles.getConflictingFiles().isEmpty()) return; + this.notifyConflicts(conflictingFiles); + } + + private List removeConflictingFromInputFilesList( + List files, ConflictsCheckResult conflictsCheckResult) { + return files.stream() + .filter( + fileCentricDocument -> !isInConflictsList(conflictsCheckResult, fileCentricDocument)) + .collect(Collectors.toUnmodifiableList()); + } + + private Mono callBatchUpsert(List conflictFreeFilesList) { + return this.fileCentricIndexAdapter.batchUpsertFileRepositories( + BatchIndexFilesCommand.builder().files(conflictFreeFilesList).build()); + } + + private Mono callBatchUpsertAnalysis(List analyses) { + return this.analysisCentricIndexAdapter.batchUpsertAnalysisRepositories( + BatchIndexAnalysisCommand.builder().analyses(analyses).build()); + } + + private void notifyIndexRequestFailures(IndexResult indexResult) { + if (!indexResult.isSuccessful()) { + notifier.notify( + new IndexerNotification( + NotificationName.INDEX_REQ_FAILED, + Map.of(FAILURE_DATA, indexResult.getFailureData()))); + } + } + + private void notifyBuildDocumentFailure( + NotificationName notificationName, + Analysis analysis, + StudyRepository repository, + Throwable e) { + notifier.notify( + new IndexerNotification( + notificationName, + Map.of( + ANALYSIS_ID, analysis.getAnalysisId(), + STUDY_ID, analysis.getStudyId(), + REPO_CODE, repository.getCode(), + ERR, e.getMessage()))); + } + + private IndexerException wrapBuildDocumentException(Analysis analysis, Throwable throwable) { + return wrapWithIndexerException( + throwable, + format( + "buildFileDocuments failed for analysis : {0}, studyId: {1}", + analysis.getAnalysisId(), analysis.getStudyId()), + FailureData.builder() + .failingIds(Map.of(ANALYSIS_ID, Set.of(analysis.getAnalysisId()))) + .build()); + } + + private Tuple2 findIfAnyStoredFileConflicts( + Map storedFiles, FileCentricDocument fileToIndex) { + if (storedFiles.containsKey(fileToIndex.getObjectId())) { + val storedFile = storedFiles.get(fileToIndex.getObjectId()); + if (fileToIndex.isValidReplica(storedFile)) { + return null; + } + return new Tuple2<>(fileToIndex, storedFiles.get(fileToIndex.getObjectId())); + } else { + return null; + } + } + + private void notifyConflicts(ConflictsCheckResult conflictsCheckResult) { + val conflictingFileList = + conflictsCheckResult.getConflictingFiles().stream() .map(tuple -> tuple.apply(this::toFileConflict)) .collect(Collectors.toUnmodifiableList()); - val notification = new IndexerNotification(NotificationName.INDEX_FILE_CONFLICT, - Map.of(CONFLICTS, conflictingFileList)); - this.notifier.notify(notification); - } - - private boolean isInConflictsList(ConflictsCheckResult conflictsCheckResult, - FileCentricDocument fileCentricDocument) { - return conflictsCheckResult.getConflictingFiles() - .stream() - .map(Tuple2::_1) - .anyMatch(conflictingFile -> conflictingFile - .getObjectId() - .equals(fileCentricDocument.getObjectId()) - ); - } - - private FileConflict toFileConflict(FileCentricDocument f1, FileCentricDocument f2) { - return FileConflict.builder() - .newFile(ConflictingFile.builder() + val notification = + new IndexerNotification( + NotificationName.INDEX_FILE_CONFLICT, Map.of(CONFLICTS, conflictingFileList)); + this.notifier.notify(notification); + } + + private boolean isInConflictsList( + ConflictsCheckResult conflictsCheckResult, FileCentricDocument fileCentricDocument) { + return conflictsCheckResult.getConflictingFiles().stream() + .map(Tuple2::_1) + .anyMatch( + conflictingFile -> + conflictingFile.getObjectId().equals(fileCentricDocument.getObjectId())); + } + + private FileConflict toFileConflict(FileCentricDocument f1, FileCentricDocument f2) { + return FileConflict.builder() + .newFile( + ConflictingFile.builder() .objectId(f1.getObjectId()) .analysisId(f1.getAnalysis().getAnalysisId()) .studyId(f1.getStudyId()) - .repoCode(f1.getRepositories().stream() - .map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet()) - ).build() - ).indexedFile(ConflictingFile.builder() + .repoCode( + f1.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .indexedFile( + ConflictingFile.builder() .objectId(f2.getObjectId()) .analysisId(f2.getAnalysis().getAnalysisId()) .studyId(f2.getStudyId()) - .repoCode(f2.getRepositories().stream() - .map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet()) - ).build() - ).build(); - } - - private Mono - batchUpsertFilesAndCollectFailures(Tuple2> tuple) { - return this.batchUpsert( tuple._2() ) - .map( upsertResult -> + .repoCode( + f2.getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .build(); + } + + private Mono batchUpsertFilesAndCollectFailures( + Tuple2> tuple) { + return this.batchUpsert(tuple._2()) + .map( + upsertResult -> reduceIndexResult( - IndexResult.builder() - .failureData(tuple._1()).build(), - upsertResult)); - } - - private Mono - batchUpsertAnalysesAndCollectFailtures(Tuple2> tuple) { - return this.batchUpsertAnalysis( tuple._2()) - .map( upsertResult -> + IndexResult.builder().failureData(tuple._1()).build(), upsertResult)); + } + + private Mono batchUpsertAnalysesAndCollectFailtures( + Tuple2> tuple) { + return this.batchUpsertAnalysis(tuple._2()) + .map( + upsertResult -> reduceIndexResult( - IndexResult.builder() - .failureData(tuple._1()).build(), - upsertResult)); - } - - private Mono handleIndexRepositoryError(Throwable e, String repositoryCode) { - val failingId = Map.of(REPO_CODE, Set.of(repositoryCode)); - val contextInfo = Map.of(REPO_CODE, repositoryCode, ERR, e.getMessage()); - return this.notifyAndReturnFallback(failingId, contextInfo); - } - - private IndexResult reduceIndexResult(IndexResult accumulatedResult, IndexResult newResult) { - log.trace("In reduceIndexResult, newResult {} ", newResult); - val both = FailureData.builder().build(); - if (!accumulatedResult.isSuccessful()) { - both.addFailures(accumulatedResult.getFailureData()); - } - if (!newResult.isSuccessful()) { - both.addFailures(newResult.getFailureData()); - } - return IndexResult.builder() - .indexName(newResult.getIndexName()) - .failureData(both) - .successful(both.getFailingIds().isEmpty()) - .build(); - } - - private String getErrorMessageOrType(Throwable e) { - return e.getMessage() == null ? e.getClass().getName() : e.getMessage(); - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - static class FileConflict { - private ConflictingFile newFile; - private ConflictingFile indexedFile; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - static class ConflictingFile { - private String objectId; - private String analysisId; - private String studyId; - private Set repoCode; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class ConflictsCheckResult { - private List> conflictingFiles; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class StudyAnalysisRepositoryTuple { - private StudyRepository studyRepository; - private Study study; - private String analysisId; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - public static class StudyAndRepository { - private StudyRepository studyRepository; - private Study study; - } - - @Getter - @Builder - @ToString - @EqualsAndHashCode - private static class AnalysisAndExclusions { - private List analyses; - private Map, List> exclusionRulesMap; - } - + IndexResult.builder().failureData(tuple._1()).build(), upsertResult)); + } + + private Mono handleIndexRepositoryError( + Throwable e, String repositoryCode) { + val failingId = Map.of(REPO_CODE, Set.of(repositoryCode)); + val contextInfo = Map.of(REPO_CODE, repositoryCode, ERR, e.getMessage()); + return this.notifyAndReturnFallback( + failingId, contextInfo, FILE_CENTRIC_INDEX + ANALYSIS_CENTRIC_INDEX); + } + + private IndexResult reduceIndexResult(IndexResult accumulatedResult, IndexResult newResult) { + log.trace("In reduceIndexResult, newResult {} ", newResult); + val both = FailureData.builder().build(); + if (!accumulatedResult.isSuccessful()) { + both.addFailures(accumulatedResult.getFailureData()); + } + if (!newResult.isSuccessful()) { + both.addFailures(newResult.getFailureData()); + } + return IndexResult.builder() + .indexName(newResult.getIndexName()) + .failureData(both) + .successful(both.getFailingIds().isEmpty()) + .build(); + } + + private String getErrorMessageOrType(Throwable e) { + return e.getMessage() == null ? e.getClass().getName() : e.getMessage(); + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + static class FileConflict { + private ConflictingFile newFile; + private ConflictingFile indexedFile; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + static class ConflictingFile { + private String objectId; + private String analysisId; + private String studyId; + private Set repoCode; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class ConflictsCheckResult { + private List> conflictingFiles; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class StudyAnalysisRepositoryTuple { + private StudyRepository studyRepository; + private Study study; + private String analysisId; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + public static class StudyAndRepository { + private StudyRepository studyRepository; + private Study study; + } + + @Getter + @Builder + @ToString + @EqualsAndHashCode + private static class AnalysisAndExclusions { + private List analyses; + private Map, List> exclusionRulesMap; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DocumentConverterHelper.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DocumentConverterHelper.java new file mode 100644 index 00000000..24356068 --- /dev/null +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/DocumentConverterHelper.java @@ -0,0 +1,142 @@ +package bio.overture.maestro.domain.api; + +import bio.overture.maestro.domain.entities.indexing.Donor; +import bio.overture.maestro.domain.entities.indexing.Specimen; +import bio.overture.maestro.domain.entities.metadata.study.Analysis; +import bio.overture.maestro.domain.entities.metadata.study.Sample; +import lombok.NonNull; +import lombok.val; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import static bio.overture.maestro.domain.api.exception.NotFoundException.checkNotFound; + +final class DocumentConverterHelper { + /** + * Converts metadata Specimen pojo and metadata Sample pojo to an indexing Specimen pojo. + */ + static List buildSpecimen( + @NonNull bio.overture.maestro.domain.entities.metadata.study.Specimen specimen, @NonNull Sample sample) { + + val sampleDoc = bio.overture.maestro.domain.entities.indexing.Sample.builder() + .sampleId(sample.getSampleId()) + .matchedNormalSubmitterSampleId(sample.getMatchedNormalSubmitterSampleId()) + .submitterSampleId(sample.getSubmitterSampleId()) + .sampleType(sample.getSampleType()) + .build(); + sampleDoc.replaceInfo(sample.getInfo()); + + val specimenDoc = bio.overture.maestro.domain.entities.indexing.Specimen.builder() + .specimenId(specimen.getSpecimenId()) + .submitterSpecimenId(specimen.getSubmitterSpecimenId()) + .specimenType(specimen.getSpecimenType()) + .specimenTissueSource(specimen.getSpecimenTissueSource()) + .tumourNormalDesignation(specimen.getTumourNormalDesignation()) + .samples(List.of(sampleDoc)) + .build(); + specimenDoc.replaceInfo(specimen.getInfo()); + return List.of(specimenDoc); + } + + /** + * Converts song metadata sample to AnalysisCentricDonor, each song Sample has one donor and one + * specimen. + * + * @param sample song metadata Sample object + * @return converted AnalysisCentricDonor object + */ + static Donor extractDonor(@NonNull Sample sample) { + val donor = sample.getDonor(); + val specimen = sample.getSpecimen(); + val donorDoc = Donor.builder() + .donorId(donor.getDonorId()) + .gender(donor.getGender()) + .submitterDonorId(donor.getSubmitterDonorId()) + .specimens(buildSpecimen(specimen, sample)) + .build(); + donorDoc.replaceInfo(donor.getInfo()); + return donorDoc; + } + + /** + * Groups specimens belonging to a AnalysisCentricDonor + * + * @param list a grouped list with each AnalysisCentricDonor element having exactly one Specimen + * @return a fully assembled Donor object with a list of specimens that belongs to + * the current donor + */ + static Donor mergeDonorBySpecimen(@NonNull List list) { + checkNotFound( + list.size() > 0, "Failed to merge FileCentricDonor by specimen: donor list is empty."); + + // Every element in list has the same donor, so just use the first donor + val anyDonor = list.get(0); + + checkNotFound( + anyDonor.getSpecimens() != null && anyDonor.getSpecimens().size() > 0, + "Failed to merge FileCentricDonor by specimen: donor doesn't have specimen,"); + + val specimenList = + list.stream() + .map(fileCentricDonor -> fileCentricDonor.getSpecimens().get(0)) + .collect(Collectors.toList()); + + val specimenMap = + specimenList.stream() + .collect(Collectors.groupingBy(Specimen::getSpecimenId, Collectors.toList())); + + val specimens = + new ArrayList<>(specimenMap.values()) + .stream() + .map(DocumentConverterHelper::groupSpecimensBySample) + .collect(Collectors.toList()); + + val donorDoc = Donor.builder() + .donorId(anyDonor.getDonorId()) + .submitterDonorId(anyDonor.getSubmitterDonorId()) + .gender(anyDonor.getGender()) + .specimens(specimens) + .build(); + + donorDoc.replaceInfo(anyDonor.getInfo()); + return donorDoc; + } + + + static bio.overture.maestro.domain.entities.indexing.Specimen groupSpecimensBySample( + @NonNull List list) { + checkNotFound(list.size() > 0, "Failed to merge Specimen by Sample: Specimen list is empty."); + + val samples = new ArrayList(); + list.forEach(specimen -> samples.addAll(specimen.getSamples())); + val specimen = list.get(0); + + // if there is more than one sample in the list, merge samples under one specimen + if (list.size() > 1) { + val specimenDoc = bio.overture.maestro.domain.entities.indexing.Specimen.builder() + .samples(samples) + .specimenId(specimen.getSpecimenId()) + .specimenType(specimen.getSpecimenType()) + .tumourNormalDesignation(specimen.getTumourNormalDesignation()) + .specimenTissueSource(specimen.getSpecimenTissueSource()) + .submitterSpecimenId(specimen.getSubmitterSpecimenId()) + .build(); + specimenDoc.replaceInfo(specimen.getInfo()); + return specimenDoc; + } else return specimen; + } + + static List getDonors(@NonNull Analysis analysis) { + val groupedByDonorMap = + analysis.getSamples().stream() + .map(DocumentConverterHelper::extractDonor) + .collect(Collectors.groupingBy(Donor::getDonorId, Collectors.toList())); + + return new ArrayList<>(groupedByDonorMap.values()) + .stream() + .map(DocumentConverterHelper::mergeDonorBySpecimen) + .collect(Collectors.toList()); + } +} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java index d3f68c44..75a77ec7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/ExclusionRulesEvaluator.java @@ -19,84 +19,75 @@ import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; import bio.overture.maestro.domain.entities.metadata.study.*; - import java.util.List; import java.util.Map; /** - * Check analysis entities to see if any is marked for exclusion and hence this whole analysis - * to be excluded. - * - * currently this checks rules in the following order: - * - Study - * - Analysis - * - File - * - Sample - * - Specimen - * - Donor + * Check analysis entities to see if any is marked for exclusion and hence this whole analysis to be + * excluded. * + *

currently this checks rules in the following order: - Study - Analysis - File - Sample - + * Specimen - Donor */ class ExclusionRulesEvaluator { - static boolean shouldExcludeAnalysis(Analysis analysis, Map, List> exclusionRules) { - if (exclusionRules.isEmpty()) return false; - return isExcludedByStudy(analysis, exclusionRules) - || isExcludedByAnalysis(analysis, exclusionRules) - || isExcludedByFile(analysis, exclusionRules) - || isExcludedBySample(analysis, exclusionRules) - || isExcludedBySpecimen(analysis, exclusionRules) - || isExcludedByDonor(analysis, exclusionRules); - } + static boolean shouldExcludeAnalysis( + Analysis analysis, Map, List> exclusionRules) { + if (exclusionRules.isEmpty()) return false; + return isExcludedByStudy(analysis, exclusionRules) + || isExcludedByAnalysis(analysis, exclusionRules) + || isExcludedByFile(analysis, exclusionRules) + || isExcludedBySample(analysis, exclusionRules) + || isExcludedBySpecimen(analysis, exclusionRules) + || isExcludedByDonor(analysis, exclusionRules); + } - private static boolean isExcludedByStudy(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Study.class) && exclusionRules.get(Study.class) - .stream() - .anyMatch(r -> r.applies( - Study.builder().studyId(analysis.getStudyId()).build() - ) - ); - } + private static boolean isExcludedByStudy( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Study.class) + && exclusionRules.get(Study.class).stream() + .anyMatch(r -> r.applies(Study.builder().studyId(analysis.getStudyId()).build())); + } - private static boolean isExcludedByAnalysis(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Analysis.class) && exclusionRules.get(Analysis.class) - .stream() - .anyMatch(r -> r.applies(analysis)); - } + private static boolean isExcludedByAnalysis( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Analysis.class) + && exclusionRules.get(Analysis.class).stream().anyMatch(r -> r.applies(analysis)); + } - private static boolean isExcludedByFile(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(File.class) && analysis.getFiles().stream() - .anyMatch(file -> exclusionRules.get(File.class) - .stream() - .anyMatch(r -> r.applies(file)) - ); - } + private static boolean isExcludedByFile( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(File.class) + && analysis.getFiles().stream() + .anyMatch( + file -> exclusionRules.get(File.class).stream().anyMatch(r -> r.applies(file))); + } - private static boolean isExcludedBySample(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Sample.class) && analysis.getSamples().stream() - .anyMatch(sample -> exclusionRules.get(Sample.class) - .stream() - .anyMatch(r -> r.applies(sample)) - ); - } + private static boolean isExcludedBySample( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Sample.class) + && analysis.getSamples().stream() + .anyMatch( + sample -> + exclusionRules.get(Sample.class).stream().anyMatch(r -> r.applies(sample))); + } - private static boolean isExcludedBySpecimen(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Specimen.class) && analysis.getSamples() - .stream() + private static boolean isExcludedBySpecimen( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Specimen.class) + && analysis.getSamples().stream() .map(Sample::getSpecimen) - .anyMatch(specimen -> exclusionRules.get(Specimen.class) - .stream() - .anyMatch(r -> r.applies(specimen)) - ); - } + .anyMatch( + specimen -> + exclusionRules.get(Specimen.class).stream().anyMatch(r -> r.applies(specimen))); + } - private static boolean isExcludedByDonor(Analysis analysis, Map, List> exclusionRules) { - return exclusionRules.containsKey(Donor.class) && analysis.getSamples() - .stream() + private static boolean isExcludedByDonor( + Analysis analysis, Map, List> exclusionRules) { + return exclusionRules.containsKey(Donor.class) + && analysis.getSamples().stream() .map(Sample::getDonor) - .anyMatch(donor -> exclusionRules.get(Donor.class) - .stream() - .anyMatch(r -> r.applies(donor)) - ); - } - + .anyMatch( + donor -> exclusionRules.get(Donor.class).stream().anyMatch(r -> r.applies(donor))); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java index d57099da..64079567 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/FileCentricDocumentConverter.java @@ -17,19 +17,21 @@ package bio.overture.maestro.domain.api; -import bio.overture.maestro.domain.api.exception.BadDataException; + import bio.overture.maestro.domain.entities.indexing.*; import bio.overture.maestro.domain.entities.metadata.repository.StudyRepository; import bio.overture.maestro.domain.entities.metadata.study.Analysis; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import lombok.val; + import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import static bio.overture.maestro.domain.api.DocumentConverterHelper.getDonors; /** * This class holds the structural changes that the indexer applies to prepare the File documents @@ -39,237 +41,208 @@ @NoArgsConstructor(access = AccessLevel.PRIVATE) final class FileCentricDocumentConverter { - /** - * Entry point for this converter, it extracts analysis files according to the FileCentricDocument structure - * this process is per each analysis and there is no effects on other analyses. - * - * @param analysis the analysis coming from the studyId metadata source - * @param repository the repository is needed to add its information to the final document. - * @return a list of documents each representing files that an analysis produced/used. - */ - static List fromAnalysis(Analysis analysis, StudyRepository repository) { - return extractFiles(analysis, repository); - } - - /** - * iterate over the files list of analysis and build a document for each one - */ - private static List extractFiles(Analysis analysis, StudyRepository repository) { - return analysis.getFiles() - .stream() - .filter(FileCentricDocumentConverter::isDataFile) - .map(f -> buildFileDocument(f, analysis, repository)) - .collect(Collectors.toList()); - } - - /** - * builds the files document from the analysis files - * - * @param file a files as represented from the source in the analysis - */ - private static FileCentricDocument buildFileDocument(bio.overture.maestro.domain.entities.metadata.study.File file, - Analysis analysis, - StudyRepository repository) { - val id = file.getObjectId(); - val metadataFileId = getMetadataFileId(analysis); - val repoFileBuilder = FileCentricDocument.builder() + private static final String EMPTY_STRING = ""; + private static final String BAM = "BAM"; + private static final String BAI = "BAI"; + private static final String CRAM = "CRAM"; + private static final String CRAI = "CRAI"; + private static final String XML = "XML"; + private static final String TBI = "TBI"; + private static final String IDX = "IDX"; + private static final String GZ = ".gz"; + private static final String ZIP = ".zip"; + private static final String B_2_ZIP = ".b2zip"; + private static final String VCF = "VCF"; + private static final String TCG = "TCG"; + private static final String IDX_EXT = "." + IDX; + private static final String TCG_EXT = "." + TCG; + private static final String BAI_EXT = "." + BAI; + private static final String CRAI_EXT = "." + CRAI; + private static final String TBI_EXT = "." + TBI; + + /** + * Entry point for this converter, it extracts analysis files according to the FileCentricDocument + * structure this process is per each analysis and there is no effects on other analyses. + * + * @param analysis the analysis coming from the studyId metadata source + * @param repository the repository is needed to add its information to the final document. + * @return a list of documents each representing files that an analysis produced/used. + */ + static List fromAnalysis(Analysis analysis, StudyRepository repository) { + return extractFiles(analysis, repository); + } + + /** iterate over the files list of analysis and build a document for each one */ + private static List extractFiles( + Analysis analysis, StudyRepository repository) { + return analysis.getFiles().stream() + .filter(FileCentricDocumentConverter::isDataFile) + .map(f -> buildFileDocument(f, analysis, repository)) + .collect(Collectors.toList()); + } + + /** + * builds the files document from the analysis files + * + * @param file a files as represented from the source in the analysis + */ + private static FileCentricDocument buildFileDocument( + bio.overture.maestro.domain.entities.metadata.study.File file, + Analysis analysis, + StudyRepository repository) { + val id = file.getObjectId(); + val repoFileBuilder = + FileCentricDocument.builder() .objectId(id) .studyId(file.getStudyId()) + .dataType(file.getDataType()) .fileType(file.getFileType()) .fileAccess(file.getFileAccess()) - .analysis(FileCentricAnalysis.builder() - .analysisId(analysis.getAnalysisId()) - .state(analysis.getAnalysisState()) - .analysisType(analysis.getAnalysisType().getName()) - .analysisVersion(analysis.getAnalysisType().getVersion()) - .studyId(analysis.getStudyId()) - .experiment(analysis.getExperiment()) - .build() - ) - .files(buildGenomeFileInfo(analysis, file)) - .repositories(List.of(Repository.builder() - .type(repository.getStorageType().name().toUpperCase()) - .organization(repository.getOrganization()) - .name(repository.getName()) - .code(repository.getCode()) - .country(repository.getCountry()) - .url(repository.getUrl()) - .dataPath(repository.getDataPath()) - .metadataPath(repository.getMetadataPath() + "/" + metadataFileId) - .build())) + .analysis( + FileCentricAnalysis.builder() + .analysisId(analysis.getAnalysisId()) + .analysisState(analysis.getAnalysisState()) + .analysisType(analysis.getAnalysisType().getName()) + .analysisVersion(analysis.getAnalysisType().getVersion()) + .experiment(analysis.getExperiment()) + .build()) + .file(buildGenomeFileInfo(analysis, file)) + .repositories( + List.of( + Repository.builder() + .type(repository.getStorageType().name().toUpperCase()) + .organization(repository.getOrganization()) + .name(repository.getName()) + .code(repository.getCode()) + .country(repository.getCountry()) + .url(repository.getUrl()) + .build())) .donors(getDonors(analysis)); - val repoFile = repoFileBuilder.build(); - repoFile.getAnalysis().replaceData(analysis.getData()); - return repoFile; - } - - private static File buildGenomeFileInfo(Analysis analysis, - bio.overture.maestro.domain.entities.metadata.study.File file) { - val fileName = file.getFileName(); - val indexFile = getIndexFile(analysis.getFiles(), fileName); - return File.builder() - .name(fileName) - .format(file.getFileType()) - .size(file.getFileSize()) - .md5sum(file.getFileMd5sum()) - .dataType(file.getDataType()) - .indexFile(indexFile) - .build(); - } - - /** - * extract metadata files if any - */ - static String getMetadataFileId(Analysis analysis) { - val xmlFile = analysis.getFiles() - .stream() - .filter(f -> isXMLFile(f.getFileName())) - .findFirst() - .orElse(null); - return xmlFile == null ? EMPTY_STRING : xmlFile.getObjectId(); - } - - /** - * get the index files associated with that files - * note that this will be removed, when the source explicitly handles associating index files to a files. - * - */ - private static IndexFile getIndexFile(List files, - String fileName) { - Optional sf = Optional.empty(); - if (hasExtension(fileName, BAM)) { - sf = findIndexFile(files, fileName + BAI_EXT); - } else if (hasExtension(fileName, VCF)) { - sf = Stream.of(TBI_EXT, IDX_EXT, TCG_EXT) - .map(suffix -> findIndexFile(files, fileName + suffix)) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst(); - } - return sf - .map(FileCentricDocumentConverter::createIndexFile) - .orElse(null); - } - - private static IndexFile createIndexFile(bio.overture.maestro.domain.entities.metadata.study.File file) { - return IndexFile.builder() - .objectId(file.getObjectId()) - .name(file.getFileName()) - .format(indexFileFormat(file.getFileName())) - .size(file.getFileSize()) - .md5sum(file.getFileMd5sum()) - .dataType(file.getDataType()) - .build(); - } - - private static Optional - findIndexFile(List files, String name) { - - return files.stream() - .filter(f -> f.getFileName().equalsIgnoreCase(name)) - .findFirst(); - } - - private static List getDonors(Analysis analysis) { - return List.of(getDonor(analysis)); - } - - private static FileCentricDonor getDonor(Analysis analysis) { - val sample = analysis.getSamples() - .stream() - .findFirst() - .orElseThrow(() -> new BadDataException("incorrect structure of song data, samples is empty")); - val donor = sample.getDonor(); - val specimen = sample.getSpecimen(); - return FileCentricDonor.builder() - .id(donor.getDonorId()) - .gender(donor.getGender()) - .specimens(Specimen.builder() - .specimenTissueSource(specimen.getSpecimenType()) - .id(specimen.getSpecimenId()) - .submitterSpecimenId(specimen.getSubmitterSpecimenId()) - .tumourNormalDesignation(specimen.getTumourNormalDesignation()) - .specimenTissueSource(specimen.getSpecimenTissueSource()) - .specimenType(specimen.getSpecimenType()) - .samples(Sample.builder() - .id(sample.getSampleId()) - .submitterSampleId(sample.getSubmitterSampleId()) - .sampleType(sample.getSampleType()) - .matchedNormalSubmitterSampleId(sample.getMatchedNormalSubmitterSampleId()) - .build() - ) - .build() - ) - .submitterDonorId(donor.getSubmitterDonorId()) - .build(); - } - - private static boolean isDataFile(bio.overture.maestro.domain.entities.metadata.study.File f) { - val name = f.getFileName(); - return !(isIndexFile(name) || isXMLFile(name)); - } - - private static boolean isXMLFile(String filename) { - return hasExtension(filename, XML); - } - - private static boolean isBAIFile(String filename) { - return hasExtension(filename, BAI); - } - - private static boolean isTBIFile(String filename) { - return hasExtension(filename, TBI); - } - - private static boolean isIDXFile(String filename) { - return hasExtension(filename, IDX); - } - - private static boolean isIndexFile(String filename) { - return isBAIFile(filename) || isIDXFile(filename) || isTBIFile(filename); - } - - private static String indexFileFormat(String fileName) { - if (isBAIFile(fileName)) { - return BAI; - } - if (isTBIFile(fileName)) { - return TBI; - } - if (isIDXFile(fileName)) { - return IDX; - } - return null; - } - - private static boolean hasExtension(String filename, String extension) { - String[] suffixes = { EMPTY_STRING, GZ, ZIP, B_2_ZIP }; - val f = filename.toLowerCase(); - val ext = extension.toLowerCase(); - for (val s : suffixes) { - if (f.endsWith(ext + s)) { - return true; - } - if (f.endsWith(s + ext)) { - return true; - } - } - return false; - } - - private static final String BAM = "BAM"; - private final static String EMPTY_STRING = ""; - private static final String BAI = "BAI"; - private static final String XML = "XML"; - private static final String TBI = "TBI"; - private static final String IDX = "IDX"; - private static final String GZ = ".gz"; - private static final String ZIP = ".zip"; - private static final String B_2_ZIP = ".b2zip"; - private static final String VCF = "VCF"; - private static final String TCG = "TCG"; - private static final String IDX_EXT = "." + IDX; - private static final String TCG_EXT = "." + TCG; - private static final String BAI_EXT = "." + BAI; - private static final String TBI_EXT = "." + TBI; + val repoFile = repoFileBuilder.build(); + repoFile.getAnalysis().replaceData(analysis.getData()); + repoFile.replaceInfo(file.getInfo()); + return repoFile; + } + + private static File buildGenomeFileInfo( + Analysis analysis, bio.overture.maestro.domain.entities.metadata.study.File file) { + val fileName = file.getFileName(); + val indexFile = getIndexFile(analysis.getFiles(), fileName); + val fileDocument = File.builder() + .name(fileName) + .size(file.getFileSize()) + .md5sum(file.getFileMd5sum()) + .dataType(file.getDataType()) + .indexFile(indexFile) + .build(); + fileDocument.replaceInfo(file.getInfo()); + return fileDocument; + } + + /** + * get the index files associated with that files note that this will be removed, when the source + * explicitly handles associating index files to a files. + */ + private static IndexFile getIndexFile( + List files, String fileName) { + + Optional sf = Optional.empty(); + if (hasExtension(fileName, BAM)) { + sf = findIndexFile(files, fileName + BAI_EXT); + } else if (hasExtension(fileName, CRAM)) { + sf = findIndexFile(files, fileName + CRAI_EXT); + } else if (hasExtension(fileName, VCF)) { + sf = + Stream.of(TBI_EXT, IDX_EXT, TCG_EXT) + .map(suffix -> findIndexFile(files, fileName + suffix)) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + } + return sf.map(FileCentricDocumentConverter::createIndexFile).orElse(null); + } + + private static IndexFile createIndexFile( + bio.overture.maestro.domain.entities.metadata.study.File file) { + val indexFileDocument = IndexFile.builder() + .objectId(file.getObjectId()) + .name(file.getFileName()) + .fileType(indexFileFormat(file.getFileName())) + .size(file.getFileSize()) + .md5sum(file.getFileMd5sum()) + .dataType(file.getDataType()) + .build(); + + indexFileDocument.replaceInfo(file.getInfo()); + return indexFileDocument; + } + + private static Optional findIndexFile( + List files, String name) { + + return files.stream().filter(f -> f.getFileName().equalsIgnoreCase(name)).findFirst(); + } + + private static boolean isDataFile(bio.overture.maestro.domain.entities.metadata.study.File f) { + val name = f.getFileName(); + return !(isIndexFile(name) || isXMLFile(name)); + } + + private static boolean isXMLFile(String filename) { + return hasExtension(filename, XML); + } + + private static boolean isBAIFile(String filename) { + return hasExtension(filename, BAI); + } + + private static boolean isCRAIFile(String filename) { + return hasExtension(filename, CRAI); + } + + private static boolean isTBIFile(String filename) { + return hasExtension(filename, TBI); + } + + private static boolean isIDXFile(String filename) { + return hasExtension(filename, IDX); + } + + private static boolean isIndexFile(String filename) { + return isBAIFile(filename) + || isCRAIFile(filename) + || isIDXFile(filename) + || isTBIFile(filename); + } + + private static String indexFileFormat(String fileName) { + if (isBAIFile(fileName)) { + return BAI; + } + if (isCRAIFile(fileName)) { + return CRAI; + } + if (isTBIFile(fileName)) { + return TBI; + } + if (isIDXFile(fileName)) { + return IDX; + } + return null; + } + + private static boolean hasExtension(String filename, String extension) { + String[] suffixes = {EMPTY_STRING, GZ, ZIP, B_2_ZIP}; + val f = filename.toLowerCase(); + val ext = extension.toLowerCase(); + for (val s : suffixes) { + if (f.endsWith(ext + s)) { + return true; + } + if (f.endsWith(s + ext)) { + return true; + } + } + return false; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java index 68792c3e..c48aa8ab 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/IndexEnabledProperties.java @@ -3,6 +3,9 @@ import lombok.NonNull; public interface IndexEnabledProperties { - @NonNull boolean isFileCentricEnabled(); - @NonNull boolean isAnalysisCentricEnabled(); + @NonNull + boolean isFileCentricEnabled(); + + @NonNull + boolean isAnalysisCentricEnabled(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java index be3ee8e7..db7920c6 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Indexer.java @@ -19,47 +19,49 @@ import bio.overture.maestro.domain.api.message.*; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; -import bio.overture.maestro.domain.port.outbound.indexing.FileCentricIndexAdapter; +import java.util.List; import lombok.NonNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - -/** - * Main entry point for the Indexer API - */ +/** Main entry point for the Indexer API */ public interface Indexer { - /** - * A generic method to index a single analysis to all indices - * @param indexAnalysisCommand - * @return failure info and success flag of all indices - */ - Flux indexAnalysis(@NonNull IndexAnalysisCommand indexAnalysisCommand); + /** + * A generic method to index a single analysis to all indices + * + * @param indexAnalysisCommand + * @return failure info and success flag of all indices + */ + Flux indexAnalysis(@NonNull IndexAnalysisCommand indexAnalysisCommand); + + /** + * Used to remove all files documents for an analysis. + * + * @param removeAnalysisCommand specify repo studyId and analysis id + * @return flag indicating success and failure info if any + */ + Flux removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand); + + /** + * A generic method to index a study. + * + * @param command + * @return failure info and success flag of all indices + */ + Flux indexStudy(@NonNull IndexStudyCommand command); - /** - * Used to remove all files documents for an analysis. - * @param removeAnalysisCommand specify repo studyId and analysis id - * @return flag indicating success and failure info if any - */ - Mono removeAnalysis(@NonNull RemoveAnalysisCommand removeAnalysisCommand); + /** + * A generic method to index the entire repository to all indices. + * + * @param command contains repository code + * @return result indicating success/fail and failure information + */ + Mono indexRepository(@NonNull IndexStudyRepositoryCommand command); - /** - * A generic method to index a study. - * @param command - * @return failure info and success flag of all indices - */ - Flux indexStudy(@NonNull IndexStudyCommand command); + void addRule(AddRuleCommand addRuleCommand); - /** - * A generic method to index the entire repository to all indices. - * @param command contains repository code - * @return result indicating success/fail and failure information - */ - Mono indexRepository(@NonNull IndexStudyRepositoryCommand command); + void deleteRule(DeleteRuleCommand deleteRuleCommand); - void addRule(AddRuleCommand addRuleCommand); - void deleteRule(DeleteRuleCommand deleteRuleCommand); - List getAllRules(); + List getAllRules(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java index be6da1b8..2caac77b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationCategory.java @@ -17,8 +17,8 @@ package bio.overture.maestro.domain.api; -public enum NotificationCategory { - ERROR, - INFO, - WARN, +public enum NotificationCategory { + ERROR, + INFO, + WARN, } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java index 023d1316..5650b34f 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationChannel.java @@ -18,16 +18,16 @@ package bio.overture.maestro.domain.api; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; +import java.util.Set; import lombok.NonNull; import reactor.core.publisher.Mono; -import java.util.Set; - /** - * A channel is an abstraction of the technology infrastructure - * that this notification will be delivered through, can be email, web api call, filesystem or anything. + * A channel is an abstraction of the technology infrastructure that this notification will be + * delivered through, can be email, web api call, filesystem or anything. */ public interface NotificationChannel { - Mono send(@NonNull IndexerNotification notification); - Set subscriptions(); + Mono send(@NonNull IndexerNotification notification); + + Set subscriptions(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java index f86db6f9..61bf1882 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/NotificationName.java @@ -21,22 +21,21 @@ @Getter public enum NotificationName { - STUDY_ANALYSES_FETCH_FAILED(NotificationCategory.ERROR), - FETCH_REPO_STUDIES_FAILED(NotificationCategory.ERROR), - INDEX_REQ_FAILED(NotificationCategory.ERROR), - CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED(NotificationCategory.ERROR), - CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED(NotificationCategory.ERROR), - INDEX_FILE_CONFLICT(NotificationCategory.WARN), - ALL(null), - UNHANDLED_ERROR(NotificationCategory.ERROR), - FAILED_TO_FETCH_ANALYSIS(NotificationCategory.ERROR), - FAILED_TO_FETCH_REPOSITORY(NotificationCategory.ERROR), - FAILED_TO_REMOVE_ANALYSIS(NotificationCategory.ERROR); + STUDY_ANALYSES_FETCH_FAILED(NotificationCategory.ERROR), + FETCH_REPO_STUDIES_FAILED(NotificationCategory.ERROR), + INDEX_REQ_FAILED(NotificationCategory.ERROR), + CONVERT_ANALYSIS_TO_FILE_DOCS_FAILED(NotificationCategory.ERROR), + CONVERT_ANALYSIS_TO_ANALYSIS_DOCS_FAILED(NotificationCategory.ERROR), + INDEX_FILE_CONFLICT(NotificationCategory.WARN), + ALL(null), + UNHANDLED_ERROR(NotificationCategory.ERROR), + FAILED_TO_FETCH_ANALYSIS(NotificationCategory.ERROR), + FAILED_TO_FETCH_REPOSITORY(NotificationCategory.ERROR), + FAILED_TO_REMOVE_ANALYSIS(NotificationCategory.ERROR); - private final NotificationCategory category; - - NotificationName(NotificationCategory category) { - this.category = category; - } + private final NotificationCategory category; + NotificationName(NotificationCategory category) { + this.category = category; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java index 2e90726b..b73ed132 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/Notifier.java @@ -18,54 +18,65 @@ package bio.overture.maestro.domain.api; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; -import lombok.extern.slf4j.Slf4j; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; - -import javax.inject.Inject; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; @Slf4j class Notifier { - private final Set notificationChannels; + private final Set notificationChannels; - @Inject - public Notifier(Set notificationChannels) { - this.notificationChannels = notificationChannels; - } + @Inject + public Notifier(Set notificationChannels) { + this.notificationChannels = notificationChannels; + } - /** - * Asynchronously calls the eligible notification channels. - * @param notification the notification to send - */ - public void notify(IndexerNotification notification) { - // some of the channels may need async I/O execution (slack) but - // the caller (indexer) doesn't need to worry about what happens here so no need to return the flux and - // we subscribe here. - Flux.fromIterable(getEligibleChannels(notification)) - .flatMap(notificationChannel -> sendNotificationThroughChannel(notification, notificationChannel)) - .subscribe(); - } + /** + * Asynchronously calls the eligible notification channels. + * + * @param notification the notification to send + */ + public void notify(IndexerNotification notification) { + // some of the channels may need async I/O execution (slack) but + // the caller (indexer) doesn't need to worry about what happens here so no need to return the + // flux and + // we subscribe here. + Flux.fromIterable(getEligibleChannels(notification)) + .flatMap( + notificationChannel -> + sendNotificationThroughChannel(notification, notificationChannel)) + .subscribe(); + } - private Mono sendNotificationThroughChannel(IndexerNotification notification, NotificationChannel notificationChannel) { - return notificationChannel.send(notification) - .onErrorResume(e -> { - log.error("failed to deliver notification {} to channel {}", notification, notificationChannel, e); - return Mono.just(false); + private Mono sendNotificationThroughChannel( + IndexerNotification notification, NotificationChannel notificationChannel) { + return notificationChannel + .send(notification) + .onErrorResume( + e -> { + log.error( + "failed to deliver notification {} to channel {}", + notification, + notificationChannel, + e); + return Mono.just(false); }); - } + } - private List getEligibleChannels(IndexerNotification notification) { - return notificationChannels.stream() - .filter(notificationChannel -> shouldReceiveNotification(notification, notificationChannel)) - .collect(Collectors.toUnmodifiableList()); - } + private List getEligibleChannels(IndexerNotification notification) { + return notificationChannels.stream() + .filter(notificationChannel -> shouldReceiveNotification(notification, notificationChannel)) + .collect(Collectors.toUnmodifiableList()); + } - private boolean shouldReceiveNotification(IndexerNotification notification, NotificationChannel notificationChannel) { - return notificationChannel.subscriptions().contains(NotificationName.ALL) - || notificationChannel.subscriptions().contains(notification.getNotificationName()); - } + private boolean shouldReceiveNotification( + IndexerNotification notification, NotificationChannel notificationChannel) { + return notificationChannel.subscriptions().contains(NotificationName.ALL) + || notificationChannel.subscriptions().contains(notification.getNotificationName()); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java index a68fd92c..4c5fd138 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/BadDataException.java @@ -17,11 +17,9 @@ package bio.overture.maestro.domain.api.exception; -/** - * This exception is to indicate that a data processing issue happened. - */ +/** This exception is to indicate that a data processing issue happened. */ public class BadDataException extends IndexerException { - public BadDataException(String message) { - super(message); - } + public BadDataException(String message) { + super(message); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java index f9887630..a50d61b4 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/FailureData.java @@ -17,40 +17,39 @@ package bio.overture.maestro.domain.api.exception; -import lombok.Builder; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.ToString; - import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import lombok.Builder; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; @Getter @Builder @ToString @EqualsAndHashCode public class FailureData { - @Builder.Default - private final Map> failingIds = new HashMap<>(); + @Builder.Default private final Map> failingIds = new HashMap<>(); - private void addFailures(String type, Set ids) { - if (failingIds.containsKey(type)) { - failingIds.put(type, - Stream.concat(failingIds.get(type).stream(), ids.stream()) - .collect(Collectors.toUnmodifiableSet())); - return; - } - failingIds.put(type, Set.copyOf(ids)); + private void addFailures(String type, Set ids) { + if (failingIds.containsKey(type)) { + failingIds.put( + type, + Stream.concat(failingIds.get(type).stream(), ids.stream()) + .collect(Collectors.toUnmodifiableSet())); + return; } + failingIds.put(type, Set.copyOf(ids)); + } - public Map> getFailingIds() { - return Map.copyOf(this.failingIds); - } + public Map> getFailingIds() { + return Map.copyOf(this.failingIds); + } - public void addFailures(FailureData failureData) { - failureData.getFailingIds().forEach(this::addFailures); - } + public void addFailures(FailureData failureData) { + failureData.getFailingIds().forEach(this::addFailures); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java index f04595f6..e53c4300 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/IndexerException.java @@ -27,13 +27,14 @@ @NoArgsConstructor @AllArgsConstructor public class IndexerException extends RuntimeException { - protected FailureData failureData; - public IndexerException(String message) { - super(message); - } + protected FailureData failureData; - public IndexerException(String message, Throwable cause, FailureData failureData) { - super(message, cause); - this.failureData = failureData; - } + public IndexerException(String message) { + super(message); + } + + public IndexerException(String message, Throwable cause, FailureData failureData) { + super(message, cause); + this.failureData = failureData; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java index e9b619c0..7c51a027 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/exception/NotFoundException.java @@ -17,29 +17,27 @@ package bio.overture.maestro.domain.api.exception; +import static java.lang.String.format; + import lombok.NoArgsConstructor; import lombok.NonNull; -import static java.lang.String.format; - -/** - * Indicates a required / requested resource / data is missing. - */ +/** Indicates a required / requested resource / data is missing. */ @NoArgsConstructor public class NotFoundException extends IndexerException { - public NotFoundException(String message) { - super(message); - } + public NotFoundException(String message) { + super(message); + } - public static NotFoundException buildNotFoundException( - @NonNull String formattedMessage, Object... args) { - return new NotFoundException(format(formattedMessage, args)); - } + public static NotFoundException buildNotFoundException( + @NonNull String formattedMessage, Object... args) { + return new NotFoundException(format(formattedMessage, args)); + } - public static void checkNotFound( - boolean expression, @NonNull String formattedMessage, @NonNull Object... args) { - if (!expression) { - throw new NotFoundException(format(formattedMessage, args)); - } + public static void checkNotFound( + boolean expression, @NonNull String formattedMessage, @NonNull Object... args) { + if (!expression) { + throw new NotFoundException(format(formattedMessage, args)); } + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java index be5a4c35..a0f622bc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AddRuleCommand.java @@ -17,5 +17,4 @@ package bio.overture.maestro.domain.api.message; -public class AddRuleCommand { -} +public class AddRuleCommand {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java index ba9d1c54..da176eb1 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/AnalysisIdentifier.java @@ -26,10 +26,7 @@ @EqualsAndHashCode @AllArgsConstructor public class AnalysisIdentifier { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java index 3b14457a..27b3856b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/DeleteRuleCommand.java @@ -17,5 +17,4 @@ package bio.overture.maestro.domain.api.message; -public class DeleteRuleCommand { -} +public class DeleteRuleCommand {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java index b3fd4350..4e2656e6 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexAnalysisCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexAnalysisCommand { - @NonNull - private AnalysisIdentifier analysisIdentifier; + @NonNull private AnalysisIdentifier analysisIdentifier; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java index e30fcefc..4d2ad14b 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexResult.java @@ -27,7 +27,7 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexResult { - private String indexName; - @Builder.Default private FailureData failureData = FailureData.builder().build(); - private boolean successful; + private String indexName; + @Builder.Default private FailureData failureData = FailureData.builder().build(); + private boolean successful; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java index 968da567..ccaadeb8 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyCommand.java @@ -26,8 +26,6 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexStudyCommand { - @NonNull - private String studyId; - @NonNull - private String repositoryCode; + @NonNull private String studyId; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java index f309656f..73cd4c97 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/IndexStudyRepositoryCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class IndexStudyRepositoryCommand { - @NonNull - private String repositoryCode; + @NonNull private String repositoryCode; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java index 099db8f4..7df34b0a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/api/message/RemoveAnalysisCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class RemoveAnalysisCommand { - @NonNull - private AnalysisIdentifier analysisIdentifier; + @NonNull private AnalysisIdentifier analysisIdentifier; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java index b7f547a1..dba76c7a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/AnalysisType.java @@ -8,6 +8,6 @@ @AllArgsConstructor @EqualsAndHashCode public class AnalysisType { - private String name; - private Integer version; + private String name; + private Integer version; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Donor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Donor.java new file mode 100644 index 00000000..13778574 --- /dev/null +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Donor.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2019. Ontario Institute for Cancer Research + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package bio.overture.maestro.domain.entities.indexing; + +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import lombok.*; +import lombok.experimental.FieldNameConstants; + +@Builder +@Getter +@ToString +@NoArgsConstructor +@AllArgsConstructor +@EqualsAndHashCode +@FieldNameConstants +public class Donor { + @NonNull private String donorId; + + @NonNull private String submitterDonorId; + + @NonNull private String gender; + + @NonNull private List specimens; + + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } +} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java index 9587dc63..d0e537cd 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/File.java @@ -17,8 +17,13 @@ package bio.overture.maestro.domain.entities.indexing; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import lombok.*; +import java.util.Map; +import java.util.TreeMap; + @Builder @Getter @ToString @@ -26,14 +31,35 @@ @AllArgsConstructor @EqualsAndHashCode public class File { - @NonNull - private String name; - @NonNull - private String format; - @NonNull - private String md5sum; - private Long size; - private Long lastModified; - private String dataType; - private IndexFile indexFile; + @NonNull private String name; + @NonNull private String md5sum; + private Long size; + private String dataType; + private IndexFile indexFile; + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java index e6c78074..b48a4e24 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricAnalysis.java @@ -19,11 +19,10 @@ import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; -import lombok.experimental.FieldNameConstants; - import java.util.Map; import java.util.TreeMap; +import lombok.*; +import lombok.experimental.FieldNameConstants; @Getter @Builder @@ -34,38 +33,31 @@ @FieldNameConstants public class FileCentricAnalysis { - @NonNull - private String analysisId; - @NonNull - private String analysisType; - @NonNull - private Integer analysisVersion; - @NonNull - private String state; - @NonNull - private Map experiment; - private String studyId; - /** - * this field is to capture the dynamic fields in the analysis. - * it's the responsibility of the users to make sure the mapping is consistent with - * the different fields that they want to add/index, they are also responsibile - * to add the mappings of these fields or reindex appropriately. - */ - @NonNull - private final Map data = new TreeMap<>(); - - @JsonAnyGetter - public Map getData() { - return data; - } - - @JsonAnySetter - public void setData(String key, Object value) { - data.put(key, value); - } - - public void replaceData(Map data) { - this.data.clear(); - this.data.putAll(data); - } + @NonNull private String analysisId; + @NonNull private String analysisType; + @NonNull private Integer analysisVersion; + @NonNull private String analysisState; + @NonNull private Map experiment; + /** + * this field is to capture the dynamic fields in the analysis. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsibile to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map data = new TreeMap<>(); + + @JsonAnyGetter + public Map getData() { + return data; + } + + @JsonAnySetter + public void setData(String key, Object value) { + data.put(key, value); + } + + public void replaceData(Map data) { + this.data.clear(); + this.data.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java index 368e91a2..919981c2 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDocument.java @@ -17,14 +17,16 @@ package bio.overture.maestro.domain.entities.indexing; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import lombok.*; import lombok.experimental.FieldNameConstants; -import java.util.List; - -/** - * Represents the structure of the index document that corresponds to an analysis "File". - */ +/** Represents the structure of the index document that corresponds to an analysis "File". */ @Builder @Getter @ToString @@ -34,54 +36,74 @@ @FieldNameConstants public class FileCentricDocument { - @NonNull - private String objectId; - - @NonNull - private String studyId; - - private String fileType; - - private String fileAccess; - - @NonNull - private FileCentricAnalysis analysis; - - /** - * The actual genome analysis files information. - */ - @NonNull - private File files; - - /** - * Each files can be hosted in more than one files repository, this references the other repositories (locations) - * where this files can be fetched from. - */ - @NonNull - private List repositories; - - @NonNull - private List donors; - - /** - * This method is to check if the files is a valid replica of another files. - * by replication we mean that an analysis can be copied to a different metadata repository to make downloading - * the files faster for different geographical locations. - * it checks all attributes except for the repository (since the repository is expected to be different) - * - * @param fileCentricDocument the other files we compare to - * @return flag indicates if this is a valid replica. - */ - public boolean isValidReplica(FileCentricDocument fileCentricDocument) { - if (fileCentricDocument == null) return false; - if (this.equals(fileCentricDocument)) return true; - return this.objectId.equals(fileCentricDocument.getObjectId()) - && this.studyId.equals(fileCentricDocument.getStudyId()) - && this.fileType.equals(fileCentricDocument.getFileType()) - && this.fileAccess.equals(fileCentricDocument.getFileAccess()) - && this.donors.equals(fileCentricDocument.getDonors()) - && this.analysis.equals(fileCentricDocument.getAnalysis()) - && this.files.equals(fileCentricDocument.getFiles()); + @NonNull private String objectId; + + @NonNull private String studyId; + + private String dataType; + + private String fileType; + + private String fileAccess; + + @NonNull private FileCentricAnalysis analysis; + + /** The actual genome analysis files information. */ + @NonNull private File file; + + /** + * Each files can be hosted in more than one files repository, this references the other + * repositories (locations) where this files can be fetched from. + */ + @NonNull private List repositories; + + @NonNull private List donors; + + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; } + this.info.clear(); + this.info.putAll(data); + } + /** + * This method is to check if the files is a valid replica of another files. by replication we + * mean that an analysis can be copied to a different metadata repository to make downloading the + * files faster for different geographical locations. it checks all attributes except for the + * repository (since the repository is expected to be different) + * + * @param fileCentricDocument the other files we compare to + * @return flag indicates if this is a valid replica. + */ + public boolean isValidReplica(FileCentricDocument fileCentricDocument) { + if (fileCentricDocument == null) return false; + if (this.equals(fileCentricDocument)) return true; + return this.objectId.equals(fileCentricDocument.getObjectId()) + && this.studyId.equals(fileCentricDocument.getStudyId()) + && this.dataType.equals(fileCentricDocument.getDataType()) + && this.fileType.equals(fileCentricDocument.getFileType()) + && this.fileAccess.equals(fileCentricDocument.getFileAccess()) + && this.donors.equals(fileCentricDocument.getDonors()) + && this.analysis.equals(fileCentricDocument.getAnalysis()) + && this.file.equals(fileCentricDocument.getFile()); + } } - diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java deleted file mode 100644 index d20ae3af..00000000 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/FileCentricDonor.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2019. Ontario Institute for Cancer Research - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -package bio.overture.maestro.domain.entities.indexing; - -import lombok.*; - -@Builder -@Getter -@ToString -@NoArgsConstructor -@EqualsAndHashCode -@AllArgsConstructor -public class FileCentricDonor { - @NonNull - private String id; - - @NonNull - private String gender; - - @NonNull - private String submitterDonorId; - - @NonNull - private Specimen specimens; -} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java index cb22878e..ec296775 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/IndexFile.java @@ -17,8 +17,13 @@ package bio.overture.maestro.domain.entities.indexing; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import lombok.*; +import java.util.Map; +import java.util.TreeMap; + @Builder @Getter @ToString @@ -26,14 +31,36 @@ @AllArgsConstructor @EqualsAndHashCode public class IndexFile { - @NonNull - private String objectId; - @NonNull - private String name; - @NonNull - private String format; - @NonNull - private String md5sum; - private String dataType; - private Long size; + @NonNull private String objectId; + @NonNull private String name; + @NonNull private String fileType; + @NonNull private String md5sum; + private String dataType; + private Long size; + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java index 00ab6092..60d95b03 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Repository.java @@ -17,7 +17,6 @@ package bio.overture.maestro.domain.entities.indexing; - import lombok.*; @Builder @@ -27,26 +26,15 @@ @NoArgsConstructor @AllArgsConstructor public class Repository { - @NonNull - private String code; - - @NonNull - private String organization; - - private String name; + @NonNull private String code; - @NonNull - private String type; + @NonNull private String organization; - @NonNull - private String country; + private String name; - @NonNull - private String url; + @NonNull private String type; - @NonNull - private String dataPath; + @NonNull private String country; - @NonNull - private String metadataPath; + @NonNull private String url; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java index aff9b370..c7fd7d85 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Sample.java @@ -17,11 +17,15 @@ package bio.overture.maestro.domain.entities.indexing; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import lombok.*; +import java.util.Map; +import java.util.TreeMap; + /** - * Many samples can belong to an Analysis, a samples represents - * a donor and a specimen composition. + * Many samples can belong to an Analysis, a samples represents a donor and a specimen composition. */ @Getter @Builder @@ -30,11 +34,34 @@ @AllArgsConstructor @EqualsAndHashCode public class Sample { - @NonNull - private String id; - @NonNull - private String submitterSampleId; - @NonNull - private String sampleType; - private String matchedNormalSubmitterSampleId; + @NonNull private String sampleId; + @NonNull private String submitterSampleId; + @NonNull private String sampleType; + private String matchedNormalSubmitterSampleId; + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java index ab405992..531a0f57 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/Specimen.java @@ -17,7 +17,13 @@ package bio.overture.maestro.domain.entities.indexing; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import com.fasterxml.jackson.annotation.JsonInclude; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + import lombok.*; @Getter @@ -28,15 +34,36 @@ @EqualsAndHashCode @JsonInclude(JsonInclude.Include.NON_NULL) public class Specimen { - @NonNull - private String id; - @NonNull - private String specimenType; - @NonNull - private String submitterSpecimenId; - @NonNull - private Sample samples; - private String tumourNormalDesignation; - private String specimenTissueSource; -} + @NonNull private String specimenId; + @NonNull private String specimenType; + @NonNull private String submitterSpecimenId; + @NonNull private List samples; + private String tumourNormalDesignation; + private String specimenTissueSource; + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } +} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java index 768fee2a..6765d609 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/StorageType.java @@ -18,5 +18,5 @@ package bio.overture.maestro.domain.entities.indexing; public enum StorageType { - S3 + S3 } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java index 31616c1e..90d7d7d3 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDocument.java @@ -1,13 +1,14 @@ package bio.overture.maestro.domain.entities.indexing.analysis; +import bio.overture.maestro.domain.entities.indexing.Donor; import bio.overture.maestro.domain.entities.indexing.Repository; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; -import lombok.experimental.FieldNameConstants; import java.util.List; import java.util.Map; import java.util.TreeMap; +import lombok.*; +import lombok.experimental.FieldNameConstants; @Builder @Getter @@ -28,7 +29,7 @@ public class AnalysisCentricDocument { @NonNull private String studyId; - @NonNull private List donors; + @NonNull private List donors; @NonNull private List files; diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java deleted file mode 100644 index 69e536ad..00000000 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricDonor.java +++ /dev/null @@ -1,29 +0,0 @@ -package bio.overture.maestro.domain.entities.indexing.analysis; - -import bio.overture.maestro.domain.entities.indexing.Specimen; -import lombok.*; -import lombok.experimental.FieldNameConstants; - -import java.util.List; - -@Builder -@Getter -@ToString -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -@FieldNameConstants -public class AnalysisCentricDonor { - @NonNull - private String id; - - @NonNull - private String submitterDonorId; - - @NonNull - private String gender; - - @NonNull - private List specimens; - -} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java index 3af72d23..2e3d293e 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricFile.java @@ -1,8 +1,13 @@ package bio.overture.maestro.domain.entities.indexing.analysis; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; import lombok.*; import lombok.experimental.FieldNameConstants; +import java.util.Map; +import java.util.TreeMap; + @Builder @Getter @ToString @@ -10,9 +15,9 @@ @AllArgsConstructor @EqualsAndHashCode @FieldNameConstants -public class AnalysisCentricFile { +public class AnalysisCentricFile { - @NonNull private String id; + @NonNull private String objectId; @NonNull private String name; @@ -25,4 +30,31 @@ public class AnalysisCentricFile { @NonNull private String fileAccess; @NonNull private String dataType; + + /** + * this field is to capture the dynamic fields in the file info. it's the responsibility of the + * users to make sure the mapping is consistent with the different fields that they want to + * add/index, they are also responsible to add the mappings of these fields or reindex + * appropriately. + */ + @NonNull private final Map info = new TreeMap<>(); + + @JsonAnyGetter + public Map getInfo() { + return info; + } + + @JsonAnySetter + public void setInfo(String key, Object value) { + info.put(key, value); + } + + public void replaceInfo(Map data) { + if (data == null) { + this.info.clear(); + return; + } + this.info.clear(); + this.info.putAll(data); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java deleted file mode 100644 index da50b957..00000000 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/analysis/AnalysisCentricSpecimen.java +++ /dev/null @@ -1,23 +0,0 @@ -package bio.overture.maestro.domain.entities.indexing.analysis; - -import bio.overture.maestro.domain.entities.indexing.Sample; -import lombok.*; -import lombok.experimental.FieldNameConstants; - -@Builder -@Getter -@ToString -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode -@FieldNameConstants -public class AnalysisCentricSpecimen { - - @NonNull private String id; - @NonNull private String type; - @NonNull private String submittedId; - @NonNull private Sample samples; - private String tumourNormalDesignation; - private String specimenTissueSource; - -} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java index 18613172..6543a2fc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionId.java @@ -23,9 +23,9 @@ import java.lang.annotation.Target; /** - * A metadata annotation to indicate a field annotated with this - * qualifies to be processed by {@link IDExclusionRule} + * A metadata annotation to indicate a field annotated with this qualifies to be processed by {@link + * IDExclusionRule} */ @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.FIELD}) -public @interface ExclusionId { } +public @interface ExclusionId {} diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java index d7cfebe6..d62e280d 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/ExclusionRule.java @@ -17,9 +17,7 @@ package bio.overture.maestro.domain.entities.indexing.rules; -/** - * A generic extendable rule to indicate if a rule applies to an instance. - */ +/** A generic extendable rule to indicate if a rule applies to an instance. */ public abstract class ExclusionRule { - abstract public boolean applies(Object instance); + public abstract boolean applies(Object instance); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java index 798d1797..8eab68fc 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/indexing/rules/IDExclusionRule.java @@ -17,19 +17,18 @@ package bio.overture.maestro.domain.entities.indexing.rules; - -import lombok.*; -import lombok.extern.slf4j.Slf4j; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import lombok.*; +import lombok.extern.slf4j.Slf4j; /** - * When applied on an instance it checks if the field annotated with {@link ExclusionId} - * is in the list of ids that should be excluded. + * When applied on an instance it checks if the field annotated with {@link ExclusionId} is in the + * list of ids that should be excluded. * - * if multiple fields in the instance marked with this, first one (as returned in the class metadata) wins + *

if multiple fields in the instance marked with this, first one (as returned in the class + * metadata) wins */ @Slf4j @Getter @@ -40,46 +39,41 @@ @EqualsAndHashCode(callSuper = true) public class IDExclusionRule extends ExclusionRule { - /** - * the class this rules applies to. - */ - private Class clazz; + /** the class this rules applies to. */ + private Class clazz; - /** - * the list of ids to be excluded. - */ - @Builder.Default - private List ids = new ArrayList<>(); + /** the list of ids to be excluded. */ + @Builder.Default private List ids = new ArrayList<>(); - @SneakyThrows - public boolean applies(Object instance) { - log.trace("checking rule against : {}", instance); - if (ids.isEmpty() || !instance.getClass().equals(clazz)) return false; + @SneakyThrows + public boolean applies(Object instance) { + log.trace("checking rule against : {}", instance); + if (ids.isEmpty() || !instance.getClass().equals(clazz)) return false; - val idExclusionField = Arrays.stream(instance.getClass().getDeclaredFields()) + val idExclusionField = + Arrays.stream(instance.getClass().getDeclaredFields()) .filter(field -> field.getAnnotationsByType(ExclusionId.class).length > 0) .findFirst() .orElse(null); - if (idExclusionField == null) { - log.trace("idExclusionField is null"); - return false; - } + if (idExclusionField == null) { + log.trace("idExclusionField is null"); + return false; + } - idExclusionField.setAccessible(true); - val value = idExclusionField.get(instance); - if (value == null) { - log.trace("value is null is null"); - return false; - } + idExclusionField.setAccessible(true); + val value = idExclusionField.get(instance); + if (value == null) { + log.trace("value is null is null"); + return false; + } - val excluded = ids.contains(String.valueOf(value)); - log.trace("id exclusion rule for value {} = {}", value, excluded); + val excluded = ids.contains(String.valueOf(value)); + log.trace("id exclusion rule for value {} = {}", value, excluded); - if (excluded) { - log.info("id {} was excluded according to the rules", value); - } - return excluded; + if (excluded) { + log.info("id {} was excluded according to the rules", value); } - + return excluded; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java index bcbc9873..dd710497 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/repository/StudyRepository.java @@ -17,13 +17,12 @@ package bio.overture.maestro.domain.entities.metadata.repository; - import bio.overture.maestro.domain.entities.indexing.StorageType; import lombok.*; /** - * This represents a studyId (including analyses & files) metadata repository, holds information about sources where this - * indexer can pull metadata from. + * This represents a studyId (including analyses & files) metadata repository, holds information + * about sources where this indexer can pull metadata from. */ @Builder @Getter @@ -32,50 +31,21 @@ @EqualsAndHashCode @AllArgsConstructor public class StudyRepository { - /** - * display name of the repository - */ - @NonNull - private String name; - - /** - * a unique code for the repository - */ - @NonNull - private String code; - - /** - * the country where this files repository resides - */ - @NonNull - private String country; + /** display name of the repository */ + @NonNull private String name; - /** - * based url of the host of this repository metadata - */ - @NonNull - private String url; + /** a unique code for the repository */ + @NonNull private String code; - /** - * url path to access the files in the object store - */ - @NonNull - private String dataPath; + /** the country where this files repository resides */ + @NonNull private String country; - /** - * url path to access metadata about the files. - */ - private String metadataPath; + /** based url of the host of this repository metadata */ + @NonNull private String url; - /** - * the block storage type of files (s3 usually) - */ - @NonNull - private StorageType storageType; + /** the block storage type of files (s3 usually) */ + @NonNull private StorageType storageType; - /** - * the organization the owns this files repository - */ - @NonNull - private String organization; + /** the organization the owns this files repository */ + @NonNull private String organization; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java index 44341149..36d51d67 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Analysis.java @@ -20,16 +20,12 @@ import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; import com.fasterxml.jackson.annotation.JsonAnyGetter; import com.fasterxml.jackson.annotation.JsonAnySetter; -import lombok.*; - import java.util.List; import java.util.Map; import java.util.TreeMap; +import lombok.*; -/** - * This corresponds to the analysis entity in the file metadata repository. - * - */ +/** This corresponds to the analysis entity in the file metadata repository. */ @Getter @Builder @ToString @@ -38,60 +34,41 @@ @EqualsAndHashCode public class Analysis { - @NonNull - @ExclusionId - private String analysisId; + @NonNull @ExclusionId private String analysisId; - /** - * Method used in this analysis (variantCall, sequenceRead) - */ - @NonNull - private AnalysisTypeId analysisType; + /** Method used in this analysis (variantCall, sequenceRead) */ + @NonNull private AnalysisTypeId analysisType; - /** - * the status of the analysis (published or other values) - */ - @NonNull - private String analysisState; + /** the status of the analysis (published or other values) */ + @NonNull private String analysisState; - /** - * the studyId Id that this analysis belongs to. - */ - @NonNull - private String studyId; + /** the studyId Id that this analysis belongs to. */ + @NonNull private String studyId; - /** - * multiple files belong to an analysis, files can be related (bam, bai, xml) - */ - @NonNull - private List files; + /** multiple files belong to an analysis, files can be related (bam, bai, xml) */ + @NonNull private List files; - /** - * An analysis can have one or more samples - */ - @NonNull - private List samples; + /** An analysis can have one or more samples */ + @NonNull private List samples; - /** - * extra information about the analysis type. - * this will contain attributes that change by the analysis type. - *

- * see the source repository api for more info if needed. - */ - private Map experiment; + /** + * extra information about the analysis type. this will contain attributes that change by the + * analysis type. + * + *

see the source repository api for more info if needed. + */ + private Map experiment; - /** - * data represents song dynamic fields. - */ - @NonNull private final Map data = new TreeMap<>(); + /** data represents song dynamic fields. */ + @NonNull private final Map data = new TreeMap<>(); - @JsonAnyGetter - public Map getData() { - return data; - } + @JsonAnyGetter + public Map getData() { + return data; + } - @JsonAnySetter - public void setData(String key, Object value) { - data.put(key, value); - } + @JsonAnySetter + public void setData(String key, Object value) { + data.put(key, value); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java index 326169b8..d8371b15 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Donor.java @@ -18,14 +18,10 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; -/** - * This represents the samples donor - * A donor is part of the {@link Sample} entity. - */ +/** This represents the samples donor A donor is part of the {@link Sample} entity. */ @Getter @Builder @ToString @@ -33,33 +29,18 @@ @AllArgsConstructor @EqualsAndHashCode public class Donor { - /** - * The id of this donor - */ - @NonNull - @ExclusionId - private String donorId; + /** The id of this donor */ + @NonNull @ExclusionId private String donorId; - /** - * the id as submitted by the analysis creator - */ - @NonNull - private String submitterDonorId; + /** the id as submitted by the analysis creator */ + @NonNull private String submitterDonorId; - /** - * the studyId which this donor belongs to - */ - @NonNull - private String studyId; + /** the studyId which this donor belongs to */ + @NonNull private String studyId; - /** - * can be Male, Female, Other - */ - @NonNull - private String gender; + /** can be Male, Female, Other */ + @NonNull private String gender; - /** - * for extra information if any. - */ - private Map info; + /** for extra information if any. */ + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java index 0ce67baf..e76a2a76 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/File.java @@ -18,16 +18,15 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; /** * A file represents an analysis output that results from the experiment on the Donor specimen. * multiple files belong to one Analysis and reside in an object store. * - * A file can reside in multiple repositories and it can have relations to other files in a single analysis - * like BAM and its index file BAI. + *

A file can reside in multiple repositories and it can have relations to other files in a + * single analysis like BAM and its index file BAI. */ @Getter @Builder @@ -36,22 +35,14 @@ @AllArgsConstructor @EqualsAndHashCode public class File { - @NonNull - @ExclusionId - private String objectId; - @NonNull - private String studyId; - @NonNull - private String analysisId; - @NonNull - private String fileName; - @NonNull - private String fileType; - @NonNull - private String fileMd5sum; - @NonNull - private String fileAccess; - private String dataType; - private long fileSize; - private Map info; + @NonNull @ExclusionId private String objectId; + @NonNull private String studyId; + @NonNull private String analysisId; + @NonNull private String fileName; + @NonNull private String fileType; + @NonNull private String fileMd5sum; + @NonNull private String fileAccess; + private String dataType; + private long fileSize; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java index d667bfee..e272ac5e 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Sample.java @@ -18,13 +18,11 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import lombok.*; - import java.util.Map; +import lombok.*; /** - * Many samples can belong to an Analysis, a samples represents - * a donor and a specimen composition. + * Many samples can belong to an Analysis, a samples represents a donor and a specimen composition. */ @Getter @Builder @@ -33,13 +31,12 @@ @EqualsAndHashCode @AllArgsConstructor public class Sample { - @ExclusionId - private String sampleId; - private String specimenId; - private String submitterSampleId; - private String matchedNormalSubmitterSampleId; - private String sampleType; - private Donor donor; - private Specimen specimen; - private Map info; + @ExclusionId private String sampleId; + private String specimenId; + private String submitterSampleId; + private String matchedNormalSubmitterSampleId; + private String sampleType; + private Donor donor; + private Specimen specimen; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java index 3989c212..8d80ebef 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Specimen.java @@ -18,14 +18,10 @@ package bio.overture.maestro.domain.entities.metadata.study; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionId; -import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.*; - import java.util.Map; +import lombok.*; -/** - * A Specimen provides information about the source of the samples - */ +/** A Specimen provides information about the source of the samples */ @Getter @Builder @ToString @@ -33,12 +29,11 @@ @AllArgsConstructor @EqualsAndHashCode public class Specimen { - @ExclusionId - private String specimenId; - private String donorId; - private String submitterSpecimenId; - private String tumourNormalDesignation; - private String specimenTissueSource; - private String specimenType; - private Map info; + @ExclusionId private String specimenId; + private String donorId; + private String submitterSpecimenId; + private String tumourNormalDesignation; + private String specimenTissueSource; + private String specimenType; + private Map info; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java index 92015eb0..148dcd66 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/entities/metadata/study/Study.java @@ -27,7 +27,5 @@ @EqualsAndHashCode @AllArgsConstructor public class Study { - @NonNull - @ExclusionId - private String studyId; + @NonNull @ExclusionId private String studyId; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java index 5470e375..078f070c 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/AnalysisCentricIndexAdapter.java @@ -4,8 +4,10 @@ import lombok.NonNull; import reactor.core.publisher.Mono; -public interface AnalysisCentricIndexAdapter { +public interface AnalysisCentricIndexAdapter { - Mono batchUpsertAnalysisRepositories(@NonNull BatchIndexAnalysisCommand batchIndexAnalysisCommand); + Mono batchUpsertAnalysisRepositories( + @NonNull BatchIndexAnalysisCommand batchIndexAnalysisCommand); + Mono removeAnalysisDocs(String analysisId); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java index 1b9faae2..cbab11d6 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexAnalysisCommand.java @@ -1,9 +1,8 @@ package bio.overture.maestro.domain.port.outbound.indexing; import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDocument; -import lombok.*; - import java.util.List; +import lombok.*; @Getter @Builder @@ -12,8 +11,7 @@ @EqualsAndHashCode public class BatchIndexAnalysisCommand { - @NonNull - private List analyses; + @NonNull private List analyses; public String toString() { val size = analyses == null ? "null" : String.valueOf(analyses.size()); diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java index 666a753a..57ca1708 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/BatchIndexFilesCommand.java @@ -18,9 +18,8 @@ package bio.overture.maestro.domain.port.outbound.indexing; import bio.overture.maestro.domain.entities.indexing.FileCentricDocument; -import lombok.*; - import java.util.List; +import lombok.*; @Getter @Builder @@ -29,12 +28,11 @@ @EqualsAndHashCode public class BatchIndexFilesCommand { - @NonNull - private List files; + @NonNull private List files; - // avoid dumping all files info as that's too much - public String toString() { - val size = files == null ? "null" : String.valueOf(files.size()); - return super.toString() + "[files = " + size + "]"; - } + // avoid dumping all files info as that's too much + public String toString() { + val size = files == null ? "null" : String.valueOf(files.size()); + return super.toString() + "[files = " + size + "]"; + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java index 62af9b7f..ca0f0bcd 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/FileCentricIndexAdapter.java @@ -19,43 +19,42 @@ import bio.overture.maestro.domain.api.message.IndexResult; import bio.overture.maestro.domain.entities.indexing.FileCentricDocument; -import lombok.NonNull; -import reactor.core.publisher.Mono; import java.util.List; import java.util.Set; +import lombok.NonNull; +import reactor.core.publisher.Mono; /** - * Adapter for the indexing server client, this provides the indexer with needed APIs - * to index files centric documents in the index server + * Adapter for the indexing server client, this provides the indexer with needed APIs to index files + * centric documents in the index server */ public interface FileCentricIndexAdapter { - /** - * Updates a fileDocument repositories field, or indexes the whole document if doesn't exist. - * - * @param batchIndexFilesCommand requires the full document to insert if doesn't exist. - * @return flag indicating if the operation was successful. - */ - Mono batchUpsertFileRepositories(@NonNull BatchIndexFilesCommand batchIndexFilesCommand); - - /** - * Batch fetch documents from the index by the specified ids. - * @param ids a list of ids to fetch documents by from elastic search. - * @return List contains found documents. - */ - Mono> fetchByIds(List ids); - - /** - * Method to delete files documents from the files centric index - * - * @param fileCentricDocumentIds the list of files to delete - * @return indexer exception instance contains the list of failures. - */ - Mono removeFiles(Set fileCentricDocumentIds); - - /** - * Remove all files documents related to the specified analysisId - */ - Mono removeAnalysisFiles(String analysisId); - + /** + * Updates a fileDocument repositories field, or indexes the whole document if doesn't exist. + * + * @param batchIndexFilesCommand requires the full document to insert if doesn't exist. + * @return flag indicating if the operation was successful. + */ + Mono batchUpsertFileRepositories( + @NonNull BatchIndexFilesCommand batchIndexFilesCommand); + + /** + * Batch fetch documents from the index by the specified ids. + * + * @param ids a list of ids to fetch documents by from elastic search. + * @return List contains found documents. + */ + Mono> fetchByIds(List ids); + + /** + * Method to delete files documents from the files centric index + * + * @param fileCentricDocumentIds the list of files to delete + * @return indexer exception instance contains the list of failures. + */ + Mono removeFiles(Set fileCentricDocumentIds); + + /** Remove all files documents related to the specified analysisId */ + Mono removeAnalysisFiles(String analysisId); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java index ba9194aa..82adaab7 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/indexing/rules/ExclusionRulesDAO.java @@ -18,11 +18,10 @@ package bio.overture.maestro.domain.port.outbound.indexing.rules; import bio.overture.maestro.domain.entities.indexing.rules.ExclusionRule; -import reactor.core.publisher.Mono; - import java.util.List; import java.util.Map; +import reactor.core.publisher.Mono; public interface ExclusionRulesDAO { - Mono, List>> getExclusionRules(); + Mono, List>> getExclusionRules(); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java index 931577d3..03ab2fc5 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/repository/StudyRepositoryDAO.java @@ -22,14 +22,16 @@ import reactor.core.publisher.Mono; /** - * Provides access to data about the studies repositories that the indexer should read metadata from to get information - * like: url, repository storage type, urls etc + * Provides access to data about the studies repositories that the indexer should read metadata from + * to get information like: url, repository storage type, urls etc */ public interface StudyRepositoryDAO { - /** - * Gets a files repository by code - * @param code the unique code of the repository - * @return the repository or empty null if not found - */ - @NonNull Mono getFilesRepository(@NonNull String code); + /** + * Gets a files repository by code + * + * @param code the unique code of the repository + * @return the repository or empty null if not found + */ + @NonNull + Mono getFilesRepository(@NonNull String code); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java index 069edddf..e58385eb 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAllStudiesCommand.java @@ -26,6 +26,5 @@ @EqualsAndHashCode @AllArgsConstructor public class GetAllStudiesCommand { - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java index b33e4b9f..d0198a28 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetAnalysisCommand.java @@ -26,10 +26,7 @@ @AllArgsConstructor @EqualsAndHashCode public class GetAnalysisCommand { - @NonNull - private String analysisId; - @NonNull - private String studyId; - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String analysisId; + @NonNull private String studyId; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java index 77d6dd1a..49e39254 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/GetStudyAnalysesCommand.java @@ -26,8 +26,6 @@ @EqualsAndHashCode @AllArgsConstructor public class GetStudyAnalysesCommand { - @NonNull - private String studyId; - @NonNull - private String filesRepositoryBaseUrl; + @NonNull private String studyId; + @NonNull private String filesRepositoryBaseUrl; } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java index 6ed06025..4a71e7a9 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/metadata/study/StudyDAO.java @@ -19,38 +19,43 @@ import bio.overture.maestro.domain.entities.metadata.study.Analysis; import bio.overture.maestro.domain.entities.metadata.study.Study; +import java.util.List; import lombok.NonNull; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; -import java.util.List; - /** - * The studyId repository, which provides APIs to read Studies, analyses, etc information from a StudyRepository + * The studyId repository, which provides APIs to read Studies, analyses, etc information from a + * StudyRepository */ public interface StudyDAO { - /** - * loads analyses for a single studyId from a single repository - * - * @param getStudyAnalysesCommand contains studyId and repository base url - * @return a mono of all analyses found related to a studyId. - * @throws bio.overture.maestro.domain.api.exception.NotFoundException - * in case the studyId wasn't found. - */ - @NonNull Mono> getStudyAnalyses(@NonNull GetStudyAnalysesCommand getStudyAnalysesCommand); + /** + * loads analyses for a single studyId from a single repository + * + * @param getStudyAnalysesCommand contains studyId and repository base url + * @return a mono of all analyses found related to a studyId. + * @throws bio.overture.maestro.domain.api.exception.NotFoundException in case the studyId wasn't + * found. + */ + @NonNull + Mono> getStudyAnalyses(@NonNull GetStudyAnalysesCommand getStudyAnalysesCommand); - /** - * loads all studies in a repository - * @param getStudyAnalysesCommand contains repository url - * @return a flux of all the studies in the specified repository - */ - @NonNull Flux getStudies(@NonNull GetAllStudiesCommand getStudyAnalysesCommand); + /** + * loads all studies in a repository + * + * @param getStudyAnalysesCommand contains repository url + * @return a flux of all the studies in the specified repository + */ + @NonNull + Flux getStudies(@NonNull GetAllStudiesCommand getStudyAnalysesCommand); - /** - * load an analysis of a studyId from a repository - * @param command specifies the analysis id, the studyId and the repository url - * @return the analysis mono - */ - @NonNull Mono getAnalysis(@NonNull GetAnalysisCommand command); + /** + * load an analysis of a studyId from a repository + * + * @param command specifies the analysis id, the studyId and the repository url + * @return the analysis mono + */ + @NonNull + Mono getAnalysis(@NonNull GetAnalysisCommand command); } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java index 47e9d758..3c4200fe 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/port/outbound/notification/IndexerNotification.java @@ -17,24 +17,22 @@ package bio.overture.maestro.domain.port.outbound.notification; +import static java.text.MessageFormat.format; import bio.overture.maestro.domain.api.NotificationName; +import java.util.Map; import lombok.AllArgsConstructor; import lombok.EqualsAndHashCode; import lombok.Getter; -import java.util.Map; - -import static java.text.MessageFormat.format; - @Getter @EqualsAndHashCode @AllArgsConstructor public class IndexerNotification { - private final NotificationName notificationName; - private final Map attributes; + private final NotificationName notificationName; + private final Map attributes; - public String toString() { - return format("{0} | {1}", notificationName.name().toUpperCase(), attributes); - } + public String toString() { + return format("{0} | {1}", notificationName.name().toUpperCase(), attributes); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java index 11bc083d..339350ee 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/CollectionsUtil.java @@ -17,22 +17,20 @@ package bio.overture.maestro.domain.utility; -import lombok.experimental.UtilityClass; +import static java.lang.Math.min; +import static java.util.stream.Collectors.toMap; import java.util.List; import java.util.Map; import java.util.stream.IntStream; - -import static java.lang.Math.min; -import static java.util.stream.Collectors.toMap; +import lombok.experimental.UtilityClass; @UtilityClass public class CollectionsUtil { - public static Map> partitionList(List list, int partSize) { - return IntStream.iterate(0, i -> i + partSize) - .limit((list.size() + partSize - 1) / partSize) - .boxed() - .collect(toMap(i -> i / partSize, - i -> list.subList(i, min(i + partSize, list.size())))); - } + public static Map> partitionList(List list, int partSize) { + return IntStream.iterate(0, i -> i + partSize) + .limit((list.size() + partSize - 1) / partSize) + .boxed() + .collect(toMap(i -> i / partSize, i -> list.subList(i, min(i + partSize, list.size())))); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java index ef7ae86d..51d30a4a 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Exceptions.java @@ -17,27 +17,28 @@ package bio.overture.maestro.domain.utility; +import static java.text.MessageFormat.format; + import bio.overture.maestro.domain.api.exception.BadDataException; import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.exception.NotFoundException; import lombok.experimental.UtilityClass; -import static java.text.MessageFormat.format; - @UtilityClass public final class Exceptions { - public static Exception notFound(String msg, Object ...args) { - return new NotFoundException(format(msg, args)); - } + public static Exception notFound(String msg, Object... args) { + return new NotFoundException(format(msg, args)); + } - public static Exception badData(String msg, Object ...args) { - return new BadDataException(format(msg, args)); - } + public static Exception badData(String msg, Object... args) { + return new BadDataException(format(msg, args)); + } - public static IndexerException wrapWithIndexerException(Throwable e, String message, FailureData failureData) { - if (e instanceof IndexerException) return (IndexerException) e; - return new IndexerException(message, e, failureData); - } + public static IndexerException wrapWithIndexerException( + Throwable e, String message, FailureData failureData) { + if (e instanceof IndexerException) return (IndexerException) e; + return new IndexerException(message, e, failureData); + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java index b595153a..b06df40f 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/Parallel.java @@ -17,57 +17,62 @@ package bio.overture.maestro.domain.utility; -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; -import lombok.extern.slf4j.Slf4j; -import lombok.val; +import static bio.overture.maestro.domain.utility.CollectionsUtil.partitionList; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Function; import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.utility.CollectionsUtil.partitionList; +import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; +import lombok.extern.slf4j.Slf4j; +import lombok.val; @Slf4j @UtilityClass public final class Parallel { - /** - * A BLOCKING (i.e. should be executed on a reactor scheduler) scatter gather to parallelize execution - * of suppliers, currently it uses completeable futures but may be transformed to reactor publishers. - * it handles paritioning an input list to a batch and create the necessary number of workers. - * it uses the default completeable future work stealing ForkJoin pool. - * - * @param inputList the list of input parameters that we want to split and pass to the supplier - * @param batchSize the size of the batch each supplier will handler - * @param supplier the function to take an input batch and supply the result - * @param the type parameter for the inputs - * @param the return type of the results - * @return a list of R - * @throws Exception this is here to force callers to handle the exceptions that can be thrown - * this method will only rethrow any exceptiona and fails fast. - * if another behaviour is desired suppliers should handle their exceptions. - */ - @SneakyThrows - public static List blockingScatterGather(List inputList, int batchSize, - Function>, R> supplier) throws Exception { - // scatter - val futures = partitionList(inputList, batchSize).entrySet().stream() + /** + * A BLOCKING (i.e. should be executed on a reactor scheduler) scatter gather to parallelize + * execution of suppliers, currently it uses completeable futures but may be transformed to + * reactor publishers. it handles paritioning an input list to a batch and create the necessary + * number of workers. it uses the default completeable future work stealing ForkJoin pool. + * + * @param inputList the list of input parameters that we want to split and pass to the supplier + * @param batchSize the size of the batch each supplier will handler + * @param supplier the function to take an input batch and supply the result + * @param the type parameter for the inputs + * @param the return type of the results + * @return a list of R + * @throws Exception this is here to force callers to handle the exceptions that can be thrown + * this method will only rethrow any exceptiona and fails fast. if another behaviour is + * desired suppliers should handle their exceptions. + */ + @SneakyThrows + public static List blockingScatterGather( + List inputList, int batchSize, Function>, R> supplier) + throws Exception { + // scatter + val futures = + partitionList(inputList, batchSize).entrySet().stream() .map((entry) -> CompletableFuture.supplyAsync(() -> supplier.apply(entry))) .collect(Collectors.toUnmodifiableList()); - //gather - val joinedFutures = CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) - .thenApply(v-> futures.stream().map(CompletableFuture::join).collect(Collectors.toUnmodifiableList())); + // gather + val joinedFutures = + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])) + .thenApply( + v -> + futures.stream() + .map(CompletableFuture::join) + .collect(Collectors.toUnmodifiableList())); - try { - return joinedFutures.get(); - } catch (Exception e) { - log.error(e.getMessage()); - throw e; - } + try { + return joinedFutures.get(); + } catch (Exception e) { + log.error(e.getMessage()); + throw e; } - + } } diff --git a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java index 3ee2e112..c7c35729 100644 --- a/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java +++ b/maestro-domain/src/main/java/bio/overture/maestro/domain/utility/StringUtilities.java @@ -17,32 +17,28 @@ package bio.overture.maestro.domain.utility; +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; import lombok.NonNull; import lombok.SneakyThrows; import lombok.experimental.UtilityClass; import lombok.val; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; - -import static java.nio.charset.StandardCharsets.UTF_8; - @UtilityClass public final class StringUtilities { - /** - * loads a string out of input stream. - */ - @SneakyThrows - public static String inputStreamToString(@NonNull InputStream inputStream) { - try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { - val buffer = new byte[1024]; - int length; - while ((length = inputStream.read(buffer)) != -1) { - result.write(buffer, 0, length); - } - return result.toString(UTF_8); - } + /** loads a string out of input stream. */ + @SneakyThrows + public static String inputStreamToString(@NonNull InputStream inputStream) { + try (ByteArrayOutputStream result = new ByteArrayOutputStream()) { + val buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) != -1) { + result.write(buffer, 0, length); + } + return result.toString(UTF_8); } - + } } diff --git a/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java b/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java index 7bd4e05b..b63469a6 100644 --- a/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java +++ b/maestro-domain/src/main/java/bio/overture/masestro/test/Fixture.java @@ -17,118 +17,134 @@ package bio.overture.masestro.test; +import static bio.overture.maestro.domain.utility.StringUtilities.inputStreamToString; + import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.PropertyNamingStrategy; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; - import java.io.File; import java.io.IOException; import java.util.Map; import java.util.Optional; import java.util.regex.Pattern; - -import static bio.overture.maestro.domain.utility.StringUtilities.inputStreamToString; +import lombok.*; +import lombok.experimental.UtilityClass; /** - * Helper to load test fixtures from resources. - * This class is only for testing purposes not to be used for non test code. + * Helper to load test fixtures from resources. This class is only for testing purposes not to be + * used for non test code. */ @UtilityClass public class Fixture { - private final static String BASE_PATH = "fixtures" + File.separator; - private final static ObjectMapper MAPPER = new ObjectMapper(); + private static final String FIXTURE_PATH = "src/test/resources/"; + private static final String BASE_PATH = "fixtures" + File.separator; + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @SneakyThrows + public static T loadJsonFixture(Class clazz, String fileName, Class targetClass) { + return loadJsonFixture(clazz, fileName, targetClass, MAPPER); + } + + public static T loadConverterTestFixture(String fileName, Class targetClass) { + return readConverterFixture(fileName, targetClass, MAPPER); + } - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, Class targetClass) { - return loadJsonFixture(clazz, fileName, targetClass, MAPPER); - } + @SneakyThrows + public static T loadJsonFixtureSnakeCase(Class clazz, String fileName, Class targetClass) { + ObjectMapper mapper = + new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); + return loadJsonFixture(clazz, fileName, targetClass, mapper); + } - @SneakyThrows - public static T loadJsonFixtureSnakeCase(Class clazz, String fileName, Class targetClass) { - ObjectMapper mapper = new ObjectMapper().setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE); - return loadJsonFixture(clazz, fileName, targetClass, mapper); - } + /** Use this overload for generics */ + @SneakyThrows + public static T loadJsonFixture(Class clazz, String fileName, TypeReference type) { + return loadJsonFixture(clazz, fileName, type, MAPPER); + } + /** + * this overload can be used to load json files and convert it to the target class using a custom + * mapper + * + * @param clazz will be used to obtain a class loader to get the resources for. + * @param fileName the fixture files we want to load + * @param targetClass the target java type we want to convert the json to + * @param customMapper in case you want to pre configure a mapper (property name case for example) + * @param type parameter of the target class + * @param templateParams parameters map to be replaced in the json files use this if you have + * dynamic values that change each test run the placeholder should be ##key## and will be + * replaced with the value : templateParams.get(key) + * @return the converted json files as java type + */ + @SneakyThrows + public static T loadJsonFixture( + Class clazz, + String fileName, + Class targetClass, + ObjectMapper customMapper, + Map templateParams) { + String json = loadJsonString(clazz, fileName); + TemplateResult replaceResult = new TemplateResult(); + replaceResult.setResult(json); + templateParams.forEach( + (name, value) -> + replaceResult.setResult( + replaceResult.getResult().replaceAll(Pattern.quote("##" + name + "##"), value))); - /** - * Use this overload for generics - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, TypeReference type) { - return loadJsonFixture(clazz, fileName, type, MAPPER); - } - /** - * this overload can be used to load json files and convert it to the target class using a custom mapper - * - * @param clazz will be used to obtain a class loader to get the resources for. - * @param fileName the fixture files we want to load - * @param targetClass the target java type we want to convert the json to - * @param customMapper in case you want to pre configure a mapper (property name case for example) - * @param type parameter of the target class - * @param templateParams parameters map to be replaced in the json files - * use this if you have dynamic values that change each test run - * the placeholder should be ##key## and will be replaced with the value : - * templateParams.get(key) - * - * @return the converted json files as java type - * - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, - String fileName, - Class targetClass, - ObjectMapper customMapper, - Map templateParams) { - String json = loadJsonString(clazz, fileName); - TemplateResult replaceResult = new TemplateResult(); - replaceResult.setResult(json); - templateParams.forEach((name, value) -> replaceResult.setResult(replaceResult.getResult() - .replaceAll(Pattern.quote("##" + name + "##"), value))); + return customMapper.readValue(replaceResult.getResult(), targetClass); + } - return customMapper.readValue(replaceResult.getResult(), targetClass); - } + /** + * this overload can be used to load json files and convert it to the target class using a custom + * mapper + * + * @param clazz will be used to obtain a class loader to get the resources for. + * @param fileName the fixture files we want to load + * @param targetClass the target java type we want to convert the json to + * @param customMapper in case you want to pre configure a mapper (property name case for example) + * @param type parameter of the target class + * @return the converted json files as java type + */ + @SneakyThrows + public static T loadJsonFixture( + Class clazz, String fileName, Class targetClass, ObjectMapper customMapper) { + String json = loadJsonString(clazz, fileName); + return customMapper.readValue(json, targetClass); + } - /** - * this overload can be used to load json files and convert it to the target class using a custom mapper - * - * @param clazz will be used to obtain a class loader to get the resources for. - * @param fileName the fixture files we want to load - * @param targetClass the target java type we want to convert the json to - * @param customMapper in case you want to pre configure a mapper (property name case for example) - * @param type parameter of the target class - * - * @return the converted json files as java type - * - */ - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, Class targetClass, ObjectMapper customMapper) { - String json = loadJsonString(clazz, fileName); - return customMapper.readValue(json, targetClass); - } + @SneakyThrows + public static T loadJsonFixture( + Class clazz, String fileName, TypeReference targetClass, ObjectMapper customMapper) { + String json = loadJsonString(clazz, fileName); + return customMapper.readValue(json, targetClass); + } - @SneakyThrows - public static T loadJsonFixture(Class clazz, String fileName, TypeReference targetClass, ObjectMapper customMapper) { - String json = loadJsonString(clazz, fileName); - return customMapper.readValue(json, targetClass); - } + @SneakyThrows + public static T readConverterFixture( + String fileName, Class targetClass, ObjectMapper customMapper) { + return customMapper.readValue( + new File(FIXTURE_PATH + BASE_PATH + "DocumentConverterTest" + File.separator + fileName), + targetClass); + } - public static String loadJsonString(Class clazz, String fileName) throws IOException { - return inputStreamToString( - Optional.ofNullable(clazz.getClassLoader() - .getResource(BASE_PATH + clazz.getSimpleName() + File.separator + fileName) - ).orElseThrow(() -> new RuntimeException("fixture not found. make sure you created the correct " + - "folder if this is a new class or if you renamed the class")).openStream()); - } + public static String loadJsonString(Class clazz, String fileName) throws IOException { + return inputStreamToString( + Optional.ofNullable( + clazz + .getClassLoader() + .getResource(BASE_PATH + clazz.getSimpleName() + File.separator + fileName)) + .orElseThrow( + () -> + new RuntimeException( + "fixture not found. make sure you created the correct " + + "folder if this is a new class or if you renamed the class")) + .openStream()); + } - @Getter - @Setter - @NoArgsConstructor - private static class TemplateResult { - private String result; - } + @Getter + @Setter + @NoArgsConstructor + private static class TemplateResult { + private String result; + } } diff --git a/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java b/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java index 64006977..fccc52de 100644 --- a/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java +++ b/maestro-domain/src/main/java/bio/overture/masestro/test/TestCategory.java @@ -21,6 +21,6 @@ @UtilityClass public class TestCategory { - public final static String UNIT_TEST = "unit_test"; - public final static String INT_TEST = "int_test"; + public static final String UNIT_TEST = "unit_test"; + public static final String INT_TEST = "int_test"; } diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java deleted file mode 100644 index 5cef0587..00000000 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/AnalysisCentricDocumentConverterTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package bio.overture.maestro.domain.api; - -import bio.overture.maestro.domain.entities.indexing.Specimen; -import bio.overture.maestro.domain.entities.indexing.analysis.AnalysisCentricDonor; -import bio.overture.maestro.domain.entities.metadata.study.*; -import lombok.val; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.junit.jupiter.MockitoExtension; -import java.util.ArrayList; -import java.util.List; - -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.TestCategory.UNIT_TEST; -import static org.junit.jupiter.api.Assertions.*; - -@ExtendWith(MockitoExtension.class) -@Tag(UNIT_TEST) -public class AnalysisCentricDocumentConverterTest { - - @Test - void testGetDonors(){ - val analysisObj = loadJsonFixture(this.getClass(), - "TEST-CA.analysis.multi-donor.json", Analysis.class); - - // expected result: - val donor_1 = AnalysisCentricDonor.builder() - .id("DO1") - .gender("female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor1()) - .build(); - - val donor_2 = AnalysisCentricDonor.builder() - .id("DO2") - .gender("female") - .submitterDonorId("MDT-AP-0749") - .specimens(buildSpecimenListForDonor2()) - .build(); - - val results = AnalysisCentricDocumentConverter.getDonors(analysisObj); - - assertNotNull(results); - assertEquals(results.size(), 2); - assertTrue(results.contains(donor_1)); - assertTrue(results.contains(donor_2)); - } - - private List buildSpecimenListForDonor1() { - val specimen_1 = bio.overture.maestro.domain.entities.indexing.Specimen.builder() - .id("SP1") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(bio.overture.maestro.domain.entities.indexing.Sample.builder() - .id("SA1") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build()) - .build(); - - val specimen_2 = bio.overture.maestro.domain.entities.indexing.Specimen.builder() - .id("SP2") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(bio.overture.maestro.domain.entities.indexing.Sample.builder() - .id("SA2") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build()) - .build(); - - val list = new ArrayList(); - list.add(specimen_1); - list.add(specimen_2); - return list; - } - - private List buildSpecimenListForDonor2() { - val specimen_1 = bio.overture.maestro.domain.entities.indexing.Specimen.builder() - .id("SP3") - .specimenTissueSource("Other") - .submitterSpecimenId("MDT-AP-0749_tumor_specimen") - .tumourNormalDesignation("Tumour") - .specimenType("Primary tumour - solid tissue") - .samples(bio.overture.maestro.domain.entities.indexing.Sample.builder() - .id("SA3") - .sampleType("DNA") - .submitterSampleId("MDT-AP-0749_tumor") - .matchedNormalSubmitterSampleId("PCSI_0216_St_R") - .build()) - .build(); - - val list = new ArrayList(); - list.add(specimen_1); - return list; - } -} diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java index 02d91687..7f9698bd 100644 --- a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DefaultIndexerTest.java @@ -17,6 +17,16 @@ package bio.overture.maestro.domain.api; +import static bio.overture.maestro.domain.api.DefaultIndexer.REPO_CODE; +import static bio.overture.masestro.test.Fixture.loadJsonFixture; +import static bio.overture.masestro.test.Fixture.loadJsonFixtureSnakeCase; +import static bio.overture.masestro.test.TestCategory.UNIT_TEST; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.BDDMockito.given; +import static org.mockito.BDDMockito.then; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; + import bio.overture.maestro.domain.api.exception.FailureData; import bio.overture.maestro.domain.api.exception.IndexerException; import bio.overture.maestro.domain.api.message.*; @@ -39,7 +49,11 @@ import bio.overture.maestro.domain.port.outbound.metadata.study.GetStudyAnalysesCommand; import bio.overture.maestro.domain.port.outbound.metadata.study.StudyDAO; import bio.overture.maestro.domain.port.outbound.notification.IndexerNotification; -import io.vavr.Tuple2; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import lombok.SneakyThrows; import lombok.val; import org.jetbrains.annotations.NotNull; @@ -54,166 +68,148 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.test.StepVerifier; -import java.util.*; -import java.util.stream.Collectors; - -import static bio.overture.maestro.domain.api.DefaultIndexer.REPO_CODE; -import static bio.overture.masestro.test.Fixture.loadJsonFixture; -import static bio.overture.masestro.test.Fixture.loadJsonFixtureSnakeCase; -import static bio.overture.masestro.test.TestCategory.UNIT_TEST; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.BDDMockito.given; -import static org.mockito.BDDMockito.then; -import static org.mockito.Mockito.*; - @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.WARN) @Tag(UNIT_TEST) class DefaultIndexerTest { - @Mock - private StudyRepositoryDAO studyRepositoryDao; + @Mock private StudyRepositoryDAO studyRepositoryDao; - @Mock - private ExclusionRulesDAO exclusionRulesDAO; + @Mock private ExclusionRulesDAO exclusionRulesDAO; - @Mock - private StudyDAO studyDAO; + @Mock private StudyDAO studyDAO; - @Mock - private FileCentricIndexAdapter indexServerAdapter; + @Mock private FileCentricIndexAdapter indexServerAdapter; - @Mock - private AnalysisCentricIndexAdapter analysisCentricIndexAdapter; + @Mock private AnalysisCentricIndexAdapter analysisCentricIndexAdapter; - @Mock - private IndexEnabledProperties indexEnabledProperties; + @Mock private IndexEnabledProperties indexEnabledProperties; - private DefaultIndexer indexer; + private DefaultIndexer indexer; - @Mock - private Notifier notifier; + @Mock private Notifier notifier; - @BeforeEach - void setUp() { - reset(studyRepositoryDao, studyDAO, indexServerAdapter, analysisCentricIndexAdapter, notifier); - given(indexEnabledProperties.isFileCentricEnabled()).willReturn(Boolean.TRUE); - this.indexer = new DefaultIndexer(indexServerAdapter, analysisCentricIndexAdapter, studyDAO, studyRepositoryDao, - exclusionRulesDAO, notifier, indexEnabledProperties); - } + @BeforeEach + void setUp() { + reset(studyRepositoryDao, studyDAO, indexServerAdapter, analysisCentricIndexAdapter, notifier); + given(indexEnabledProperties.isFileCentricEnabled()).willReturn(Boolean.TRUE); + this.indexer = + new DefaultIndexer( + indexServerAdapter, + analysisCentricIndexAdapter, + studyDAO, + studyRepositoryDao, + exclusionRulesDAO, + notifier, + indexEnabledProperties); + } - @Test - void indexStudyshouldNotifyOnStudyFetchError() { - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val repositoryMono = Mono.just(filesRepository); - val failure = FailureData.builder() - .failingIds(Map.of("studyId", Set.of("anyStudy"))) - .build(); - val output = IndexResult.builder() - .failureData(failure) - .successful(false) - .build(); + @Test + void indexStudyshouldNotifyOnStudyFetchError() { + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val repositoryMono = Mono.just(filesRepository); + val failure = FailureData.builder().failingIds(Map.of("studyId", Set.of("anyStudy"))).build(); + val output = IndexResult.builder().failureData(failure).successful(false).build(); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(repositoryMono); - given(studyDAO.getStudyAnalyses(any(GetStudyAnalysesCommand.class))) - .willReturn(Mono.error(new IndexerException("failed", new RuntimeException(""), failure))); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(repositoryMono); + given(studyDAO.getStudyAnalyses(any(GetStudyAnalysesCommand.class))) + .willReturn(Mono.error(new IndexerException("failed", new RuntimeException(""), failure))); - // When - val indexResult = indexer.indexStudy(IndexStudyCommand.builder() - .studyId("anyStudy") - .repositoryCode(filesRepository.getCode()) - .build() - ); + // When + val indexResult = + indexer.indexStudy( + IndexStudyCommand.builder() + .studyId("anyStudy") + .repositoryCode(filesRepository.getCode()) + .build()); - // Then - StepVerifier.create(indexResult) - .expectNext(output) - .expectComplete() - .verify(); + // Then + StepVerifier.create(indexResult).expectNext(output).expectComplete().verify(); - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(indexServerAdapter).should(times(0)).batchUpsertFileRepositories(any()); - } + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(indexServerAdapter).should(times(0)).batchUpsertFileRepositories(any()); + } - @Test - void indexRepositoryShouldHandleFetchStudiesFailure() { - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val fileRepo = Mono.just(getStubFilesRepository()); - val failure = FailureData.builder() - .failingIds(Map.of(REPO_CODE, Set.of(repoCode))).build(); - val failedIndexResult = IndexResult.builder().failureData(failure).successful(false).build(); - val getStudiesCmd = GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); - - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); - given(studyDAO.getStudies(eq(getStudiesCmd))).willReturn(Flux.error(new RuntimeException("sike!"))); - - // When - val indexResultMono = indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode("TEST-REPO") - .build()); - - // Then - StepVerifier.create(indexResultMono) - .expectNext(failedIndexResult) - .expectComplete() - .verify(); - - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(studyDAO).should(times(0)).getStudyAnalyses(any()); - then(indexServerAdapter).should(times(0)).batchUpsertFileRepositories(any()); - then(notifier).should(times(1)).notify(any()); - } + @Test + void indexRepositoryShouldHandleFetchStudiesFailure() { + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val fileRepo = Mono.just(getStubFilesRepository()); + val failure = FailureData.builder().failingIds(Map.of(REPO_CODE, Set.of(repoCode))).build(); + val failedIndexResult = IndexResult.builder().failureData(failure).successful(false).build(); + val getStudiesCmd = + GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + given(studyDAO.getStudies(eq(getStudiesCmd))) + .willReturn(Flux.error(new RuntimeException("sike!"))); + + // When + val indexResultMono = + indexer.indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode("TEST-REPO").build()); + + // Then + StepVerifier.create(indexResultMono).expectNext(failedIndexResult).expectComplete().verify(); + + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(studyDAO).should(times(0)).getStudyAnalyses(any()); + then(indexServerAdapter).should(times(0)).batchUpsertFileRepositories(any()); + then(notifier).should(times(1)).notify(any()); + } - @Test - void shouldExcludeSampleIdFromIndexing() { - // Given - val studyId = "PACA-CA"; - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val a1 = getStudyAnalyses(studyId); - // load the fixture with - val fileCentricDocuments = Arrays.asList(loadJsonFixtureSnakeCase(getClass(), - studyId + ".files.excluded.SA520221.json", FileCentricDocument[].class)); - val repositoryMono = Mono.just(filesRepository); - val studyAnalyses = Mono.just(a1); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); - val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - val getStudyAnalysesCommand = GetStudyAnalysesCommand.builder() + @Test + void shouldExcludeSampleIdFromIndexing() { + // Given + val studyId = "PACA-CA"; + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val a1 = getStudyAnalyses(studyId); + // load the fixture with + val fileCentricDocuments = + Arrays.asList( + loadJsonFixtureSnakeCase( + getClass(), + studyId + ".files.excluded.SA520221.json", + FileCentricDocument[].class)); + val repositoryMono = Mono.just(filesRepository); + val studyAnalyses = Mono.just(a1); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); + val getStudyAnalysesCommand = + GetStudyAnalysesCommand.builder() .studyId(studyId) .filesRepositoryBaseUrl(filesRepository.getUrl()) .build(); - Mono, List>> sampleExclusionRule = Mono.just( - Map.of( - Sample.class, List.of(new IDExclusionRule(Sample.class, List.of("SA520221"))) - ) - ); - - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(repositoryMono); - given(studyDAO.getStudyAnalyses(eq(getStudyAnalysesCommand))).willReturn(studyAnalyses); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); - given(exclusionRulesDAO.getExclusionRules()).willReturn(sampleExclusionRule); - - // When - val indexResultFlux = indexer.indexStudy(IndexStudyCommand.builder() - .studyId(studyId) - .repositoryCode(filesRepository.getCode()) - .build()); + Mono, List>> sampleExclusionRule = + Mono.just( + Map.of(Sample.class, List.of(new IDExclusionRule(Sample.class, List.of("SA520221"))))); - // Then - StepVerifier.create(indexResultFlux) - .expectNext(result) - .expectComplete() - .verify(); + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(repositoryMono); + given(studyDAO.getStudyAnalyses(eq(getStudyAnalysesCommand))).willReturn(studyAnalyses); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); + given(exclusionRulesDAO.getExclusionRules()).willReturn(sampleExclusionRule); - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - } + // When + val indexResultFlux = + indexer.indexStudy( + IndexStudyCommand.builder() + .studyId(studyId) + .repositoryCode(filesRepository.getCode()) + .build()); + + // Then + StepVerifier.create(indexResultFlux).expectNext(result).expectComplete().verify(); + + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + } @Test void shouldDetectConflictInFileAndDeleteItFromIndex() { @@ -221,344 +217,419 @@ void shouldDetectConflictInFileAndDeleteItFromIndex() { val studyId = "MALY-DE"; val repoCode = "TEST-REPO"; val filesRepository = getStubFilesRepository(); - // Note: in MALE-DE.conflicting.analysis.json, conflict is: analysis 0a1df2a2-029d-48cc-839c-0a7c89ff972f is UNPUBLISHED - val a1 = Arrays.asList(loadJsonFixture(getClass(), studyId +".conflicting.analysis.json", Analysis[].class)); + // Note: in MALE-DE.conflicting.analysis.json, conflict is: analysis + // 0a1df2a2-029d-48cc-839c-0a7c89ff972f is UNPUBLISHED + val a1 = + Arrays.asList( + loadJsonFixture(getClass(), studyId + ".conflicting.analysis.json", Analysis[].class)); val fileCentricDocuments = getExpectedFileCentricDocument(studyId); val nonConflictingDocs = fileCentricDocuments.subList(1, fileCentricDocuments.size()); val fileRepo = Mono.just(getStubFilesRepository()); val studyAnalyses = Mono.just(a1); val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); + val monoResult = Mono.just(result); val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(nonConflictingDocs).build(); - val getStudyAnalysesCommand = GetStudyAnalysesCommand.builder() - .studyId(studyId) - .filesRepositoryBaseUrl(filesRepository.getUrl()) - .build(); - val expectedNotification = new IndexerNotification(NotificationName.INDEX_FILE_CONFLICT, - getConflicts(fileCentricDocuments)); + val getStudyAnalysesCommand = + GetStudyAnalysesCommand.builder() + .studyId(studyId) + .filesRepositoryBaseUrl(filesRepository.getUrl()) + .build(); + val expectedNotification = + new IndexerNotification( + NotificationName.INDEX_FILE_CONFLICT, getConflicts(fileCentricDocuments)); given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); given(studyDAO.getStudyAnalyses(eq(getStudyAnalysesCommand))).willReturn(studyAnalyses); - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of(fileCentricDocuments.get(0)))); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); + given(indexServerAdapter.fetchByIds(anyList())) + .willReturn(Mono.just(List.of(fileCentricDocuments.get(0)))); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); // When - val command = IndexStudyCommand.builder() - .studyId(studyId) - .repositoryCode(filesRepository.getCode()) - .build(); + val command = + IndexStudyCommand.builder() + .studyId(studyId) + .repositoryCode(filesRepository.getCode()) + .build(); val indexResultMono = indexer.indexStudy(command); // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); then(notifier).should(times(1)).notify(eq(expectedNotification)); - then(indexServerAdapter).should(times(1)).batchUpsertFileRepositories(eq(batchIndexFilesCommand)); - then(indexServerAdapter).should(times(0)).removeFiles(eq(Set.of(fileCentricDocuments.get(0).getObjectId()))); + then(indexServerAdapter) + .should(times(1)) + .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + then(indexServerAdapter) + .should(times(0)) + .removeFiles(eq(Set.of(fileCentricDocuments.get(0).getObjectId()))); } - @NotNull - private Map getConflicts(List fileCentricDocuments) { - return Map.of("conflicts", List.of(DefaultIndexer.FileConflict.builder() - .indexedFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(fileCentricDocuments.get(0).getStudyId()) - .analysisId(fileCentricDocuments.get(0).getAnalysis().getAnalysisId()) - .objectId(fileCentricDocuments.get(0).getObjectId()) - .repoCode(fileCentricDocuments.get(0).getRepositories().stream().map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet())) - .build() - ).newFile( - DefaultIndexer.ConflictingFile.builder() - .studyId(fileCentricDocuments.get(0).getStudyId()) - .analysisId(fileCentricDocuments.get(0).getAnalysis().getAnalysisId()) - .objectId(fileCentricDocuments.get(0).getObjectId()) - .repoCode(fileCentricDocuments.get(0).getRepositories().stream().map(Repository::getCode) - .collect(Collectors.toUnmodifiableSet())) - .build() - ).build())); - } + @NotNull + private Map getConflicts(List fileCentricDocuments) { + return Map.of( + "conflicts", + List.of( + DefaultIndexer.FileConflict.builder() + .indexedFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(fileCentricDocuments.get(0).getStudyId()) + .analysisId(fileCentricDocuments.get(0).getAnalysis().getAnalysisId()) + .objectId(fileCentricDocuments.get(0).getObjectId()) + .repoCode( + fileCentricDocuments.get(0).getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .newFile( + DefaultIndexer.ConflictingFile.builder() + .studyId(fileCentricDocuments.get(0).getStudyId()) + .analysisId(fileCentricDocuments.get(0).getAnalysis().getAnalysisId()) + .objectId(fileCentricDocuments.get(0).getObjectId()) + .repoCode( + fileCentricDocuments.get(0).getRepositories().stream() + .map(Repository::getCode) + .collect(Collectors.toUnmodifiableSet())) + .build()) + .build())); + } - @Test - void shouldIndexSingleAnalysis() { - // Given - val studyId = "PEME-CA"; - val repoCode = "TEST-REPO"; - val analysisId = "EGAZ00001254368"; - val filesRepository = getStubFilesRepository(); - val a1 = loadJsonFixture(getClass(), studyId +".analysis.EGAZ00001254368.json", Analysis.class); - val fileCentricDocuments = getExpectedFileCentricDocument(studyId); - val fileRepo = Mono.just(getStubFilesRepository()); - val studyAnalysis = Mono.just(a1); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); - val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - val getStudyAnalysesCommand = GetAnalysisCommand.builder() + @Test + void shouldIndexSingleAnalysis() { + // Given + val studyId = "PEME-CA"; + val repoCode = "TEST-REPO"; + val analysisId = "EGAZ00001254368"; + val filesRepository = getStubFilesRepository(); + val a1 = + loadJsonFixture(getClass(), studyId + ".analysis.EGAZ00001254368.json", Analysis.class); + val fileCentricDocuments = getExpectedFileCentricDocument(studyId); + val fileRepo = Mono.just(getStubFilesRepository()); + val studyAnalysis = Mono.just(a1); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); + val getStudyAnalysesCommand = + GetAnalysisCommand.builder() .studyId(studyId) .analysisId(analysisId) .filesRepositoryBaseUrl(filesRepository.getUrl()) .build(); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); - given(studyDAO.getAnalysis(eq(getStudyAnalysesCommand))).willReturn(studyAnalysis); - given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + given(studyDAO.getAnalysis(eq(getStudyAnalysesCommand))).willReturn(studyAnalysis); + given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - // When - val indexResultMono = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId(studyId) - .analysisId(analysisId) - .repositoryCode(filesRepository.getCode()) - .build() - ).build() - ); - - // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); - - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(studyDAO).should(times(1)).getAnalysis(eq(getStudyAnalysesCommand)); - then(indexServerAdapter).should(times(1)).batchUpsertFileRepositories(eq(batchIndexFilesCommand)); - } + // When + val indexResultMono = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(studyId) + .analysisId(analysisId) + .repositoryCode(filesRepository.getCode()) + .build()) + .build()); - @Test - void shouldRemoveSingleAnalysis() { - // Given - val studyId = "PEME-CA"; - val analysisId = "EGAZ00001254368"; - val filesRepository = getStubFilesRepository(); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.fromSupplier(() -> null); + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); - given(indexServerAdapter.removeAnalysisFiles(eq(analysisId))).willReturn(monoResult); + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(studyDAO).should(times(1)).getAnalysis(eq(getStudyAnalysesCommand)); + then(indexServerAdapter) + .should(times(1)) + .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + } - // When - val indexResultMono = indexer.removeAnalysis(RemoveAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId(studyId) - .analysisId(analysisId) - .repositoryCode(filesRepository.getCode()) - .build() - ).build() - ); + @Test + void shouldIndexSingleAnalysisWithCRAMFiles() { + // Given + val studyId = "CRAM-TEST"; + val repoCode = "TEST-REPO"; + val analysisId = "EGAZ00001254369"; + val filesRepository = getStubFilesRepository(); + val a1 = + loadJsonFixture(getClass(), studyId + ".analysis.EGAZ00001254369.json", Analysis.class); + val fileCentricDocuments = getExpectedFileCentricDocument(studyId); + val fileRepo = Mono.just(getStubFilesRepository()); + val studyAnalysis = Mono.just(a1); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); + val getStudyAnalysesCommand = + GetAnalysisCommand.builder() + .studyId(studyId) + .analysisId(analysisId) + .filesRepositoryBaseUrl(filesRepository.getUrl()) + .build(); + + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + given(studyDAO.getAnalysis(eq(getStudyAnalysesCommand))).willReturn(studyAnalysis); + given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); + // When + val indexResultMono = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(studyId) + .analysisId(analysisId) + .repositoryCode(filesRepository.getCode()) + .build()) + .build()); - then(indexServerAdapter).should(times(1)).removeAnalysisFiles(eq(analysisId)); + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); - } + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(studyDAO).should(times(1)).getAnalysis(eq(getStudyAnalysesCommand)); + then(indexServerAdapter) + .should(times(1)) + .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + } + + @Test + void shouldRemoveSingleAnalysis() { + // Given + val studyId = "PEME-CA"; + val analysisId = "EGAZ00001254368"; + val filesRepository = getStubFilesRepository(); + val result = IndexResult.builder().indexName("file_centric_1.0").successful(true).build(); + val monoResult = Mono.fromSupplier(() -> null); + + given(indexServerAdapter.removeAnalysisFiles(eq(analysisId))).willReturn(monoResult); + + // When + val indexResultMono = + indexer.removeAnalysis( + RemoveAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(studyId) + .analysisId(analysisId) + .repositoryCode(filesRepository.getCode()) + .build()) + .build()); + + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); + + then(indexServerAdapter).should(times(1)).removeAnalysisFiles(eq(analysisId)); + } - @Test - void shouldNotIndexAnalysisIfExcludedByRule() { - // Given - val studyId = "PEME-CA"; - val repoCode = "TEST-REPO"; - val analysisId = "EGAZ00001254368"; - val filesRepository = getStubFilesRepository(); - val a1 = loadJsonFixture(getClass(), studyId +".analysis.EGAZ00001254368.json", Analysis.class); - val fileCentricDocuments = List.of(); - val fileRepoMono = Mono.just(getStubFilesRepository()); - val studyAnalysis = Mono.just(a1); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); - val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - val getStudyAnalysesCommand = GetAnalysisCommand.builder() + @Test + void shouldNotIndexAnalysisIfExcludedByRule() { + // Given + val studyId = "PEME-CA"; + val repoCode = "TEST-REPO"; + val analysisId = "EGAZ00001254368"; + val filesRepository = getStubFilesRepository(); + val a1 = + loadJsonFixture(getClass(), studyId + ".analysis.EGAZ00001254368.json", Analysis.class); + val fileCentricDocuments = List.of(); + val fileRepoMono = Mono.just(getStubFilesRepository()); + val studyAnalysis = Mono.just(a1); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); + val getStudyAnalysesCommand = + GetAnalysisCommand.builder() .studyId(studyId) .analysisId(analysisId) .filesRepositoryBaseUrl(filesRepository.getUrl()) .build(); - val sampleExclusionRule = Mono., List>>just( + val sampleExclusionRule = + Mono., List>>just( Map.of( - Analysis.class, List.of(new IDExclusionRule(Analysis.class, List.of("EGAZ00001254368"))) - ) - ); - - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepoMono); - given(studyDAO.getAnalysis(eq(getStudyAnalysesCommand))).willReturn(studyAnalysis); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); - given(exclusionRulesDAO.getExclusionRules()).willReturn(sampleExclusionRule); - - // When - val indexResultMono = indexer.indexAnalysisToFileCentric(IndexAnalysisCommand.builder() - .analysisIdentifier(AnalysisIdentifier.builder() - .studyId(studyId) - .analysisId(analysisId) - .repositoryCode(filesRepository.getCode()) - .build() - ).build() - ); + Analysis.class, + List.of(new IDExclusionRule(Analysis.class, List.of("EGAZ00001254368"))))); - // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepoMono); + given(studyDAO.getAnalysis(eq(getStudyAnalysesCommand))).willReturn(studyAnalysis); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); + given(exclusionRulesDAO.getExclusionRules()).willReturn(sampleExclusionRule); - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(studyDAO).should(times(1)).getAnalysis(eq(getStudyAnalysesCommand)); - then(indexServerAdapter).should(times(1)) - .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + // When + val indexResultMono = + indexer.indexAnalysisToFileCentric( + IndexAnalysisCommand.builder() + .analysisIdentifier( + AnalysisIdentifier.builder() + .studyId(studyId) + .analysisId(analysisId) + .repositoryCode(filesRepository.getCode()) + .build()) + .build()); - } + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); - @Test - void indexRepositoryshouldNotifyOnStudyFetchError() { - //Given - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val fileRepo = Mono.just(getStubFilesRepository()); - val failure = FailureData.builder() - .failingIds(Map.of("studyId", Set.of("PACA-CA"))).build(); - val failedIndexResult = IndexResult.builder().failureData(failure).successful(false).build(); - val getStudiesCmd = GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); - - given(studyDAO.getStudies(eq(getStudiesCmd))).willReturn(Flux.error(new IndexerException("failed", new RuntimeException(""), failure))); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); - - // When - val indexResultMono = indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode("TEST-REPO") - .build()); - - // Then - StepVerifier.create(indexResultMono) - .expectNext(failedIndexResult) - .expectComplete() - .verify(); - - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(notifier).should(times(1)).notify(any()); - } + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(studyDAO).should(times(1)).getAnalysis(eq(getStudyAnalysesCommand)); + then(indexServerAdapter) + .should(times(1)) + .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + } - @Test - void shouldIndexAllRepositoryStudies() { - //Given - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val studies = getExpectedStudies(); - val fileRepo = Mono.just(getStubFilesRepository()); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); - val getStudiesCmd = GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); - - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - given(studyDAO.getStudies(eq(getStudiesCmd))).willReturn(Flux.fromIterable(studies)); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); - given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); - - for(Study study: studies) { - val studyId = study.getStudyId(); - val command = GetStudyAnalysesCommand.builder() - .filesRepositoryBaseUrl(filesRepository.getUrl()).studyId(studyId) - .build(); - val studyAnalyses = getStudyAnalyses(studyId); - val fileCentricDocuments = getExpectedFileCentricDocument(studyId); - val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - - given(studyDAO.getStudyAnalyses(eq(command))).willReturn(Mono.just(studyAnalyses)); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); - } - - // When - val indexResultMono = indexer.indexRepository(IndexStudyRepositoryCommand.builder() - .repositoryCode("TEST-REPO") - .build()); - - // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); - - then(studyRepositoryDao).should(times(4)).getFilesRepository(repoCode); - then(studyDAO).should(times(3)).getStudyAnalyses(any()); - then(indexServerAdapter).should(times(3)).batchUpsertFileRepositories(any()); - } + @Test + void indexRepositoryshouldNotifyOnStudyFetchError() { + // Given + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val fileRepo = Mono.just(getStubFilesRepository()); + val failure = FailureData.builder().failingIds(Map.of("studyId", Set.of("PACA-CA"))).build(); + val failedIndexResult = IndexResult.builder().failureData(failure).successful(false).build(); + val getStudiesCmd = + GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); + + given(studyDAO.getStudies(eq(getStudiesCmd))) + .willReturn(Flux.error(new IndexerException("failed", new RuntimeException(""), failure))); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + + // When + val indexResultMono = + indexer.indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode("TEST-REPO").build()); + + // Then + StepVerifier.create(indexResultMono).expectNext(failedIndexResult).expectComplete().verify(); + + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(notifier).should(times(1)).notify(any()); + } - @Test - void shouldIndexSingleStudy() { - // Given - val studyId = "PEME-CA"; - val repoCode = "TEST-REPO"; - val filesRepository = getStubFilesRepository(); - val a1 = getStudyAnalyses(studyId); + @Test + void shouldIndexAllRepositoryStudies() { + // Given + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val studies = getExpectedStudies(); + val fileRepo = Mono.just(getStubFilesRepository()); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val getStudiesCmd = + GetAllStudiesCommand.builder().filesRepositoryBaseUrl(filesRepository.getUrl()).build(); + + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); + given(studyDAO.getStudies(eq(getStudiesCmd))).willReturn(Flux.fromIterable(studies)); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); + + for (Study study : studies) { + val studyId = study.getStudyId(); + val command = + GetStudyAnalysesCommand.builder() + .filesRepositoryBaseUrl(filesRepository.getUrl()) + .studyId(studyId) + .build(); + val studyAnalyses = getStudyAnalyses(studyId); val fileCentricDocuments = getExpectedFileCentricDocument(studyId); - val fileRepo = Mono.just(getStubFilesRepository()); - val studyAnalyses = Mono.just(a1); - val result = IndexResult.builder().successful(true).build(); - val monoResult = Mono.just(result); - val batchIndexFilesCommand = BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - val getStudyAnalysesCommand = GetStudyAnalysesCommand.builder() - .studyId(studyId) - .filesRepositoryBaseUrl(filesRepository.getUrl()) - .build(); - - given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); - given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); - given(studyDAO.getStudyAnalyses(eq(getStudyAnalysesCommand))).willReturn(studyAnalyses); - given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))).willReturn(monoResult); - given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); - - // When - val indexResultMono = indexer.indexStudy(IndexStudyCommand.builder() - .studyId(studyId) - .repositoryCode(filesRepository.getCode()) - .build() - ); - - // Then - StepVerifier.create(indexResultMono) - .expectNext(result) - .expectComplete() - .verify(); - - then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); - then(studyDAO).should(times(1)).getStudyAnalyses(eq(getStudyAnalysesCommand)); - then(indexServerAdapter).should(times(1)).batchUpsertFileRepositories(eq(batchIndexFilesCommand)); - } + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); - @SneakyThrows - private List getExpectedFileCentricDocument(String studyId) { - return Arrays.asList(loadJsonFixtureSnakeCase(getClass(), studyId + ".files.json", FileCentricDocument[].class)); + given(studyDAO.getStudyAnalyses(eq(command))).willReturn(Mono.just(studyAnalyses)); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); } - private List getExpectedStudies() { - return Arrays.stream(loadJsonFixture(getClass(), "studies.json", String[].class)) - .map(s -> Study.builder().studyId(s).build()) - .collect(Collectors.toList()); - } + // When + val indexResultMono = + indexer.indexRepository( + IndexStudyRepositoryCommand.builder().repositoryCode("TEST-REPO").build()); - private List getStudyAnalyses(String studyId) { - return Arrays.asList(loadJsonFixture(getClass(), studyId +".analysis.json", Analysis[].class)); - } + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); - private StudyRepository getStubFilesRepository() { - return StudyRepository.builder() - .name("singer") - .url("http://song.sing.sung") - .code("TEST-REPO") - .country("CA") - .dataPath("/p1/p2") - .organization("org") - .storageType(StorageType.S3) - .metadataPath("/m1/m2") + then(studyRepositoryDao).should(times(4)).getFilesRepository(repoCode); + then(studyDAO).should(times(3)).getStudyAnalyses(any()); + then(indexServerAdapter).should(times(3)).batchUpsertFileRepositories(any()); + } + + @Test + void shouldIndexSingleStudy() { + // Given + val studyId = "PEME-CA"; + val repoCode = "TEST-REPO"; + val filesRepository = getStubFilesRepository(); + val a1 = getStudyAnalyses(studyId); + val fileCentricDocuments = getExpectedFileCentricDocument(studyId); + val fileRepo = Mono.just(getStubFilesRepository()); + val studyAnalyses = Mono.just(a1); + val result = IndexResult.builder().successful(true).build(); + val monoResult = Mono.just(result); + val batchIndexFilesCommand = + BatchIndexFilesCommand.builder().files(fileCentricDocuments).build(); + val getStudyAnalysesCommand = + GetStudyAnalysesCommand.builder() + .studyId(studyId) + .filesRepositoryBaseUrl(filesRepository.getUrl()) .build(); - } -} \ No newline at end of file + given(indexServerAdapter.fetchByIds(anyList())).willReturn(Mono.just(List.of())); + given(studyRepositoryDao.getFilesRepository(eq(repoCode))).willReturn(fileRepo); + given(studyDAO.getStudyAnalyses(eq(getStudyAnalysesCommand))).willReturn(studyAnalyses); + given(indexServerAdapter.batchUpsertFileRepositories(eq(batchIndexFilesCommand))) + .willReturn(monoResult); + given(exclusionRulesDAO.getExclusionRules()).willReturn(Mono.just(Map.of())); + + // When + val indexResultMono = + indexer.indexStudy( + IndexStudyCommand.builder() + .studyId(studyId) + .repositoryCode(filesRepository.getCode()) + .build()); + + // Then + StepVerifier.create(indexResultMono).expectNext(result).expectComplete().verify(); + + then(studyRepositoryDao).should(times(1)).getFilesRepository(repoCode); + then(studyDAO).should(times(1)).getStudyAnalyses(eq(getStudyAnalysesCommand)); + then(indexServerAdapter) + .should(times(1)) + .batchUpsertFileRepositories(eq(batchIndexFilesCommand)); + } + + @SneakyThrows + private List getExpectedFileCentricDocument(String studyId) { + return Arrays.asList( + loadJsonFixtureSnakeCase(getClass(), studyId + ".files.json", FileCentricDocument[].class)); + } + + private List getExpectedStudies() { + return Arrays.stream(loadJsonFixture(getClass(), "studies.json", String[].class)) + .map(s -> Study.builder().studyId(s).build()) + .collect(Collectors.toList()); + } + + private List getStudyAnalyses(String studyId) { + return Arrays.asList(loadJsonFixture(getClass(), studyId + ".analysis.json", Analysis[].class)); + } + + private StudyRepository getStubFilesRepository() { + return StudyRepository.builder() + .name("singer") + .url("http://song.sing.sung") + .code("TEST-REPO") + .country("CA") + .organization("org") + .storageType(StorageType.S3) + .build(); + } +} diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DocumentConverterHelperTest.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DocumentConverterHelperTest.java new file mode 100644 index 00000000..a1e18d80 --- /dev/null +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/DocumentConverterHelperTest.java @@ -0,0 +1,74 @@ +package bio.overture.maestro.domain.api; + +import static bio.overture.maestro.domain.api.EntityGenerator.*; +import static bio.overture.masestro.test.Fixture.loadConverterTestFixture; +import static bio.overture.masestro.test.TestCategory.UNIT_TEST; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import bio.overture.maestro.domain.entities.indexing.Donor; +import bio.overture.maestro.domain.entities.metadata.study.Analysis; +import lombok.val; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +@Tag(UNIT_TEST) +public class DocumentConverterHelperTest { + + @Test + void testGetDonor() { + // metadata: + val analysisObj = loadConverterTestFixture("TEST-CA.analysis.json", Analysis.class); + + // expected: + val donor = + Donor.builder() + .donorId("DO1") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor()) + .build(); + + val results = DocumentConverterHelper.getDonors(analysisObj); + + assertEquals(1, results.size()); + assertEquals(donor, results.get(0)); + } + + @Test + void testGetDonors_multi_donor() { + // Expected FileCentricDonor data structure: + // Analysis => d1 -> sp1 -> [sa1] + // d1 -> sp2 -> [sa2] + // d2 -> sp3 -> [sa3, sa4] + // d2 -> sp4 -> [sa5, sa6] + val analysisObj = loadConverterTestFixture("TEST-CA.analysis.multi-donor.json", Analysis.class); + + // expected results: + val donor_1 = + Donor.builder() + .donorId("DO1") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor1()) + .build(); + + val donor_2 = + Donor.builder() + .donorId("DO2") + .gender("Female") + .submitterDonorId("MDT-AP-0749") + .specimens(buildSpecimenListForDonor2()) + .build(); + + val results = DocumentConverterHelper.getDonors(analysisObj); + + assertNotNull(results); + assertEquals(2, results.size()); + assertEquals(donor_2, results.get(0)); + assertEquals(donor_1, results.get(1)); + } +} diff --git a/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java new file mode 100644 index 00000000..7990412c --- /dev/null +++ b/maestro-domain/src/test/java/bio/overture/maestro/domain/api/EntityGenerator.java @@ -0,0 +1,133 @@ +package bio.overture.maestro.domain.api; + +import bio.overture.maestro.domain.entities.indexing.Sample; +import bio.overture.maestro.domain.entities.indexing.Specimen; +import java.util.ArrayList; +import java.util.List; +import lombok.val; + +/** Helper class for AnalysisCentricDocumentConverterTest and FileCentricDocumentConverterTest. */ +public class EntityGenerator { + public static List buildSpecimenListForDonor() { + val specimen = + Specimen.builder() + .specimenId("SP1") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA1") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); + return List.of(specimen); + } + + public static List buildSpecimenListForDonor1() { + val specimen_1 = + Specimen.builder() + .specimenId("SP1") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA1") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); + + val specimen_2 = + Specimen.builder() + .specimenId("SP2") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples( + List.of( + Sample.builder() + .sampleId("SA2") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build())) + .build(); + + val list = new ArrayList(); + list.add(specimen_2); + list.add(specimen_1); + return list; + } + + public static List buildSpecimenListForDonor2() { + val specimen_3 = + Specimen.builder() + .specimenId("SP3") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples(buuldSamplesForDonor2_sp3()) + .build(); + val specimen_4 = + Specimen.builder() + .specimenId("SP4") + .specimenTissueSource("Other") + .submitterSpecimenId("MDT-AP-0749_tumor_specimen") + .tumourNormalDesignation("Tumour") + .specimenType("Primary tumour - solid tissue") + .samples(buildSamplesForDonor2_sp4()) + .build(); + + val list = new ArrayList(); + list.add(specimen_4); + list.add(specimen_3); + return list; + } + + public static List buuldSamplesForDonor2_sp3() { + val sample_3 = + Sample.builder() + .sampleId("SA3") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + val sample_4 = + Sample.builder() + .sampleId("SA4") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + return List.of(sample_3, sample_4); + } + + public static List buildSamplesForDonor2_sp4() { + val sample_5 = + Sample.builder() + .sampleId("SA5") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + val sample_6 = + Sample.builder() + .sampleId("SA6") + .sampleType("DNA") + .submitterSampleId("MDT-AP-0749_tumor") + .matchedNormalSubmitterSampleId("PCSI_0216_St_R") + .build(); + return List.of(sample_5, sample_6); + } +} diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.analysis.EGAZ00001254369.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.analysis.EGAZ00001254369.json new file mode 100644 index 00000000..a1f5c25f --- /dev/null +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.analysis.EGAZ00001254369.json @@ -0,0 +1,77 @@ +{ + "analysisType": { "name": "sequencingRead", "version": 1 }, + "info": { + "dcc_project_code": "CRAM-TEST", + "isPcawg": false + }, + "analysisId": "EGAZ00001254369", + "studyId": "CRAM-TEST", + "analysisState": "PUBLISHED", + "samples": [ + { + "info": {}, + "sampleId": "SA604924", + "specimenId": "SP201301", + "submitterSampleId": "MDT-AP-0749_tumor", + "sampleType": "DNA", + "specimen": { + "info": {}, + "specimenId": "SP201301", + "donorId": "DO232978", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO232978", + "submitterDonorId": "MDT-AP-0749", + "studyId": "CRAM-TEST", + "gender": "Female", + "info": {} + } + } + ], + "files": [ + { + "info": {}, + "objectId": "41ba4fb3-9428-50b5-af6c-d779cd59b04d", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "bundle.EGAZ00001254369.xml", + "fileSize": 6342, + "fileType": "XML", + "fileMd5sum": "fb157ece007dc31b3d34add273efedcb", + "fileAccess": "open", + "dataType": "AlignedReads" + }, + { + "info": {}, + "objectId": "d819f154-2292-56f7-94f3-bbe03fb45bc5", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.cram.crai", + "fileSize": 8939816, + "fileType": "CRAI", + "fileMd5sum": "e2a83668aa4cfe4f4048e75212d44f19", + "fileAccess": "controlled", + "dataType": "AlignedReads" + }, + { + "info": {}, + "objectId": "43d29864-6756-51f0-bb2a-ba7cff860778", + "studyId": "CRAM-TEST", + "analysisId": "EGAZ00001254369", + "fileName": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.cram", + "fileSize": 88906416144, + "fileType": "CRAM", + "fileMd5sum": "29ff9df727803d20834b9997bc17e970", + "fileAccess": "controlled", + "dataType": "AlignedReads" + } + ], + "experiment": { + "analysisId": "EGAZ00001254369", + "aligned": true, + "libraryStrategy": "WGS", + "info": {} + } +} \ No newline at end of file diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.files.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.files.json new file mode 100644 index 00000000..b13b4bc5 --- /dev/null +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/CRAM-TEST.files.json @@ -0,0 +1,67 @@ +[ + { + "object_id": "43d29864-6756-51f0-bb2a-ba7cff860778", + "file_access": "controlled", + "file_type": "CRAM", + "data_type": "AlignedReads", + "study_id": "CRAM-TEST", + "analysis": { + "analysis_id": "EGAZ00001254369", + "analysis_type" : "sequencingRead", + "analysis_version" : 1, + "analysis_state" : "PUBLISHED", + "experiment": { + "analysisId": "EGAZ00001254369", + "aligned": true, + "libraryStrategy": "WGS", + "info": {} + }, + "info": { + "dcc_project_code": "CRAM-TEST", + "isPcawg": false + } + }, + "file": { + "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.cram", + "md5sum": "29ff9df727803d20834b9997bc17e970", + "size": 88906416144, + "data_type": "AlignedReads", + "index_file": { + "object_id": "d819f154-2292-56f7-94f3-bbe03fb45bc5", + "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.cram.crai", + "file_type": "CRAI", + "md5sum": "e2a83668aa4cfe4f4048e75212d44f19", + "data_type": "AlignedReads", + "size": 8939816 + } + }, + "repositories": [ + { + "code": "TEST-REPO", + "organization": "org", + "name": "singer", + "type": "S3", + "country": "CA", + "url": "http://song.sing.sung" + } + ], + "donors": [ + { + "donor_id": "DO232978", + "submitter_donor_id": "MDT-AP-0749", + "gender" : "Female", + "specimens": [{ + "specimen_id": "SP201301", + "specimen_type": "Primary tumour - solid tissue", + "submitter_specimen_id": "MDT-AP-0749_tumor_specimen", + "samples": [{ + "sample_id": "SA604924", + "submitter_sample_id": "MDT-AP-0749_tumor", + "sample_type": "DNA", + "matched_normal_submitter_sample_id": null + }] + }] + } + ] + } +] \ No newline at end of file diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/LIRI-JP.files.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/LIRI-JP.files.json index 86f8b299..646c058c 100644 --- a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/LIRI-JP.files.json +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/LIRI-JP.files.json @@ -3,30 +3,28 @@ "object_id": "b63dff38-e239-5a5d-9c36-ec851049fd12", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "909614f0-ceeb-11e5-8498-af7e4d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "93354882b0fc014b49be1b24df8f435e", "size": 44440835, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "eb2832eb-ad54-574b-bb60-1abf74489ed8", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "ade04a5fd0e9d4052b0ec0b280a9be60", "data_type": "AlignedReads", "size": 1458418 @@ -39,27 +37,26 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", + "specimens": [ + { + "specimen_id": "SP107109", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", + "samples": [{ + "sample_id": "SA515389", "submitter_sample_id": "RK202_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -67,30 +64,28 @@ "object_id": "feecfb7d-2aa9-5434-a0a0-56a6c5ee4a70", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "909614f0-ceeb-11e5-8498-af7e4d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svfix2_4-0-12.20160209.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "ae2e0a8eb02414569fa06cd56a3e809e", "size": 11431, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "e14e73a5-a869-5b9a-a89c-be77fa54135e", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svfix2_4-0-12.20160209.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "9ce7cdedb7450a05c03e419235ed56a6", "data_type": "AlignedReads", "size": 1756 @@ -103,27 +98,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", + "specimens": [{ + "specimen_id": "SP107109", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", + "samples": [{ + "sample_id": "SA515389", "submitter_sample_id": "RK202_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -131,30 +124,28 @@ "object_id": "3106300b-3666-52b7-a3fa-a5096a51814b", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "909614f0-ceeb-11e5-8498-af7e4d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "492989e81699bd5a56b7956e0be7416d", "size": 3648, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "ad4e821d-1908-5e37-8b09-f2bcbe841a6b", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "acdf137ad635f95ee93412fa9153d066", "data_type": "AlignedReads", "size": 5539 @@ -167,27 +158,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", + "specimens": [{ + "specimen_id": "SP107109", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", + "samples": [{ + "sample_id": "SA515389", "submitter_sample_id": "RK202_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -195,30 +184,28 @@ "object_id": "bed33bd4-eec1-5905-8480-5def36d7f6de", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "909614f0-ceeb-11e5-8498-af7e4d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "909614f0-ceeb-11e5-8498-af7e4d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "RK202_B01" } }, - "files": { + "file": { "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "1f3cfd1e1ad462cff9871eb2373935ac", "size": 1418099, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "deab15a3-6820-5bcc-926a-bca2ab5ddaa0", "name": "1fd69adc-c623-11e3-bf01-24c6515278c0.svcp_1-0-6.20150511.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "2233fb90872d759d279395b1a7ad3461", "data_type": "AlignedReads", "size": 236926 @@ -231,27 +218,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/233bc863-114c-5ec4-842b-7f44d189eab0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO48736", + "donor_id": "DO48736", "gender": "Female", "submitter_donor_id": "RK202", - "specimens": { - "id": "SP107109", + "specimens": [{ + "specimen_id": "SP107109", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK202_C01", - "samples": { - "id": "SA515389", + "samples": [{ + "sample_id": "SA515389", "submitter_sample_id": "RK202_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -259,30 +244,28 @@ "object_id": "24329157-e218-5854-8f45-7fa6b9e046c6", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "7d0676c6a85696e1cb2919836b0ab71e", "size": 10536170, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "ad8a45f3-ec8d-5391-ae93-b19dc982698a", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "8269ee64b80e6ec66e590767f697d37b", "data_type": "AlignedReads", "size": 1031078 @@ -295,27 +278,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -323,30 +304,28 @@ "object_id": "4d070189-bd33-5391-b7af-ea32c6782875", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "1d0a8bcf2abf8373c6f8cc63cb748aa8", "size": 59633, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "14e94abe-1fd1-5a85-8d26-a7291edd80ef", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "fdb961feac82a7912bf657450e02c44d", "data_type": "AlignedReads", "size": 11325 @@ -359,27 +338,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -387,30 +364,28 @@ "object_id": "107c83c7-595e-5b36-957b-ae706611d5b9", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "fc246f5e1f670b1f756b8e4294f59504", "size": 51548, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "c5496778-0d72-5cc6-aac4-98777cfbc0c2", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "5aeb4749609bd0a4d81ecffd673a36e1", "data_type": "AlignedReads", "size": 10447 @@ -423,27 +398,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -451,30 +424,28 @@ "object_id": "da524891-37af-5a21-9867-c6812018aa22", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "1f22fff9e578244ba16762a4c82792e8", "size": 1710383, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "d87c4599-bba4-5c23-b7a2-57bbeed7eacb", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.germline.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "ade5449b588c81496e4e7d2b0b49e3d0", "data_type": "AlignedReads", "size": 61514 @@ -487,27 +458,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -515,30 +484,28 @@ "object_id": "46194bd5-b131-55f7-b7b1-99293ba1d6b9", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "e75eef7af227c5c80d0b499ffd39c43b", "size": 31242, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "55283d00-b1a2-521c-99e1-e9d8d88c660d", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-snowman-10.20151220.somatic.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "8cae7bbcd28b6a78ad5c53642ae9695c", "data_type": "AlignedReads", "size": 14606 @@ -551,27 +518,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -579,30 +544,28 @@ "object_id": "ae5b1daf-05d5-5e41-be2a-9f1d3369d517", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger-10.20151220.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "f199c6714f7a6880d30bf7aa56e00625", "size": 10191, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "c4d10f9f-7e9e-5cb3-a30a-0ff8865952ce", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-dRanger-10.20151220.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "139ce5f29fc0e877d5ae5a5d5e03c3f0", "data_type": "AlignedReads", "size": 4212 @@ -615,27 +578,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -643,30 +604,28 @@ "object_id": "8b28796a-1630-5d12-92a7-d8264ee57bc6", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a3c8def-5ef9-4470-b64d-91f62da29abf", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK156_B01" } }, - "files": { + "file": { "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "d641e6e373d21d2fc882f2e4550b7b2c", "size": 382815, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "a99d2a9c-96c4-5788-9149-79a146f0f925", "name": "f064f762-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "797e4ef5c348d4a23e9e83790812d857", "data_type": "AlignedReads", "size": 151104 @@ -679,27 +638,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bd7aa748-d781-523e-99dc-cb7e3469ddc6" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO23549", + "donor_id": "DO23549", "gender": "Female", "submitter_donor_id": "RK156", - "specimens": { - "id": "SP50179", + "specimens": [{ + "specimen_id": "SP50179", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK156_C01", - "samples": { - "id": "SA270462", + "samples": [{ + "sample_id": "SA270462", "submitter_sample_id": "RK156_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -707,30 +664,28 @@ "object_id": "316177db-022d-5121-aba8-ce72a92ee778", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "33b96c263be31852de8d5d15775561c9", "size": 404406, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "97c3147f-dd1d-507f-a73d-8646f7ce2a0b", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "3ed262dbc3b1c427177bdd5af5ef5633", "data_type": "AlignedReads", "size": 27885 @@ -743,27 +698,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -771,30 +724,28 @@ "object_id": "18917df0-2cc6-5ac5-88d8-f48acf24c92e", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "d3227038bd37e74ff88b07e5408c76ed", "size": 1853889, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "9abe4aad-9d35-5d4f-955c-8bceb559827e", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "a7cf0d572f1649c8791cda7892c7be55", "data_type": "AlignedReads", "size": 67312 @@ -807,27 +758,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -835,30 +784,28 @@ "object_id": "5c222a32-a79d-55e0-a490-440217c63393", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "64ef1b8daf5b6eec161555f5cdad3420", "size": 10429379, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "ad68b7cb-e91c-506e-acae-812ae2ea7678", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.germline.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "78b2dd0696a1f2ea8ac8e25a77c4e239", "data_type": "AlignedReads", "size": 1032907 @@ -871,27 +818,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -899,30 +844,28 @@ "object_id": "2af2e398-c27b-5645-b60b-b932a916184d", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "7e7f3e23aa8869bfd82ce16382b7bc7d", "size": 400597, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "bbb31cb3-f175-52c4-ba66-569e1a1d2ed7", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger_snowman-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "130e37274d3c7bebcb41d027eea9148d", "data_type": "AlignedReads", "size": 27877 @@ -935,27 +878,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -963,30 +904,28 @@ "object_id": "d9388ed4-1603-584f-9b18-5ff53b9bffeb", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "ee313d9cbe077063174d92050dde03e0", "size": 25270, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "95e7563e-e8d1-5377-8cfc-1c45a72448da", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-snowman-11.20160302.somatic.indel.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "941a7d48a3250de6edcbbde1cbb4b7e1", "data_type": "AlignedReads", "size": 12044 @@ -999,27 +938,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -1027,30 +964,28 @@ "object_id": "60307755-185e-536e-aa3b-190fb9934844", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger-11.20160302.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "cbafecb01dea96496176bf7497cb9e80", "size": 2540, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "221e532c-a0c8-5573-8399-81f785423e57", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-dRanger-11.20160302.somatic.sv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "de502cd9a25efe6f868f3156e8fd302e", "data_type": "AlignedReads", "size": 1336 @@ -1063,27 +998,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -1091,30 +1024,28 @@ "object_id": "2664e052-ba85-5be4-89e0-4f2f06a998c0", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "LIRI-JP", "analysis": { "analysis_id": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "LIRI-JP", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "cbc64c23-1d6f-40db-a49e-dc54ef222ae9", "variantCallingTool": "Broad variant call pipeline", "matchedNormalSampleSubmitterId": "RK091_B01" } }, - "files": { + "file": { "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "ca10e3bd248498c9427b2b5e324f0242", "size": 422502, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "87d9f1f2-3d13-5dd2-8ed5-d175952e2586", "name": "aabddb20-c622-11e3-bf01-24c6515278c0.broad-mutect-v3.20160222.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "9029f92d2d1ea6d76b3a88795756aacc", "data_type": "AlignedReads", "size": 160972 @@ -1127,27 +1058,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/8d8a4db8-071d-5385-9266-06e93ca029d0" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO45211", + "donor_id": "DO45211", "gender": "Female", "submitter_donor_id": "RK091", - "specimens": { - "id": "SP99145", + "specimens": [{ + "specimen_id": "SP99145", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "RK091_C01", - "samples": { - "id": "SA501653", + "samples": [{ + "sample_id": "SA501653", "submitter_sample_id": "RK091_C01", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] } diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/MALY-DE.files.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/MALY-DE.files.json index 8f618c98..466a1a6d 100644 --- a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/MALY-DE.files.json +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/MALY-DE.files.json @@ -3,30 +3,28 @@ "object_id": "316612a4-4c23-5434-976a-810f8cde06b7", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "MALY-DE", "analysis": { "analysis_id": "0a1df2a2-029d-48cc-839c-0a7c89ff972f", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "MALY-DE", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "0a1df2a2-029d-48cc-839c-0a7c89ff972f", "variantCallingTool": "MUSE variant call pipeline", "matchedNormalSampleSubmitterId": "control_4128852" } }, - "files": { + "file": { "name": "e2fa7251-507e-4d76-95a3-a228adc3885a.MUSE_1-0rc-vcf.20151107.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "f83b0f9099e8594b589440819ab119cd", "size": 273787, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "a9c47630-491b-5587-a58d-6736734d6163", "name": "e2fa7251-507e-4d76-95a3-a228adc3885a.MUSE_1-0rc-vcf.20151107.somatic.snv_mnv.vcf.gz.idx", - "format": "IDX", + "file_type": "IDX", "md5sum": "f5629a05fe5b8dd6a63381de259aa935", "data_type": "AlignedReads", "size": 146924 @@ -39,27 +37,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/95cbd8c1-1cb1-5c18-8c2d-6f54867d7b72" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO221129", + "donor_id": "DO221129", "gender" : "Female", "submitter_donor_id": "4128852", - "specimens": { - "id": "SP124981", + "specimens": [{ + "specimen_id": "SP124981", "specimen_type": "Primary tumour - lymph node", "submitter_specimen_id": "tumor_4128852", - "samples": { - "id": "SA557689", + "samples": [{ + "sample_id": "SA557689", "submitter_sample_id": "tumor_4128852", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -67,30 +63,28 @@ "object_id": "0bafbb56-728e-5e6c-bb87-88e7bb789dab", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "MALY-DE", "analysis": { "analysis_id": "64e6f498-20ae-4c59-a5f5-36b18f81bd99", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "MALY-DE", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "64e6f498-20ae-4c59-a5f5-36b18f81bd99", "variantCallingTool": " PCAWG SNV-MNV callers", "matchedNormalSampleSubmitterId": "control_4113191" } }, - "files": { + "file": { "name": "4e7e6e1f-c648-446f-bdf6-0b1fcc6dfa83.consensus.20160830.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "02b4602861de0a5529e42d3e66eaa2a6", "size": 188550, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "21126ff1-cd5e-5bb2-a55e-3c9f68b2ad14", "name": "4e7e6e1f-c648-446f-bdf6-0b1fcc6dfa83.consensus.20160830.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "c00c6672cb752c71dd2d1ef23ea559fa", "data_type": "AlignedReads", "size": 101608 @@ -103,27 +97,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/5d360754-ee81-581f-9b51-dfdd24f7a8d2" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO52693", + "donor_id": "DO52693", "gender" : "Female", "submitter_donor_id": "4113191", - "specimens": { - "id": "SP116718", + "specimens": [{ + "specimen_id": "SP116718", "specimen_type": "Primary tumour - lymph node", "submitter_specimen_id": "tumor_4113191", - "samples": { - "id": "SA542152", + "samples": [{ + "sample_id": "SA542152", "submitter_sample_id": "tumor_4113191", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -131,30 +123,28 @@ "object_id": "7b32b455-d8e2-5e51-bcad-dad7945c8b15", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "MALY-DE", "analysis": { "analysis_id": "327c74cd-ee0b-4885-bafd-741e17b8b163", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "MALY-DE", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "327c74cd-ee0b-4885-bafd-741e17b8b163", "variantCallingTool": "PCAWG InDel callers", "matchedNormalSampleSubmitterId": "control_4166503" } }, - "files": { + "file": { "name": "866ecfe7-caa6-4565-9418-6b9d6c8a3b43.consensus.20161006.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "020a444fb10ec02b40e141aa9cd0de87", "size": 38671, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "4ba753ee-bbb7-5f57-9cda-6e15c519f834", "name": "866ecfe7-caa6-4565-9418-6b9d6c8a3b43.consensus.20161006.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "7c606af5d37d3788850da1a5f1705feb", "data_type": "AlignedReads", "size": 19306 @@ -167,27 +157,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/46c2fec8-fd4a-5b62-8819-647bf23ac15c" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO52662", + "donor_id": "DO52662", "gender" : "Female", "submitter_donor_id": "4166503", - "specimens": { - "id": "SP116638", + "specimens": [{ + "specimen_id": "SP116638", "specimen_type": "Primary tumour - lymph node", "submitter_specimen_id": "tumor_4166503", - "samples": { - "id": "SA542043", + "samples": [{ + "sample_id": "SA542043", "submitter_sample_id": "tumor_4166503", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] } diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.excluded.SA520221.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.excluded.SA520221.json index 98f91b2c..ac551cbf 100644 --- a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.excluded.SA520221.json +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.excluded.SA520221.json @@ -3,30 +3,28 @@ "object_id": "e86158ec-0eb6-50e2-bbb5-7bfed4745db0", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "14ded2a7342f9124421ace5aa920510d", "size": 543337, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "bb949d15-334f-5712-9f04-41225a7beb8d", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "99f0b87f5113494e2fc2284c91dc15e0", "data_type": "AlignedReads", "size": 224729 @@ -39,27 +37,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -67,30 +63,28 @@ "object_id": "0474e5f1-3b49-5ec5-8186-a1ffc6952f37", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "0e1e3c924b37d90c99bb04f719a56ce4", "size": 5565355, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "2910e07c-268b-57a5-877a-0cec35298319", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1abed463bc92e0d8486699b37af73737", "data_type": "AlignedReads", "size": 546153 @@ -103,27 +97,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -131,30 +123,28 @@ "object_id": "54abbcb5-485f-5acd-88e9-50261e0c8e17", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "19b56e546bdcdb7f68dfd603b1b7a8d7", "size": 101384873, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "425898fd-35d3-5ee2-ac00-50dfd724f4c6", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "a6113c54193fd7539dffd06f50f04779", "data_type": "AlignedReads", "size": 1541330 @@ -167,27 +157,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -195,30 +183,28 @@ "object_id": "0d7ee358-01fe-59ce-b62b-0663bec8596d", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "c102abc0bfa0b63b2e1ea2ae62e18b80", "size": 72850649, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "04c24136-c19d-5042-a3cd-36f63c7b4d6b", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "156880f52b94b3119afa3ac6dd7b8ced", "data_type": "AlignedReads", "size": 1502241 @@ -231,27 +217,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -259,30 +243,28 @@ "object_id": "432779b5-95db-5caf-a529-85a36a707c04", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "93b219c94687e12b5fda564399fa3319", "size": 23763, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "51306186-092d-5e3f-bb3c-028873052eb2", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "68d76af7c7b9b7c00fa0438101739798", "data_type": "AlignedReads", "size": 10691 @@ -295,27 +277,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -323,30 +303,28 @@ "object_id": "4f2bee69-e66e-5af5-856a-a0f3c34b12ff", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "6ebd4aa8a821da40955d7fcb879be82a", "size": 18226, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "e5d387a1-3b51-5526-ae22-270df2e1fd2f", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "96b05a083d371ae8317651ceffb0f48b", "data_type": "AlignedReads", "size": 6134 @@ -359,27 +337,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -387,30 +363,28 @@ "object_id": "3f571505-3bea-5065-956a-4b5a793ad1ed", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "f3a99bd737b3b416aa34e131b939b2a0", "size": 161511, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "e8569e7c-6ae4-5aeb-ae50-b94366aae5df", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "cb856f2d5837c5f065c7c598dcd795b5", "data_type": "AlignedReads", "size": 19769 @@ -423,27 +397,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -451,30 +423,28 @@ "object_id": "e122b153-c98b-530d-879a-11871a427d1b", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "4824d68baf7c2b06f5a652bb5462b9b4", "size": 58091383, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "b26a535c-3531-55ad-8725-17aa6afb22c2", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "894fa70742c16ed7f845abd68609f20c", "data_type": "AlignedReads", "size": 1524609 @@ -487,27 +457,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -515,30 +483,28 @@ "object_id": "36ede5a5-fcde-5b6a-95c1-af28321fbd39", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "0502184e75c3a3fb83e2aa6ab536e78a", "size": 59407, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "b9800f14-13d5-55cf-9383-a65b5446116a", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1ba234eb0c457dba2942215d3fc4741a", "data_type": "AlignedReads", "size": 2961 @@ -551,27 +517,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -579,30 +543,28 @@ "object_id": "5c1fcacb-7c66-53a6-8fb6-93d1281e55b8", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "bafebb1c613c348c696dc8f940ce1039", "size": 2964, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "52db8e51-5b3e-56ce-b939-0a96b4dfbce3", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "725becdd11e63c371e57a987420846bd", "data_type": "AlignedReads", "size": 4783 @@ -615,27 +577,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -643,31 +603,28 @@ "object_id": "e704d3f5-1274-5cdd-924f-609159e121cc", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "study_id": "PACA-CA", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "37ee70b282788eb0660f7d965f355898", "size": 1704966, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "6b32cfec-e82c-5528-b908-8e00ae927578", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "91dece67c383a67123caae382a130cb9", "data_type": "AlignedReads", "size": 303747 @@ -680,27 +637,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] } diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.json index 8dbe9df1..f76af147 100644 --- a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.json +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PACA-CA.files.json @@ -3,30 +3,28 @@ "object_id": "e86158ec-0eb6-50e2-bbb5-7bfed4745db0", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "14ded2a7342f9124421ace5aa920510d", "size": 543337, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "bb949d15-334f-5712-9f04-41225a7beb8d", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "99f0b87f5113494e2fc2284c91dc15e0", "data_type": "AlignedReads", "size": 224729 @@ -39,27 +37,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -67,30 +63,28 @@ "object_id": "0474e5f1-3b49-5ec5-8186-a1ffc6952f37", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "0e1e3c924b37d90c99bb04f719a56ce4", "size": 5565355, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "2910e07c-268b-57a5-877a-0cec35298319", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1abed463bc92e0d8486699b37af73737", "data_type": "AlignedReads", "size": 546153 @@ -103,27 +97,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -131,30 +123,28 @@ "object_id": "54abbcb5-485f-5acd-88e9-50261e0c8e17", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "19b56e546bdcdb7f68dfd603b1b7a8d7", "size": 101384873, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "425898fd-35d3-5ee2-ac00-50dfd724f4c6", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-snvCalling_1-0-132-1.20150722.germline.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "a6113c54193fd7539dffd06f50f04779", "data_type": "AlignedReads", "size": 1541330 @@ -167,27 +157,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -195,30 +183,28 @@ "object_id": "0d7ee358-01fe-59ce-b62b-0663bec8596d", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz", - "format": "VCF", "md5sum": "c102abc0bfa0b63b2e1ea2ae62e18b80", "size": 72850649, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "04c24136-c19d-5042-a3cd-36f63c7b4d6b", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-indelCalling_1-0-132-1.20150722.germline.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "156880f52b94b3119afa3ac6dd7b8ced", "data_type": "AlignedReads", "size": 1502241 @@ -231,27 +217,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -259,30 +243,28 @@ "object_id": "432779b5-95db-5caf-a529-85a36a707c04", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "93b219c94687e12b5fda564399fa3319", "size": 23763, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "51306186-092d-5e3f-bb3c-028873052eb2", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.dkfz-copyNumberEstimation_1-0-189-hpc-fix.1508271702.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "68d76af7c7b9b7c00fa0438101739798", "data_type": "AlignedReads", "size": 10691 @@ -295,27 +277,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -323,30 +303,28 @@ "object_id": "4f2bee69-e66e-5af5-856a-a0f3c34b12ff", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "6ebd4aa8a821da40955d7fcb879be82a", "size": 18226, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "e5d387a1-3b51-5526-ae22-270df2e1fd2f", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "96b05a083d371ae8317651ceffb0f48b", "data_type": "AlignedReads", "size": 6134 @@ -359,27 +337,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -387,30 +363,28 @@ "object_id": "3f571505-3bea-5065-956a-4b5a793ad1ed", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "744c62bf-4747-4ab2-aca9-50879cea82e0", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "744c62bf-4747-4ab2-aca9-50879cea82e0", "variantCallingTool": "DKFZ/EMBL variant call pipeline", "matchedNormalSampleSubmitterId": "PCSI_0233_Ly_R" } }, - "files": { + "file": { "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz", - "format": "VCF", "md5sum": "f3a99bd737b3b416aa34e131b939b2a0", "size": 161511, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "e8569e7c-6ae4-5aeb-ae50-b94366aae5df", "name": "bcef0b6c-6584-4090-9d28-ef784a7e5fbb.embl-delly_1-0-0-preFilter.20150722.germline.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "cb856f2d5837c5f065c7c598dcd795b5", "data_type": "AlignedReads", "size": 19769 @@ -423,27 +397,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/bbbbc35b-9ea3-5191-b6c8-9092975d8033" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51523", + "donor_id": "DO51523", "gender": "Female", "submitter_donor_id": "PCSI_0233", - "specimens": { - "id": "SP125730", + "specimens": [{ + "specimen_id": "SP125730", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0233_Pa_P_526", - "samples": { - "id": "SA533729", + "samples": [{ + "sample_id": "SA533729", "submitter_sample_id": "PCSI_0233_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -451,30 +423,28 @@ "object_id": "e122b153-c98b-530d-879a-11871a427d1b", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "4824d68baf7c2b06f5a652bb5462b9b4", "size": 58091383, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "b26a535c-3531-55ad-8725-17aa6afb22c2", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "894fa70742c16ed7f845abd68609f20c", "data_type": "AlignedReads", "size": 1524609 @@ -487,27 +457,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -515,30 +483,28 @@ "object_id": "36ede5a5-fcde-5b6a-95c1-af28321fbd39", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz", - "format": "VCF", "md5sum": "0502184e75c3a3fb83e2aa6ab536e78a", "size": 59407, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "b9800f14-13d5-55cf-9383-a65b5446116a", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svfix2_4-0-12.20160208.somatic.sv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "1ba234eb0c457dba2942215d3fc4741a", "data_type": "AlignedReads", "size": 2961 @@ -551,27 +517,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -579,30 +543,28 @@ "object_id": "5c1fcacb-7c66-53a6-8fb6-93d1281e55b8", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz", - "format": "VCF", "md5sum": "bafebb1c613c348c696dc8f940ce1039", "size": 2964, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "52db8e51-5b3e-56ce-b939-0a96b4dfbce3", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.cnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "725becdd11e63c371e57a987420846bd", "data_type": "AlignedReads", "size": 4783 @@ -615,27 +577,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -643,30 +603,28 @@ "object_id": "e704d3f5-1274-5cdd-924f-609159e121cc", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "916b5a92-cea6-11e5-af30-d4714d100685", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "916b5a92-cea6-11e5-af30-d4714d100685", "variantCallingTool": "Sanger variant call pipeline", "matchedNormalSampleSubmitterId": "ASHPC_0022_Pa_R" } }, - "files": { + "file": { "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz", - "format": "VCF", "md5sum": "37ee70b282788eb0660f7d965f355898", "size": 1704966, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "6b32cfec-e82c-5528-b908-8e00ae927578", "name": "94652d14-2e4d-4f4a-a4f7-8df77df788c0.svcp_1-0-8.20150911.somatic.snv_mnv.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "91dece67c383a67123caae382a130cb9", "data_type": "AlignedReads", "size": 303747 @@ -679,27 +637,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/47fc8458-df52-5ad1-a0fc-cd2c277ecb93" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO51498", + "donor_id": "DO51498", "gender": "Female", "submitter_donor_id": "PCSI_0450", - "specimens": { - "id": "SP125791", + "specimens": [{ + "specimen_id": "SP125791", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "ASHPC_0022_Pa_P", - "samples": { - "id": "SA533758", + "samples": [{ + "sample_id": "SA533758", "submitter_sample_id": "ASHPC_0022_Pa_P", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] }, @@ -707,30 +663,28 @@ "object_id": "9727c61d-3871-54f1-a65a-2d1d91acd242", "file_access": "controlled", "file_type": "VCF", + "data_type": "AlignedReads", "study_id": "PACA-CA", "analysis": { "analysis_id": "b8e09611-225a-4de6-87dc-104753072252", "analysis_type": "variantCall", "analysis_version": "1", - "state": "PUBLISHED", - "study_id": "PACA-CA", + "analysis_state": "PUBLISHED", "experiment": { "analysisId": "b8e09611-225a-4de6-87dc-104753072252", "variantCallingTool": "PCAWG InDel callers", "matchedNormalSampleSubmitterId": "PCSI_0083_Du_R" } }, - "files": { + "file": { "name": "536dedba-46c4-4a21-b112-13c030b13069.consensus.20161006.somatic.indel.vcf.gz", - "format": "VCF", "md5sum": "01e54034c70b85812e9aac58f2525594", "size": 329863, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "08519ab5-8665-58dc-8476-1c40cae5b789", "name": "536dedba-46c4-4a21-b112-13c030b13069.consensus.20161006.somatic.indel.vcf.gz.tbi", - "format": "TBI", + "file_type": "TBI", "md5sum": "5b83369174136b88f16a08ab87bb67f7", "data_type": "AlignedReads", "size": 148925 @@ -743,27 +697,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/ca8479dc-8f00-5757-b433-c3cf5b76f255" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO35442", + "donor_id": "DO35442", "gender": "Female", "submitter_donor_id": "PCSI_0083", - "specimens": { - "id": "SP125732", + "specimens": [{ + "specimen_id": "SP125732", "specimen_type": "Primary tumour - other", "submitter_specimen_id": "PCSI_0083_Pa_P_526", - "samples": { - "id": "SA520221", + "samples": [{ + "sample_id": "SA520221", "submitter_sample_id": "PCSI_0083_Pa_P_526", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] } diff --git a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PEME-CA.files.json b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PEME-CA.files.json index 1eeed275..ee8e1dc4 100644 --- a/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PEME-CA.files.json +++ b/maestro-domain/src/test/resources/fixtures/DefaultIndexerTest/PEME-CA.files.json @@ -3,13 +3,13 @@ "object_id": "43d29864-6756-51f0-bb2a-ba7cff860778", "file_access": "controlled", "file_type": "BAM", + "data_type": "AlignedReads", "study_id": "PEME-CA", "analysis": { "analysis_id": "EGAZ00001254368", "analysis_type" : "sequencingRead", "analysis_version" : 1, - "state" : "PUBLISHED", - "study_id": "PEME-CA", + "analysis_state" : "PUBLISHED", "experiment": { "analysisId": "EGAZ00001254368", "aligned": true, @@ -21,17 +21,15 @@ "isPcawg": false } }, - "files": { + "file": { "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam", - "format": "BAM", "md5sum": "29ff9df727803d20834b9997bc17e970", "size": 88906416144, - "last_modified": null, "data_type": "AlignedReads", "index_file": { "object_id": "d819f154-2292-56f7-94f3-bbe03fb45bc5", "name": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam.bai", - "format": "BAI", + "file_type": "BAI", "md5sum": "e2a83668aa4cfe4f4048e75212d44f19", "data_type": "AlignedReads", "size": 8939816 @@ -44,27 +42,25 @@ "name": "singer", "type": "S3", "country": "CA", - "url": "http://song.sing.sung", - "data_path": "/p1/p2", - "metadata_path": "/m1/m2/41ba4fb3-9428-50b5-af6c-d779cd59b04d" + "url": "http://song.sing.sung" } ], "donors": [ { - "id": "DO232978", + "donor_id": "DO232978", "submitter_donor_id": "MDT-AP-0749", "gender" : "Female", - "specimens": { - "id": "SP201301", + "specimens": [{ + "specimen_id": "SP201301", "specimen_type": "Primary tumour - solid tissue", "submitter_specimen_id": "MDT-AP-0749_tumor_specimen", - "samples": { - "id": "SA604924", + "samples": [{ + "sample_id": "SA604924", "submitter_sample_id": "MDT-AP-0749_tumor", "sample_type": "DNA", "matched_normal_submitter_sample_id": null - } - } + }] + }] } ] } diff --git a/maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.json b/maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.json new file mode 100644 index 00000000..3dbbfe80 --- /dev/null +++ b/maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.json @@ -0,0 +1,80 @@ +{ + "analysisType": { "name": "sequencingRead", "version": 1 }, + "info": { + "dcc_project_code": "TEST-CA", + "isPcawg": false + }, + "analysisId": "EGAZ00001254368", + "studyId": "TEST-CA", + "analysisState": "PUBLISHED", + "samples": [ + { + "info": {}, + "sampleId": "SA1", + "specimenId": "SP1", + "submitterSampleId": "MDT-AP-0749_tumor", + "matchedNormalSubmitterSampleId": "PCSI_0216_St_R", + "sampleType": "DNA", + "specimen": { + "info": {}, + "specimenId": "SP1", + "donorId": "DO1", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "tumourNormalDesignation": "Tumour", + "specimenTissueSource": "Other", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO1", + "submitterDonorId": "MDT-AP-0749", + "studyId": "TEST-CA", + "gender": "Female", + "info": {} + } + } + ], + "files": [ + { + "info": {}, + "objectId": "41ba4fb3-9428-50b5-af6c-d779cd59b04d", + "studyId": "TEST-CA", + "analysisId": "EGAZ00001254368", + "fileName": "bundle.EGAZ00001254368.xml", + "fileSize": 6342, + "fileType": "XML", + "fileMd5sum": "fb157ece007dc31b3d34add273efedcb", + "fileAccess": "open", + "dataType": "AlignedReads" + }, + { + "info": {}, + "objectId": "d819f154-2292-56f7-94f3-bbe03fb45bc5", + "studyId": "TEST-CA", + "analysisId": "EGAZ00001254368", + "fileName": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam.bai", + "fileSize": 8939816, + "fileType": "BAI", + "fileMd5sum": "e2a83668aa4cfe4f4048e75212d44f19", + "fileAccess": "controlled", + "dataType": "AlignedReads" + }, + { + "info": {}, + "objectId": "43d29864-6756-51f0-bb2a-ba7cff860778", + "studyId": "TEST-CA", + "analysisId": "EGAZ00001254368", + "fileName": "29ff9df727803d20834b9997bc17e970.tumor_MDT-AP-0749_merged.mdup.bam", + "fileSize": 88906416144, + "fileType": "BAM", + "fileMd5sum": "29ff9df727803d20834b9997bc17e970", + "fileAccess": "controlled", + "dataType": "AlignedReads" + } + ], + "experiment": { + "analysisId": "EGAZ00001254368", + "aligned": true, + "libraryStrategy": "WGS", + "info": {} + } +} diff --git a/maestro-domain/src/test/resources/fixtures/AnalysisCentricDocumentConverterTest/TEST-CA.analysis.multi-donor.json b/maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.multi-donor.json similarity index 62% rename from maestro-domain/src/test/resources/fixtures/AnalysisCentricDocumentConverterTest/TEST-CA.analysis.multi-donor.json rename to maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.multi-donor.json index 9ccf3b0d..5fedceb6 100644 --- a/maestro-domain/src/test/resources/fixtures/AnalysisCentricDocumentConverterTest/TEST-CA.analysis.multi-donor.json +++ b/maestro-domain/src/test/resources/fixtures/DocumentConverterTest/TEST-CA.analysis.multi-donor.json @@ -28,7 +28,7 @@ "donorId": "DO1", "submitterDonorId": "MDT-AP-0749", "studyId": "TEST-CA", - "gender": "female", + "gender": "Female", "info": {} } }, @@ -52,7 +52,7 @@ "donorId": "DO1", "submitterDonorId": "MDT-AP-0749", "studyId": "TEST-CA", - "gender": "female", + "gender": "Female", "info": {} } }, @@ -76,7 +76,79 @@ "donorId": "DO2", "submitterDonorId": "MDT-AP-0749", "studyId": "TEST-CA", - "gender": "female", + "gender": "Female", + "info": {} + } + }, + { + "info": {}, + "sampleId": "SA4", + "specimenId": "SP3", + "submitterSampleId": "MDT-AP-0749_tumor", + "matchedNormalSubmitterSampleId": "PCSI_0216_St_R", + "sampleType": "DNA", + "specimen": { + "info": {}, + "specimenId": "SP3", + "donorId": "DO2", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "tumourNormalDesignation": "Tumour", + "specimenTissueSource": "Other", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO2", + "submitterDonorId": "MDT-AP-0749", + "studyId": "TEST-CA", + "gender": "Female", + "info": {} + } + }, + { + "info": {}, + "sampleId": "SA5", + "specimenId": "SP4", + "submitterSampleId": "MDT-AP-0749_tumor", + "matchedNormalSubmitterSampleId": "PCSI_0216_St_R", + "sampleType": "DNA", + "specimen": { + "info": {}, + "specimenId": "SP4", + "donorId": "DO2", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "tumourNormalDesignation": "Tumour", + "specimenTissueSource": "Other", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO2", + "submitterDonorId": "MDT-AP-0749", + "studyId": "TEST-CA", + "gender": "Female", + "info": {} + } + }, + { + "info": {}, + "sampleId": "SA6", + "specimenId": "SP4", + "submitterSampleId": "MDT-AP-0749_tumor", + "matchedNormalSubmitterSampleId": "PCSI_0216_St_R", + "sampleType": "DNA", + "specimen": { + "info": {}, + "specimenId": "SP4", + "donorId": "DO2", + "submitterSpecimenId": "MDT-AP-0749_tumor_specimen", + "tumourNormalDesignation": "Tumour", + "specimenTissueSource": "Other", + "specimenType": "Primary tumour - solid tissue" + }, + "donor": { + "donorId": "DO2", + "submitterDonorId": "MDT-AP-0749", + "studyId": "TEST-CA", + "gender": "Female", "info": {} } } diff --git a/pom.xml b/pom.xml index c8a33f84..8fb3e11b 100644 --- a/pom.xml +++ b/pom.xml @@ -295,7 +295,8 @@ Hoxton.SR1 1.18.6 1.10.6 - 5.6.0 + 5.5.2 + 1.5.2 1.7.16 1.2.3 3.3.3.RELEASE @@ -304,7 +305,6 @@ 2.23.4 2.10.2 2.21.0 - 1.6.0 2.1.3 2.10.2 1.19 @@ -371,6 +371,18 @@ + + com.coveo + fmt-maven-plugin + 2.9 + + + + format + + + +