From 47271f071ae785189e4d960a0cc56d35bf027506 Mon Sep 17 00:00:00 2001 From: wivern Date: Fri, 22 Dec 2023 17:08:12 +0300 Subject: [PATCH 01/32] ATL-29 API Support for R Shiny Apps --- pom.xml | 79 +++++++++++ src/main/assembly/shiny-cohortCounts.xml | 18 +++ .../ohdsi/webapi/service/ShinyService.java | 128 ++++++++++++++++++ .../ohdsi/webapi/shiny/ApplicationBrief.java | 31 +++++ .../CohortCountsShinyPackagingService.java | 104 ++++++++++++++ .../webapi/shiny/PositConnectClient.java | 127 +++++++++++++++++ .../shiny/PositConnectClientException.java | 11 ++ .../webapi/shiny/ShinyPackagingService.java | 11 ++ .../webapi/shiny/ShinyPublishedEntity.java | 86 ++++++++++++ .../shiny/ShinyPublishedRepository.java | 11 ++ .../org/ohdsi/webapi/shiny/TemporaryFile.java | 22 +++ .../org/ohdsi/webapi/util/TempFileUtils.java | 18 +++ .../resources/application-shiny.properties | 5 + src/main/resources/application.properties | 3 + src/main/resources/i18n/messages_en.json | 9 ++ ...0.20231220173146__add_shiny_permission.sql | 16 +++ .../V20231222125707__add_shiny_published.sql | 14 ++ 17 files changed, 693 insertions(+) create mode 100644 src/main/assembly/shiny-cohortCounts.xml create mode 100644 src/main/java/org/ohdsi/webapi/service/ShinyService.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java create mode 100644 src/main/resources/application-shiny.properties create mode 100644 src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql create mode 100644 src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql diff --git a/pom.xml b/pom.xml index 962fac6b09..a22226a04a 100644 --- a/pom.xml +++ b/pom.xml @@ -297,6 +297,9 @@ false /tmp/atlas/audit/audit.log /tmp/atlas/audit/audit-extra.log + + + false WebAPI @@ -1863,5 +1866,81 @@ + + webapi-shiny + + true + http://localhost/Atlas + src/main/resources/shiny + + + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + + + + org.apache.maven.plugins + maven-clean-plugin + 3.3.2 + + + + ${shiny.directory} + + shiny-cohortCounts.zip + + + + + + + org.apache.maven.plugins + maven-scm-plugin + 2.0.1 + + scm:git:git@github.com:OdyOSG/AtlasShinyExport.git + main + branch + + + + checkout-rshiny + generate-resources + + checkout + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + build-cohortCounts-archive + + single + + generate-resources + + false + ${project.build.directory}/checkout + ${shiny.directory} + + src/main/assembly/shiny-cohortCounts.xml + + shiny-cohortCounts + + + + + + + diff --git a/src/main/assembly/shiny-cohortCounts.xml b/src/main/assembly/shiny-cohortCounts.xml new file mode 100644 index 0000000000..5c43b81bec --- /dev/null +++ b/src/main/assembly/shiny-cohortCounts.xml @@ -0,0 +1,18 @@ + + + shiny-cohortCounts + + zip + + false + + + ./apps/cohortCounts + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java new file mode 100644 index 0000000000..14750b3197 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -0,0 +1,128 @@ +package org.ohdsi.webapi.service; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.glassfish.jersey.media.multipart.ContentDisposition; +import org.ohdsi.webapi.shiny.ApplicationBrief; +import org.ohdsi.webapi.shiny.PositConnectClient; +import org.ohdsi.webapi.shiny.ShinyPackagingService; +import org.ohdsi.webapi.shiny.ShinyPublishedEntity; +import org.ohdsi.webapi.shiny.ShinyPublishedRepository; +import org.ohdsi.webapi.shiny.TemporaryFile; +import org.ohdsi.webapi.shiro.PermissionManager; +import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; +import org.ohdsi.webapi.shiro.annotations.SourceKey; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.NotFoundException; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Date; +import java.text.MessageFormat; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +@ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") +@Path("/shiny") +public class ShinyService { + private final Map servicesMap; + @Autowired + private ShinyPublishedRepository shinyPublishedRepository; + @Autowired + private PermissionManager permissionManager; + @Autowired + private PositConnectClient connectClient; + + public ShinyService(List services) { + servicesMap = services.stream().collect(Collectors.toMap(ShinyPackagingService::getType, Function.identity())); + } + + @GET + @Path("/download/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + public Response downloadShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) throws IOException { + TemporaryFile data = packageShinyApp(type, id, sourceKey); + ContentDisposition contentDisposition = ContentDisposition.type("attachment") + .fileName(data.getFilename()) + .build(); + return Response + .ok(Files.newInputStream(data.getFile())) + .header(HttpHeaders.CONTENT_TYPE, "application/zip") + .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition) + .build(); + } + + @GET + @Path("/publish/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + @Transactional + public Response publishShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) { + publishApp(type, id, sourceKey); + return Response.ok().build(); + } + + private void publishApp(String type, int id, String sourceKey) { + TemporaryFile data = packageShinyApp(type, id, sourceKey); + ShinyPublishedEntity publication = getPublication(id, sourceKey); + ShinyPackagingService service = findShinyService(CommonAnalysisType.valueOf(type.toUpperCase())); + UUID contentId = Optional.ofNullable(publication.getContentId()) + .orElseGet(() -> connectClient.createContentItem(service.getBrief(id, sourceKey))); + TemporaryFile bundle = prepareBundle(data); + Integer bundleId = connectClient.uploadBundle(contentId, bundle); + connectClient.deployBundle(contentId, bundleId); + } + + private TemporaryFile prepareBundle(TemporaryFile data) { + return null; + } + + private ShinyPublishedEntity getPublication(int id, String sourceKey) { + return shinyPublishedRepository.findByAnalysisIdAAndSourceKey(Integer.toUnsignedLong(id), sourceKey).orElseGet(() -> { + ShinyPublishedEntity entity = new ShinyPublishedEntity(); + entity.setAnalysisId(Integer.toUnsignedLong(id)); + entity.setSourceKey(sourceKey); + entity.setCreatedBy(permissionManager.getCurrentUser()); + entity.setCreatedDate(Date.from(Instant.now())); + return entity; + }); + } + + private TemporaryFile packageShinyApp(String type, int id, String sourceKey) { + CommonAnalysisType analysisType = CommonAnalysisType.valueOf(type.toUpperCase()); + ShinyPackagingService service = findShinyService(analysisType); + return service.packageApp(id, sourceKey); + } + + private ShinyPackagingService findShinyService(CommonAnalysisType type) { + return Optional.ofNullable(servicesMap.get(type)) + .orElseThrow(() -> new NotFoundException(MessageFormat.format("Shiny application download is not supported for [{0}] analyses.", type))); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java b/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java new file mode 100644 index 0000000000..7905a8ed5d --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ApplicationBrief.java @@ -0,0 +1,31 @@ +package org.ohdsi.webapi.shiny; + +public class ApplicationBrief { + private String name; + private String title; + private String description; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java new file mode 100644 index 0000000000..dabbd86d0f --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -0,0 +1,104 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; +import com.odysseusinc.arachne.commons.utils.ZipUtils; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.ohdsi.webapi.cohortdefinition.CohortDefinition; +import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; +import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; +import org.ohdsi.webapi.service.CohortDefinitionService; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.function.Consumer; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortCountsShinyPackagingService implements ShinyPackagingService { + + private static final Logger log = LoggerFactory.getLogger(CohortCountsShinyPackagingService.class); + private static String SHINY_COHORT_COUNTS = "/shiny/shiny-cohortCounts.zip"; + private final ObjectMapper objectMapper = new ObjectMapper(); + @Autowired + private CohortDefinitionService cohortDefinitionService; + @Autowired + private CohortDefinitionRepository cohortDefinitionRepository; + @Value("${shiny.atlas.url}") + private String atlasUrl; + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT; + } + + @Override + public TemporaryFile packageApp(Integer cohortId, String sourceKey) { + return TempFileUtils.doInDirectory(path -> { + CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); + ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", cohortId)); + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_COUNTS, "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 0); //by event + InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 1); //by person + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + writeInclusionRuleReport(dataDir, byEventReport, sourceKey + "_by_event.json"); + writeInclusionRuleReport(dataDir, byPersonReport, sourceKey + "_by_person.json"); + writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)); + writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())); + Path appArchive = Files.createTempFile("shinyapp_", ".zip"); + ZipUtils.zipDirectory(appArchive, path); + return new TemporaryFile(String.format("%s_cohortCounts_shinyApp.zip", CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); + } catch (IOException e) { + log.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + @Override + public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { + CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); + ApplicationBrief brief = new ApplicationBrief(); + brief.setName(MessageFormat.format("cohort_{0}_{1}", cohort.getId(), sourceKey)); + brief.setTitle(cohort.getName()); + brief.setDescription(cohort.getDescription()); + return brief; + } + + private void writeTextFile(Path path, Consumer writer) { + try(OutputStream out = Files.newOutputStream(path); PrintWriter printWriter = new PrintWriter(out)) { + writer.accept(printWriter); + } catch (IOException e) { + log.error("Filed to write file", e); + throw new InternalServerErrorException(); + } + } + + private void writeInclusionRuleReport(Path parentDir, InclusionRuleReport report, String filename) { + try(OutputStream out = Files.newOutputStream(Files.createFile(parentDir.resolve(filename)))) { + objectMapper.writeValue(out, report); + } catch (IOException e) { + log.error("Failed to package Cohort Counts Shiny application", e); + throw new InternalServerErrorException(); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java new file mode 100644 index 0000000000..d6a794ba2c --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -0,0 +1,127 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import okhttp3.Call; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.apache.commons.lang3.StringUtils; +import org.ohdsi.webapi.service.ShinyService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.BeanInitializationException; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.context.annotation.Scope; +import org.springframework.context.annotation.ScopedProxyMode; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.util.UUID; + +@Service +@ConditionalOnBean(ShinyService.class) +@Scope(proxyMode = ScopedProxyMode.TARGET_CLASS) +public class PositConnectClient implements InitializingBean { + + private static final Logger log = LoggerFactory.getLogger(PositConnectClient.class); + private static final MediaType JSON_TYPE = MediaType.parse(org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE); + private static final String HEADER_AUTH = "Authorization"; + private static final String AUTH_PREFIX = "Key"; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Value("${shiny.connect.api_key}") + private String apiKey; + @Value("${shiny.connect.url}") + private String connectUrl; + + public UUID createContentItem(ApplicationBrief brief) { + ContentItem contentItem = new ContentItem(); + contentItem.accessType = "acl"; + contentItem.name = brief.getName(); + contentItem.title = brief.getTitle(); + contentItem.description = brief.getDescription(); + RequestBody body = RequestBody.create(toJson(contentItem), JSON_TYPE); + String url = connect("/v1/content"); + Request.Builder request = new Request.Builder() + .url(url) + .post(body); + Call call = call(request, apiKey); + try(Response response = call.execute()) { + log.debug("Call [{}] returned [{}]", url, response.code()); + if (response.body() == null) { + log.error("Failed to create a content, an empty result returned [{}]", url); + throw new PositConnectClientException("Failed to create a content, an empty result returned"); + } + ContentItemResponse contentItemResponse = objectMapper.readValue(response.body().charStream(), + ContentItemResponse.class); + return contentItemResponse.guid; + } catch (IOException e) { + log.error("Failed to execute call [{}]", url, e); + throw new PositConnectClientException("Failed to execute call: " + e.getMessage()); + } + } + + public Integer uploadBundle(UUID contentId, TemporaryFile bundle) { + return null; + } + + public String deployBundle(UUID contentId, Integer bundleId) { + return null; + } + + private String toJson(T value) { + try { + return objectMapper.writeValueAsString(value); + } catch (JsonProcessingException e) { + log.error("Failed to execute Connect request", e); + throw new PositConnectClientException("Failed to execute Connect request", e); + } + } + + private Call call(Request.Builder request, String token) { + OkHttpClient client = new OkHttpClient.Builder() + .retryOnConnectionFailure(false) + .build(); + return client.newCall(request.header(HEADER_AUTH, AUTH_PREFIX + " " + token).build()); + } + + @Override + public void afterPropertiesSet() throws Exception { + if (StringUtils.isBlank(apiKey)) { + log.error("Set Posit Connect API Key to property \"shiny.connect.api_key\""); + throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api_key\""); + } + if (StringUtils.isBlank(connectUrl)) { + log.error("Set Posit Connect URL to property \"shiny.connect.url\""); + throw new BeanInitializationException("Set Posit Connect URL to property \"shiny.connect.url\""); + } + } + + private String connect(String path) { + return StringUtils.removeEnd(connectUrl, "/") + "/" + StringUtils.removeStart(path, "/"); + } + + static class ContentItem { + public String name; + public String title; + public String description; + @JsonProperty("access_type") + public String accessType; + } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class ContentItemResponse extends ContentItem { + public UUID guid; + @JsonProperty("owner_guid") + public UUID ownerGuid; + public String id; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java new file mode 100644 index 0000000000..37a98fc18b --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClientException.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +public class PositConnectClientException extends RuntimeException { + public PositConnectClientException(String message) { + super(message); + } + + public PositConnectClientException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java new file mode 100644 index 0000000000..a0c2ab5590 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; + +import java.nio.file.Path; + +public interface ShinyPackagingService { + CommonAnalysisType getType(); + TemporaryFile packageApp(Integer analysisId, String sourceKey); + ApplicationBrief getBrief(Integer analysisId, String sourceKey); +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java new file mode 100644 index 0000000000..a78766d47b --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java @@ -0,0 +1,86 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; +import org.ohdsi.webapi.model.CommonEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; +import java.util.UUID; + +@Entity +@Table(name = "shiny_published") +public class ShinyPublishedEntity extends CommonEntity { + + @Id + @GenericGenerator( + name = "shiny_published_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "shiny_published_sequence"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "shiny_published_generator") + private Long id; + private CommonAnalysisType type; + @Column(name = "analysis_id") + private Long analysisId; + @Column(name = "source_key") + private String sourceKey; + @Column(name = "execution_id") + private Long executionId; + private UUID contentId; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public CommonAnalysisType getType() { + return type; + } + + public void setType(CommonAnalysisType type) { + this.type = type; + } + + public Long getAnalysisId() { + return analysisId; + } + + public void setAnalysisId(Long analysisId) { + this.analysisId = analysisId; + } + + public Long getExecutionId() { + return executionId; + } + + public void setExecutionId(Long executionId) { + this.executionId = executionId; + } + + public UUID getContentId() { + return contentId; + } + + public void setContentId(UUID contentId) { + this.contentId = contentId; + } + + public String getSourceKey() { + return sourceKey; + } + + public void setSourceKey(String sourceKey) { + this.sourceKey = sourceKey; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java new file mode 100644 index 0000000000..0d589c3eff --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +import java.util.Optional; + +@Repository +public interface ShinyPublishedRepository extends JpaRepository { + Optional findByAnalysisIdAAndSourceKey(Long id, String sourceKey); +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java b/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java new file mode 100644 index 0000000000..406f83ebe9 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/TemporaryFile.java @@ -0,0 +1,22 @@ +package org.ohdsi.webapi.shiny; + + +import java.nio.file.Path; + +public class TemporaryFile { + private final String filename; + private final Path file; + + public TemporaryFile(String filename, Path file) { + this.filename = filename; + this.file = file; + } + + public String getFilename() { + return filename; + } + + public Path getFile() { + return file; + } +} diff --git a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java index c2f555e322..e3182976a0 100644 --- a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java +++ b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java @@ -1,8 +1,13 @@ package org.ohdsi.webapi.util; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; +import org.springframework.core.NestedIOException; import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Function; public class TempFileUtils { @@ -16,4 +21,17 @@ public static File copyResourceToTempFile(String resource, String prefix, String } return tempFile; } + + public static F doInDirectory(Function action) { + try { + Path tempDir = Files.createTempDirectory("webapi-"); + try { + return action.apply(tempDir); + } finally { + FileUtils.deleteQuietly(tempDir.toFile()); + } + } catch (IOException e) { + throw new RuntimeException("Failed to create temp directory, " + e.getMessage()); + } + } } diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties new file mode 100644 index 0000000000..d390958938 --- /dev/null +++ b/src/main/resources/application-shiny.properties @@ -0,0 +1,5 @@ +flyway.locations=${flyway.locations},classpath:shiny/migration + +shiny.atlas.url=${shiny.atlas.url} +shiny.connect.api_key= +shiny.connect.url= \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index cd1afb2013..cef396237f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -278,3 +278,6 @@ versioning.maxAttempt=${versioning.maxAttempt} audit.trail.enabled=${audit.trail.enabled} audit.trail.log.file=${audit.trail.log.file} audit.trail.log.extraFile=${audit.trail.log.extraFile} + +#Shiny +shiny.enabled=${shiny.enabled} \ No newline at end of file diff --git a/src/main/resources/i18n/messages_en.json b/src/main/resources/i18n/messages_en.json index be9e537c5c..3ad9e3ab07 100644 --- a/src/main/resources/i18n/messages_en.json +++ b/src/main/resources/i18n/messages_en.json @@ -1233,6 +1233,15 @@ "tag": "Tag:" } } + }, + "shiny": { + "button": { + "title": "Shiny App", + "menu": { + "download": "Download", + "publish": "Publish" + } + } } }, "facets": { diff --git a/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql b/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql new file mode 100644 index 0000000000..3d0c691c39 --- /dev/null +++ b/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql @@ -0,0 +1,16 @@ +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:download:*:*:*:get', + 'Download Shiny Application'; +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:publish:*:*:*:get', + 'Download Shiny Application'; + +INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission sp, + ${ohdsiSchema}.sec_role sr +WHERE sp."value" in + ('shiny:download:*:*:*:get', 'shiny:publish:*:*:*:get') + AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql b/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql new file mode 100644 index 0000000000..4845d3e55e --- /dev/null +++ b/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql @@ -0,0 +1,14 @@ +CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; + +CREATE TABLE ${ohdsiSchema}.shiny_published( + id BIGINT PRIMARY KEY default nextval('${ohdsiSchema}.shiny_published_sequence'), + type VARCHAR NOT NULL, + analysis_id BIGINT NOT NULL, + execution_id BIGINT, + source_key VARCHAR, + content_id UUID, + created_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + modified_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), + modified_date TIMESTAMP WITH TIME ZONE +); From 34bab7dac4b3621c4a8c92d6bffa8d9f85c959ef Mon Sep 17 00:00:00 2001 From: wivern Date: Fri, 22 Dec 2023 19:39:50 +0300 Subject: [PATCH 02/32] ATL-29 API Support for R Shiny Apps --- .../java/org/ohdsi/webapi/JerseyConfig.java | 8 ++ .../ohdsi/webapi/service/ShinyService.java | 58 +++----------- .../CohortCountsShinyPackagingService.java | 5 +- .../webapi/shiny/PackagingStrategies.java | 75 +++++++++++++++++++ .../ohdsi/webapi/shiny/PackagingStrategy.java | 7 ++ .../ohdsi/webapi/shiny/ShinyController.java | 66 ++++++++++++++++ .../webapi/shiny/ShinyPackagingService.java | 2 +- .../webapi/shiny/ShinyPublishedEntity.java | 1 + .../shiny/ShinyPublishedRepository.java | 2 +- 9 files changed, 171 insertions(+), 53 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyController.java diff --git a/src/main/java/org/ohdsi/webapi/JerseyConfig.java b/src/main/java/org/ohdsi/webapi/JerseyConfig.java index 91dbd2d15f..ed18c18840 100644 --- a/src/main/java/org/ohdsi/webapi/JerseyConfig.java +++ b/src/main/java/org/ohdsi/webapi/JerseyConfig.java @@ -26,10 +26,12 @@ import org.ohdsi.webapi.service.IRAnalysisResource; import org.ohdsi.webapi.service.JobService; import org.ohdsi.webapi.service.PersonService; +import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.service.SqlRenderService; import org.ohdsi.webapi.service.TherapyPathResultsService; import org.ohdsi.webapi.service.UserService; import org.ohdsi.webapi.service.VocabularyService; +import org.ohdsi.webapi.shiny.ShinyController; import org.ohdsi.webapi.source.SourceController; import org.springframework.beans.factory.InitializingBean; import org.springframework.beans.factory.annotation.Value; @@ -37,6 +39,7 @@ import javax.inject.Singleton; import javax.ws.rs.ext.RuntimeDelegate; +import java.util.List; /** * @@ -46,6 +49,8 @@ public class JerseyConfig extends ResourceConfig implements InitializingBean { @Value("${jersey.resources.root.package}") private String rootPackage; + @Value("${shiny.enabled:false}") + private Boolean shinyEnabled; public JerseyConfig() { RuntimeDelegate.setInstance(new org.glassfish.jersey.internal.RuntimeDelegateImpl()); @@ -92,5 +97,8 @@ protected void configure() { .in(Singleton.class); } }); + if (shinyEnabled) { + register(ShinyController.class); + } } } diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java index 14750b3197..3c9df0ce6c 100644 --- a/src/main/java/org/ohdsi/webapi/service/ShinyService.java +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -3,6 +3,8 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.glassfish.jersey.media.multipart.ContentDisposition; import org.ohdsi.webapi.shiny.ApplicationBrief; +import org.ohdsi.webapi.shiny.PackagingStrategies; +import org.ohdsi.webapi.shiny.PackagingStrategy; import org.ohdsi.webapi.shiny.PositConnectClient; import org.ohdsi.webapi.shiny.ShinyPackagingService; import org.ohdsi.webapi.shiny.ShinyPublishedEntity; @@ -16,6 +18,7 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; +import javax.inject.Inject; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.NotFoundException; @@ -39,7 +42,6 @@ @Component @ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") -@Path("/shiny") public class ShinyService { private final Map servicesMap; @Autowired @@ -49,63 +51,23 @@ public class ShinyService { @Autowired private PositConnectClient connectClient; + @Inject public ShinyService(List services) { servicesMap = services.stream().collect(Collectors.toMap(ShinyPackagingService::getType, Function.identity())); } - @GET - @Path("/download/{type}/{id}/{sourceKey}") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_OCTET_STREAM) - @DataSourceAccess - public Response downloadShinyApp( - @PathParam("type") String type, - @PathParam("id") final int id, - @PathParam("sourceKey") @SourceKey String sourceKey - ) throws IOException { - TemporaryFile data = packageShinyApp(type, id, sourceKey); - ContentDisposition contentDisposition = ContentDisposition.type("attachment") - .fileName(data.getFilename()) - .build(); - return Response - .ok(Files.newInputStream(data.getFile())) - .header(HttpHeaders.CONTENT_TYPE, "application/zip") - .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition) - .build(); - } - - @GET - @Path("/publish/{type}/{id}/{sourceKey}") - @Consumes(MediaType.APPLICATION_JSON) - @Produces(MediaType.APPLICATION_OCTET_STREAM) - @DataSourceAccess - @Transactional - public Response publishShinyApp( - @PathParam("type") String type, - @PathParam("id") final int id, - @PathParam("sourceKey") @SourceKey String sourceKey - ) { - publishApp(type, id, sourceKey); - return Response.ok().build(); - } - - private void publishApp(String type, int id, String sourceKey) { - TemporaryFile data = packageShinyApp(type, id, sourceKey); + public void publishApp(String type, int id, String sourceKey) { + TemporaryFile data = packageShinyApp(type, id, sourceKey, PackagingStrategies.targz()); ShinyPublishedEntity publication = getPublication(id, sourceKey); ShinyPackagingService service = findShinyService(CommonAnalysisType.valueOf(type.toUpperCase())); UUID contentId = Optional.ofNullable(publication.getContentId()) .orElseGet(() -> connectClient.createContentItem(service.getBrief(id, sourceKey))); - TemporaryFile bundle = prepareBundle(data); - Integer bundleId = connectClient.uploadBundle(contentId, bundle); + Integer bundleId = connectClient.uploadBundle(contentId, data); connectClient.deployBundle(contentId, bundleId); } - private TemporaryFile prepareBundle(TemporaryFile data) { - return null; - } - private ShinyPublishedEntity getPublication(int id, String sourceKey) { - return shinyPublishedRepository.findByAnalysisIdAAndSourceKey(Integer.toUnsignedLong(id), sourceKey).orElseGet(() -> { + return shinyPublishedRepository.findByAnalysisIdAndSourceKey(Integer.toUnsignedLong(id), sourceKey).orElseGet(() -> { ShinyPublishedEntity entity = new ShinyPublishedEntity(); entity.setAnalysisId(Integer.toUnsignedLong(id)); entity.setSourceKey(sourceKey); @@ -115,10 +77,10 @@ private ShinyPublishedEntity getPublication(int id, String sourceKey) { }); } - private TemporaryFile packageShinyApp(String type, int id, String sourceKey) { + public TemporaryFile packageShinyApp(String type, int id, String sourceKey, PackagingStrategy packaging) { CommonAnalysisType analysisType = CommonAnalysisType.valueOf(type.toUpperCase()); ShinyPackagingService service = findShinyService(analysisType); - return service.packageApp(id, sourceKey); + return service.packageApp(id, sourceKey, packaging); } private ShinyPackagingService findShinyService(CommonAnalysisType type) { diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index dabbd86d0f..d5b1eccfdf 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -49,7 +49,7 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer cohortId, String sourceKey) { + public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStrategy packaging) { return TempFileUtils.doInDirectory(path -> { CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", cohortId)); @@ -64,8 +64,7 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey) { writeInclusionRuleReport(dataDir, byPersonReport, sourceKey + "_by_person.json"); writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)); writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())); - Path appArchive = Files.createTempFile("shinyapp_", ".zip"); - ZipUtils.zipDirectory(appArchive, path); + Path appArchive = packaging.apply(path); return new TemporaryFile(String.format("%s_cohortCounts_shinyApp.zip", CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); } catch (IOException e) { log.error("Failed to prepare Shiny application", e); diff --git a/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java new file mode 100644 index 0000000000..b09fbe134d --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategies.java @@ -0,0 +1,75 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.utils.ZipUtils; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.apache.commons.io.IOUtils; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class PackagingStrategies { + public static PackagingStrategy zip() { + return path -> { + try { + Path appArchive = Files.createTempFile("shinyapp_", ".zip"); + ZipUtils.zipDirectory(appArchive, path); + return appArchive; + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + + public static PackagingStrategy targz() { + return path -> { + try { + Path archive = Files.createTempFile("shinyapp_", ".tar.gz"); + try (OutputStream out = Files.newOutputStream(archive); OutputStream gzout = new GzipCompressorOutputStream(out); ArchiveOutputStream arch = new TarArchiveOutputStream(gzout)) { + packDirectoryFiles(path, arch); + } + return archive; + } catch (IOException e) { + throw new RuntimeException(e); + } + }; + } + + private static void packDirectoryFiles(Path path, ArchiveOutputStream arch) throws IOException { + packDirectoryFiles(path, null, arch); + } + + private static void packDirectoryFiles(Path path, String parentDir, ArchiveOutputStream arch) throws IOException { + try (Stream files = Files.list(path)) { + files.forEach(p -> { + try { + File file = p.toFile(); + String filePath = Stream.of(parentDir, p.getFileName().toString()).filter(Objects::nonNull).collect(Collectors.joining("/")); + ArchiveEntry entry = arch.createArchiveEntry(file, filePath); + arch.putArchiveEntry(entry); + if (file.isFile()) { + try (InputStream in = Files.newInputStream(p)) { + IOUtils.copy(in, arch); + } + } + arch.closeArchiveEntry(); + if (file.isDirectory()) { + packDirectoryFiles(p, filePath, arch); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java new file mode 100644 index 0000000000..a2ecfb0c64 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PackagingStrategy.java @@ -0,0 +1,7 @@ +package org.ohdsi.webapi.shiny; + +import java.nio.file.Path; +import java.util.function.Function; + +public interface PackagingStrategy extends Function { +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java new file mode 100644 index 0000000000..f7ba193e13 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyController.java @@ -0,0 +1,66 @@ +package org.ohdsi.webapi.shiny; + +import org.glassfish.jersey.media.multipart.ContentDisposition; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; +import org.ohdsi.webapi.shiro.annotations.SourceKey; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.HttpHeaders; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.nio.file.Files; + +@Component +@ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") +@Path("/shiny") +public class ShinyController { + + @Autowired + private ShinyService service; + + @GET + @Path("/download/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + public Response downloadShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) throws IOException { + TemporaryFile data = service.packageShinyApp(type, id, sourceKey, PackagingStrategies.zip()); + ContentDisposition contentDisposition = ContentDisposition.type("attachment") + .fileName(data.getFilename()) + .build(); + return Response + .ok(Files.newInputStream(data.getFile())) + .header(HttpHeaders.CONTENT_TYPE, "application/zip") + .header(HttpHeaders.CONTENT_DISPOSITION, contentDisposition) + .build(); + } + + @GET + @Path("/publish/{type}/{id}/{sourceKey}") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @DataSourceAccess + @Transactional + public Response publishShinyApp( + @PathParam("type") String type, + @PathParam("id") final int id, + @PathParam("sourceKey") @SourceKey String sourceKey + ) { + service.publishApp(type, id, sourceKey); + return Response.ok().build(); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java index a0c2ab5590..c1e58ad12c 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -6,6 +6,6 @@ public interface ShinyPackagingService { CommonAnalysisType getType(); - TemporaryFile packageApp(Integer analysisId, String sourceKey); + TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingStrategy packaging); ApplicationBrief getBrief(Integer analysisId, String sourceKey); } diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java index a78766d47b..92e207a82b 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedEntity.java @@ -34,6 +34,7 @@ public class ShinyPublishedEntity extends CommonEntity { private String sourceKey; @Column(name = "execution_id") private Long executionId; + @Column(name = "content_id") private UUID contentId; public Long getId() { diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java index 0d589c3eff..e76130b10f 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPublishedRepository.java @@ -7,5 +7,5 @@ @Repository public interface ShinyPublishedRepository extends JpaRepository { - Optional findByAnalysisIdAAndSourceKey(Long id, String sourceKey); + Optional findByAnalysisIdAndSourceKey(Long id, String sourceKey); } From e44462a2764db728c93aba5e029ce051ee205655 Mon Sep 17 00:00:00 2001 From: wivern Date: Tue, 9 Jan 2024 16:17:26 +0300 Subject: [PATCH 03/32] ATL-29 API Support for R Shiny Apps --- .../ohdsi/webapi/service/ShinyService.java | 8 +- .../webapi/shiny/PositConnectClient.java | 81 +++++++++++++++---- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java index 3c9df0ce6c..d67c5caf38 100644 --- a/src/main/java/org/ohdsi/webapi/service/ShinyService.java +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -13,6 +13,8 @@ import org.ohdsi.webapi.shiro.PermissionManager; import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; import org.ohdsi.webapi.shiro.annotations.SourceKey; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -43,6 +45,7 @@ @Component @ConditionalOnProperty(name = "shiny.enabled", havingValue = "true") public class ShinyService { + private static final Logger log = LoggerFactory.getLogger(ShinyService.class); private final Map servicesMap; @Autowired private ShinyPublishedRepository shinyPublishedRepository; @@ -62,8 +65,9 @@ public void publishApp(String type, int id, String sourceKey) { ShinyPackagingService service = findShinyService(CommonAnalysisType.valueOf(type.toUpperCase())); UUID contentId = Optional.ofNullable(publication.getContentId()) .orElseGet(() -> connectClient.createContentItem(service.getBrief(id, sourceKey))); - Integer bundleId = connectClient.uploadBundle(contentId, data); - connectClient.deployBundle(contentId, bundleId); + String bundleId = connectClient.uploadBundle(contentId, data); + String taskId = connectClient.deployBundle(contentId, bundleId); + log.debug("Bundle [{}] is deployed to Shiny server, task id: [{}]", id, taskId); } private ShinyPublishedEntity getPublication(int id, String sourceKey) { diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java index d6a794ba2c..fa4852f628 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -1,5 +1,6 @@ package org.ohdsi.webapi.shiny; +import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; @@ -11,6 +12,7 @@ import okhttp3.RequestBody; import okhttp3.Response; import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.cert.ocsp.Req; import org.ohdsi.webapi.service.ShinyService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,6 +25,8 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.text.MessageFormat; +import java.time.Instant; import java.util.UUID; @Service @@ -32,6 +36,7 @@ public class PositConnectClient implements InitializingBean { private static final Logger log = LoggerFactory.getLogger(PositConnectClient.class); private static final MediaType JSON_TYPE = MediaType.parse(org.springframework.http.MediaType.APPLICATION_JSON_UTF8_VALUE); + private static final MediaType OCTET_STREAM_TYPE = MediaType.parse(org.springframework.http.MediaType.APPLICATION_OCTET_STREAM_VALUE); private static final String HEADER_AUTH = "Authorization"; private static final String AUTH_PREFIX = "Key"; @@ -50,33 +55,46 @@ public UUID createContentItem(ApplicationBrief brief) { contentItem.description = brief.getDescription(); RequestBody body = RequestBody.create(toJson(contentItem), JSON_TYPE); String url = connect("/v1/content"); + ContentItemResponse response = doCall(ContentItemResponse.class, url, body); + return response.guid; + } + + public String uploadBundle(UUID contentId, TemporaryFile bundle) { + String url = connect(MessageFormat.format("/v1/content/{0}/bundles", contentId)); + BundleResponse response = doCall(BundleResponse.class, url, RequestBody.create(bundle.getFile().toFile(), OCTET_STREAM_TYPE)); + return response.id; + } + + public String deployBundle(UUID contentId, String bundleId) { + String url = connect(MessageFormat.format("/v1/content/{0}/deploy", contentId)); + BundleRequest request = new BundleRequest(); + request.bundleId = bundleId; + RequestBody requestBody = RequestBody.create(toJson(request), JSON_TYPE); + BundleDeploymentResponse response = doCall(BundleDeploymentResponse.class, url, requestBody); + return response.taskId; + } + + private T doCall(Class responseClass, String url, RequestBody requestBody) { Request.Builder request = new Request.Builder() .url(url) - .post(body); + .post(requestBody); Call call = call(request, apiKey); try(Response response = call.execute()) { - log.debug("Call [{}] returned [{}]", url, response.code()); + if (!response.isSuccessful()) { + log.error("Request [{}] returned code: [{}], message: [{}]", url, response.code(), response.message()); + throw new PositConnectClientException(MessageFormat.format("Request [{0}] returned code: [{1}], message: [{2}]", url, response.code(), response.message())); + } if (response.body() == null) { log.error("Failed to create a content, an empty result returned [{}]", url); throw new PositConnectClientException("Failed to create a content, an empty result returned"); } - ContentItemResponse contentItemResponse = objectMapper.readValue(response.body().charStream(), - ContentItemResponse.class); - return contentItemResponse.guid; + return objectMapper.readValue(response.body().charStream(), responseClass); } catch (IOException e) { log.error("Failed to execute call [{}]", url, e); - throw new PositConnectClientException("Failed to execute call: " + e.getMessage()); + throw new PositConnectClientException(MessageFormat.format("Failed to execute call [{0}]: {1}", url, e.getMessage())); } } - public Integer uploadBundle(UUID contentId, TemporaryFile bundle) { - return null; - } - - public String deployBundle(UUID contentId, Integer bundleId) { - return null; - } - private String toJson(T value) { try { return objectMapper.writeValueAsString(value); @@ -124,4 +142,39 @@ static class ContentItemResponse extends ContentItem { public UUID ownerGuid; public String id; } + + @JsonIgnoreProperties(ignoreUnknown = true) + static class BundleResponse { + public String id; + @JsonProperty("content_guid") + public String contentGuid; + @JsonProperty("created_time") + public Instant createdTime; + @JsonProperty("cluster_name") + public String clusterName; + @JsonProperty("image_name") + public String imageName; + @JsonProperty("r_version") + public String rVersion; + @JsonProperty("r_environment_management") + public Boolean rEnvironmentManagement; + @JsonProperty("py_version") + public String pyVersion; + @JsonProperty("py_environment_management") + public Boolean pyEnvironmentManagement; + @JsonProperty("quarto_version") + public String quartoVersion; + public Boolean active; + public Integer size; + } + + static class BundleRequest { + @JsonProperty("bundle_id") + public String bundleId; + } + + static class BundleDeploymentResponse { + @JsonProperty("task_id") + public String taskId; + } } From 9a714bd588bd305d0bee124b108e0989fe3094b2 Mon Sep 17 00:00:00 2001 From: git Date: Thu, 29 Feb 2024 10:36:57 +0300 Subject: [PATCH 04/32] Migrations --- ....20240227162911__add_shiny_permissions.sql | 20 +++++++++++++++++++ ....20240227181535__add_shiny_publication.sql | 15 ++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql create mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql new file mode 100644 index 0000000000..10f3bec3fb --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql @@ -0,0 +1,20 @@ +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:download:*:*:*:get', + 'Download Shiny Application presenting analysis results'; + +INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) +SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), + 'shiny:publish:*:*:*:get', + 'Publish Shiny Application presenting analysis results to external resource'; + +INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) +SELECT sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission sp, + ${ohdsiSchema}.sec_role sr +WHERE sp."value" in + ( + 'shiny:download:*:*:*:get', + 'shiny:publish:*:*:*:get' + ) + AND sr.name IN ('Atlas users'); \ No newline at end of file diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql new file mode 100644 index 0000000000..999a5e659e --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql @@ -0,0 +1,15 @@ +CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; + +CREATE TABLE ${ohdsiSchema}.shiny_published +( + id BIGINT NOT NULL PRIMARY KEY DEFAULT nextval('${ohdsiSchema}.shiny_published_sequence'), + type VARCHAR, + analysis_id BIGINT, + source_key VARCHAR, + execution_id BIGINT, + content_id UUID, + created_by_id BIGINT, + modified_by_id BIGINT, + created_date TIMESTAMP, + modified_date TIMESTAMP +); \ No newline at end of file From 3246472b1932e3ff54c5b74ed5720f5d9b9212e3 Mon Sep 17 00:00:00 2001 From: git Date: Thu, 29 Feb 2024 18:58:42 +0300 Subject: [PATCH 05/32] Posit API Client updates --- pom.xml | 16 +++--- .../ohdsi/webapi/service/ShinyService.java | 18 ++++++- .../shiny/ConflictPositConnectException.java | 11 ++++ .../webapi/shiny/PositConnectClient.java | 51 ++++++++++++++----- 4 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java diff --git a/pom.xml b/pom.xml index a22226a04a..2e2548100e 100644 --- a/pom.xml +++ b/pom.xml @@ -1238,6 +1238,15 @@ syslog-java-client 1.1.7 + + com.squareup.okhttp3 + okhttp + 4.12.0 + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + @@ -1873,13 +1882,6 @@ http://localhost/Atlas src/main/resources/shiny - - - com.squareup.okhttp3 - okhttp - 4.12.0 - - diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java index d67c5caf38..ea413d9504 100644 --- a/src/main/java/org/ohdsi/webapi/service/ShinyService.java +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -3,9 +3,11 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.glassfish.jersey.media.multipart.ContentDisposition; import org.ohdsi.webapi.shiny.ApplicationBrief; +import org.ohdsi.webapi.shiny.ConflictPositConnectException; import org.ohdsi.webapi.shiny.PackagingStrategies; import org.ohdsi.webapi.shiny.PackagingStrategy; import org.ohdsi.webapi.shiny.PositConnectClient; +import org.ohdsi.webapi.shiny.PositConnectClientException; import org.ohdsi.webapi.shiny.ShinyPackagingService; import org.ohdsi.webapi.shiny.ShinyPublishedEntity; import org.ohdsi.webapi.shiny.ShinyPublishedRepository; @@ -37,6 +39,7 @@ import java.time.Instant; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.UUID; import java.util.function.Function; @@ -64,12 +67,25 @@ public void publishApp(String type, int id, String sourceKey) { ShinyPublishedEntity publication = getPublication(id, sourceKey); ShinyPackagingService service = findShinyService(CommonAnalysisType.valueOf(type.toUpperCase())); UUID contentId = Optional.ofNullable(publication.getContentId()) - .orElseGet(() -> connectClient.createContentItem(service.getBrief(id, sourceKey))); + .orElseGet(() -> createOrFindItem(service.getBrief(id, sourceKey))); String bundleId = connectClient.uploadBundle(contentId, data); String taskId = connectClient.deployBundle(contentId, bundleId); log.debug("Bundle [{}] is deployed to Shiny server, task id: [{}]", id, taskId); } + private UUID createOrFindItem(ApplicationBrief brief) { + try { + return connectClient.createContentItem(brief); + } catch (ConflictPositConnectException e) { + log.warn("Content item [{}] already exist, will update", brief.getName()); + return connectClient.listContentItems().stream() + .filter(i -> Objects.equals(i.name, brief.getName())) + .findFirst() + .map(i -> i.guid) + .orElseThrow(NotFoundException::new); + } + } + private ShinyPublishedEntity getPublication(int id, String sourceKey) { return shinyPublishedRepository.findByAnalysisIdAndSourceKey(Integer.toUnsignedLong(id), sourceKey).orElseGet(() -> { ShinyPublishedEntity entity = new ShinyPublishedEntity(); diff --git a/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java b/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java new file mode 100644 index 0000000000..3f51cd85c1 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ConflictPositConnectException.java @@ -0,0 +1,11 @@ +package org.ohdsi.webapi.shiny; + +public class ConflictPositConnectException extends PositConnectClientException { + public ConflictPositConnectException(String message) { + super(message); + } + + public ConflictPositConnectException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java index fa4852f628..7aff39f398 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -1,10 +1,11 @@ package org.ohdsi.webapi.shiny; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import okhttp3.Call; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -12,7 +13,6 @@ import okhttp3.RequestBody; import okhttp3.Response; import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.cert.ocsp.Req; import org.ohdsi.webapi.service.ShinyService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,8 +25,11 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.lang.reflect.Type; import java.text.MessageFormat; import java.time.Instant; +import java.util.List; +import java.util.Optional; import java.util.UUID; @Service @@ -40,7 +43,7 @@ public class PositConnectClient implements InitializingBean { private static final String HEADER_AUTH = "Authorization"; private static final String AUTH_PREFIX = "Key"; - private final ObjectMapper objectMapper = new ObjectMapper(); + private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); @Value("${shiny.connect.api_key}") private String apiKey; @@ -55,13 +58,20 @@ public UUID createContentItem(ApplicationBrief brief) { contentItem.description = brief.getDescription(); RequestBody body = RequestBody.create(toJson(contentItem), JSON_TYPE); String url = connect("/v1/content"); - ContentItemResponse response = doCall(ContentItemResponse.class, url, body); + ContentItemResponse response = doPost(ContentItemResponse.class, url, body); return response.guid; } + public List listContentItems() { + String url = connect("/v1/content"); + Request.Builder request = new Request.Builder() + .url(url); + return doCall(new TypeReference>() {}, request, url); + } + public String uploadBundle(UUID contentId, TemporaryFile bundle) { String url = connect(MessageFormat.format("/v1/content/{0}/bundles", contentId)); - BundleResponse response = doCall(BundleResponse.class, url, RequestBody.create(bundle.getFile().toFile(), OCTET_STREAM_TYPE)); + BundleResponse response = doPost(BundleResponse.class, url, RequestBody.create(bundle.getFile().toFile(), OCTET_STREAM_TYPE)); return response.id; } @@ -70,19 +80,36 @@ public String deployBundle(UUID contentId, String bundleId) { BundleRequest request = new BundleRequest(); request.bundleId = bundleId; RequestBody requestBody = RequestBody.create(toJson(request), JSON_TYPE); - BundleDeploymentResponse response = doCall(BundleDeploymentResponse.class, url, requestBody); + BundleDeploymentResponse response = doPost(BundleDeploymentResponse.class, url, requestBody); return response.taskId; } - private T doCall(Class responseClass, String url, RequestBody requestBody) { + private T doPost(Class responseClass, String url, RequestBody requestBody) { Request.Builder request = new Request.Builder() .url(url) .post(requestBody); + return doCall(responseClass, request, url); + } + + private T doCall(Class responseClass, Request.Builder request, String url) { + return doCall(new TypeReference() { + @Override + public Type getType() { + return responseClass; + } + }, request, url); + } + + private T doCall(TypeReference responseClass, Request.Builder request, String url) { Call call = call(request, apiKey); try(Response response = call.execute()) { if (!response.isSuccessful()) { log.error("Request [{}] returned code: [{}], message: [{}]", url, response.code(), response.message()); - throw new PositConnectClientException(MessageFormat.format("Request [{0}] returned code: [{1}], message: [{2}]", url, response.code(), response.message())); + String message = MessageFormat.format("Request [{0}] returned code: [{1}], message: [{2}]", url, response.code(), response.message()); + if (response.code() == 409) { + throw new ConflictPositConnectException(message); + } + throw new PositConnectClientException(message); } if (response.body() == null) { log.error("Failed to create a content, an empty result returned [{}]", url); @@ -124,10 +151,10 @@ public void afterPropertiesSet() throws Exception { } private String connect(String path) { - return StringUtils.removeEnd(connectUrl, "/") + "/" + StringUtils.removeStart(path, "/"); + return StringUtils.removeEnd(connectUrl, "/") + "/__api__/" + StringUtils.removeStart(path, "/"); } - static class ContentItem { + public static class ContentItem { public String name; public String title; public String description; @@ -136,7 +163,7 @@ static class ContentItem { } @JsonIgnoreProperties(ignoreUnknown = true) - static class ContentItemResponse extends ContentItem { + public static class ContentItemResponse extends ContentItem { public UUID guid; @JsonProperty("owner_guid") public UUID ownerGuid; @@ -144,7 +171,7 @@ static class ContentItemResponse extends ContentItem { } @JsonIgnoreProperties(ignoreUnknown = true) - static class BundleResponse { + public static class BundleResponse { public String id; @JsonProperty("content_guid") public String contentGuid; From 6c0d9a6881e321c9095c501e1b5708b830d389fa Mon Sep 17 00:00:00 2001 From: git Date: Wed, 27 Mar 2024 19:39:14 +0300 Subject: [PATCH 06/32] Add data files to manifest before publishing --- pom.xml | 29 ++----- .../CohortCountsShinyPackagingService.java | 78 ++++++++++++++++--- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/pom.xml b/pom.xml index 2e2548100e..fd84bf5f09 100644 --- a/pom.xml +++ b/pom.xml @@ -1880,7 +1880,9 @@ true http://localhost/Atlas - src/main/resources/shiny + src/main/resources/shiny + + default,shiny @@ -1891,7 +1893,7 @@ - ${shiny.directory} + ${shiny.output.directory} shiny-cohortCounts.zip @@ -1899,25 +1901,6 @@ - - org.apache.maven.plugins - maven-scm-plugin - 2.0.1 - - scm:git:git@github.com:OdyOSG/AtlasShinyExport.git - main - branch - - - - checkout-rshiny - generate-resources - - checkout - - - - org.apache.maven.plugins maven-assembly-plugin @@ -1931,8 +1914,8 @@ generate-resources false - ${project.build.directory}/checkout - ${shiny.directory} + ${shiny.app.directory} + ${shiny.output.directory} src/main/assembly/shiny-cohortCounts.xml diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index d5b1eccfdf..b55fe99986 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -1,10 +1,12 @@ package org.ohdsi.webapi.shiny; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; -import com.odysseusinc.arachne.commons.utils.ZipUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.apache.commons.codec.digest.DigestUtils; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; @@ -22,12 +24,14 @@ import javax.ws.rs.InternalServerErrorException; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; import java.util.function.Consumer; +import java.util.stream.Stream; @Service @ConditionalOnBean(ShinyService.class) @@ -56,14 +60,23 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr try { File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_COUNTS, "shiny", ".zip"); CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = parseManifest(manifestPath); + InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 0); //by event InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 1); //by person Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); - writeInclusionRuleReport(dataDir, byEventReport, sourceKey + "_by_event.json"); - writeInclusionRuleReport(dataDir, byPersonReport, sourceKey + "_by_person.json"); - writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)); - writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())); + Stream.of( + writeInclusionRuleReport(dataDir, byEventReport, sourceKey + "_by_event.json"), + writeInclusionRuleReport(dataDir, byPersonReport, sourceKey + "_by_person.json"), + writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)), + writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())) + ).forEach(addDataToManifest(manifest, path)); + writeManifest(manifest, manifestPath); Path appArchive = packaging.apply(path); return new TemporaryFile(String.format("%s_cohortCounts_shinyApp.zip", CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); } catch (IOException e) { @@ -73,6 +86,48 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr }); } + private Consumer addDataToManifest(JsonNode manifest, Path root) { + return file -> { + JsonNode node = manifest.get("files"); + if (node.isObject()) { + ObjectNode filesNode = (ObjectNode) node; + Path relative = root.relativize(file); + ObjectNode item = filesNode.putObject(relative.toString().replace("\\", "/")); + item.put("checksum", checksum(file)); + } else { + log.error("Wrong manifest.json, there is no files section"); + throw new InternalServerErrorException(); + } + }; + } + + private String checksum(Path path) { + try(InputStream in = Files.newInputStream(path)) { + return DigestUtils.md5Hex(in); + } catch (IOException e) { + log.error("Failed to calculate checksum", e); + throw new InternalServerErrorException(); + } + } + + private JsonNode parseManifest(Path path) { + try(InputStream in = Files.newInputStream(path)) { + return objectMapper.readTree(in); + } catch (IOException e) { + log.error("Failed to parse manifest", e); + throw new InternalServerErrorException(); + } + } + + private void writeManifest(JsonNode manifest, Path path) { + try { + objectMapper.writeValue(path.toFile(), manifest); + } catch (IOException e) { + log.error("Failed to write manifest.json", e); + throw new InternalServerErrorException(); + } + } + @Override public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); @@ -83,18 +138,23 @@ public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { return brief; } - private void writeTextFile(Path path, Consumer writer) { + private Path writeTextFile(Path path, Consumer writer) { try(OutputStream out = Files.newOutputStream(path); PrintWriter printWriter = new PrintWriter(out)) { writer.accept(printWriter); + return path; } catch (IOException e) { log.error("Filed to write file", e); throw new InternalServerErrorException(); } } - private void writeInclusionRuleReport(Path parentDir, InclusionRuleReport report, String filename) { - try(OutputStream out = Files.newOutputStream(Files.createFile(parentDir.resolve(filename)))) { - objectMapper.writeValue(out, report); + private Path writeInclusionRuleReport(Path parentDir, InclusionRuleReport report, String filename) { + try { + Path file = Files.createFile(parentDir.resolve(filename)); + try (OutputStream out = Files.newOutputStream(file)) { + objectMapper.writeValue(out, report); + } + return file; } catch (IOException e) { log.error("Failed to package Cohort Counts Shiny application", e); throw new InternalServerErrorException(); From f5e31b4c9056dc6d3c57e36c57312527345a6a62 Mon Sep 17 00:00:00 2001 From: Alex Manoylenko Date: Sun, 7 Apr 2024 10:37:23 +0000 Subject: [PATCH 07/32] Handling the disabled security case --- .../org/ohdsi/webapi/service/ShinyService.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java index ea413d9504..700e7364cd 100644 --- a/src/main/java/org/ohdsi/webapi/service/ShinyService.java +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -1,6 +1,7 @@ package org.ohdsi.webapi.service; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; + import org.glassfish.jersey.media.multipart.ContentDisposition; import org.ohdsi.webapi.shiny.ApplicationBrief; import org.ohdsi.webapi.shiny.ConflictPositConnectException; @@ -13,11 +14,14 @@ import org.ohdsi.webapi.shiny.ShinyPublishedRepository; import org.ohdsi.webapi.shiny.TemporaryFile; import org.ohdsi.webapi.shiro.PermissionManager; +import org.ohdsi.webapi.shiro.Entities.UserRepository; import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; import org.ohdsi.webapi.shiro.annotations.SourceKey; +import org.ohdsi.webapi.shiro.management.Security; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; @@ -32,6 +36,7 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; + import java.io.IOException; import java.nio.file.Files; import java.sql.Date; @@ -56,6 +61,13 @@ public class ShinyService { private PermissionManager permissionManager; @Autowired private PositConnectClient connectClient; + @Autowired + protected Security security; + @Autowired + protected UserRepository userRepository; + + @Value("#{!'${security.provider}'.equals('DisabledSecurity')}") + private boolean securityEnabled; @Inject public ShinyService(List services) { @@ -91,7 +103,7 @@ private ShinyPublishedEntity getPublication(int id, String sourceKey) { ShinyPublishedEntity entity = new ShinyPublishedEntity(); entity.setAnalysisId(Integer.toUnsignedLong(id)); entity.setSourceKey(sourceKey); - entity.setCreatedBy(permissionManager.getCurrentUser()); + entity.setCreatedBy(securityEnabled ? permissionManager.getCurrentUser() : userRepository.findByLogin(security.getSubject())); entity.setCreatedDate(Date.from(Instant.now())); return entity; }); From 2ac1eb27ae478ec6ae7048889816473a1d005959 Mon Sep 17 00:00:00 2001 From: Alex Manoylenko Date: Sun, 7 Apr 2024 10:40:00 +0000 Subject: [PATCH 08/32] Adding missing placeholders for Posit Connect and API key --- src/main/resources/application-shiny.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties index d390958938..bdc96251da 100644 --- a/src/main/resources/application-shiny.properties +++ b/src/main/resources/application-shiny.properties @@ -1,5 +1,5 @@ flyway.locations=${flyway.locations},classpath:shiny/migration shiny.atlas.url=${shiny.atlas.url} -shiny.connect.api_key= -shiny.connect.url= \ No newline at end of file +shiny.connect.api_key=${shiny.connect.api_key} +shiny.connect.url=${shiny.connect.url} \ No newline at end of file From 57b2cb9675c326a8b14e1cfb08ea5f8c99a801a1 Mon Sep 17 00:00:00 2001 From: git Date: Thu, 16 May 2024 15:39:59 +0300 Subject: [PATCH 09/32] Adds default value for shiny properties --- .../webapi/shiny/PositConnectClient.java | 29 +++++++++---------- .../webapi/shiny/PositConnectProperties.java | 25 ++++++++++++++++ .../webapi/shiny/ShinyConfiguration.java | 12 ++++++++ 3 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java index 7aff39f398..5677a6381d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -18,7 +18,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.BeanInitializationException; import org.springframework.beans.factory.InitializingBean; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.ScopedProxyMode; @@ -29,7 +29,6 @@ import java.text.MessageFormat; import java.time.Instant; import java.util.List; -import java.util.Optional; import java.util.UUID; @Service @@ -45,10 +44,8 @@ public class PositConnectClient implements InitializingBean { private final ObjectMapper objectMapper = new ObjectMapper().registerModule(new JavaTimeModule()); - @Value("${shiny.connect.api_key}") - private String apiKey; - @Value("${shiny.connect.url}") - private String connectUrl; + @Autowired(required = false) + private PositConnectProperties properties; public UUID createContentItem(ApplicationBrief brief) { ContentItem contentItem = new ContentItem(); @@ -101,7 +98,7 @@ public Type getType() { } private T doCall(TypeReference responseClass, Request.Builder request, String url) { - Call call = call(request, apiKey); + Call call = call(request, properties.getApiKey()); try(Response response = call.execute()) { if (!response.isSuccessful()) { log.error("Request [{}] returned code: [{}], message: [{}]", url, response.code(), response.message()); @@ -140,18 +137,20 @@ private Call call(Request.Builder request, String token) { @Override public void afterPropertiesSet() throws Exception { - if (StringUtils.isBlank(apiKey)) { - log.error("Set Posit Connect API Key to property \"shiny.connect.api_key\""); - throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api_key\""); - } - if (StringUtils.isBlank(connectUrl)) { - log.error("Set Posit Connect URL to property \"shiny.connect.url\""); - throw new BeanInitializationException("Set Posit Connect URL to property \"shiny.connect.url\""); + if (properties != null) { + if (StringUtils.isBlank(properties.getApiKey())) { + log.error("Set Posit Connect API Key to property \"shiny.connect.api_key\""); + throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api_key\""); + } + if (StringUtils.isBlank(properties.getUrl())) { + log.error("Set Posit Connect URL to property \"shiny.connect.url\""); + throw new BeanInitializationException("Set Posit Connect URL to property \"shiny.connect.url\""); + } } } private String connect(String path) { - return StringUtils.removeEnd(connectUrl, "/") + "/__api__/" + StringUtils.removeStart(path, "/"); + return StringUtils.removeEnd(properties.getUrl(), "/") + "/__api__/" + StringUtils.removeStart(path, "/"); } public static class ContentItem { diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java new file mode 100644 index 0000000000..6260449f8d --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java @@ -0,0 +1,25 @@ +package org.ohdsi.webapi.shiny; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "shiny.connect") +public class PositConnectProperties { + private String apiKey; + private String url; + + public String getApiKey() { + return apiKey; + } + + public void setApiKey(String apiKey) { + this.apiKey = apiKey; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java new file mode 100644 index 0000000000..f7f2b413f0 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyConfiguration.java @@ -0,0 +1,12 @@ +package org.ohdsi.webapi.shiny; + +import org.ohdsi.webapi.service.ShinyService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnBean(ShinyService.class) +@EnableConfigurationProperties(PositConnectProperties.class) +public class ShinyConfiguration { +} From 9e761fdfb8f9baa6ed87b467dfeebec98890dc39 Mon Sep 17 00:00:00 2001 From: fedor_glushchenko Date: Fri, 17 May 2024 13:24:34 +0300 Subject: [PATCH 10/32] ATL-46: Changed filenaming --- .../webapi/shiny/CohortCountsShinyPackagingService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index b55fe99986..ee6d3d2133 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -6,6 +6,9 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; import org.apache.commons.codec.digest.DigestUtils; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; @@ -78,7 +81,8 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr ).forEach(addDataToManifest(manifest, path)); writeManifest(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s_cohortCounts_shinyApp.zip", CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); + return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), + CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); } catch (IOException e) { log.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); From 5968034bd483aa322ebc1672401ab2a532047598 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 31 May 2024 21:10:07 +0200 Subject: [PATCH 11/32] [ATL-48] Implemented Incidence Rates Analysis Shiny app publishing --- pom.xml | 17 ++ src/main/assembly/shiny-incidenceRates.xml | 18 ++ .../ohdsi/webapi/service/ShinyService.java | 15 -- .../CohortCountsShinyPackagingService.java | 101 ++--------- .../org/ohdsi/webapi/shiny/FileWriter.java | 56 ++++++ .../IncidenceRatesShinyPackagingService.java | 162 ++++++++++++++++++ .../org/ohdsi/webapi/shiny/ManifestUtils.java | 57 ++++++ .../webapi/shiny/ShinyPackagingService.java | 3 + .../resources/application-shiny.properties | 1 + ...cidenceRatesShinyPackagingServiceTest.java | 96 +++++++++++ 10 files changed, 427 insertions(+), 99 deletions(-) create mode 100644 src/main/assembly/shiny-incidenceRates.xml create mode 100644 src/main/java/org/ohdsi/webapi/shiny/FileWriter.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java create mode 100644 src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java diff --git a/pom.xml b/pom.xml index fd84bf5f09..522c4e4df2 100644 --- a/pom.xml +++ b/pom.xml @@ -1896,6 +1896,7 @@ ${shiny.output.directory} shiny-cohortCounts.zip + shiny-incidenceRates.zip @@ -1922,6 +1923,22 @@ shiny-cohortCounts + + build-incidenceRates-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-incidenceRates.xml + + shiny-incidenceRates + + diff --git a/src/main/assembly/shiny-incidenceRates.xml b/src/main/assembly/shiny-incidenceRates.xml new file mode 100644 index 0000000000..0c03a906b8 --- /dev/null +++ b/src/main/assembly/shiny-incidenceRates.xml @@ -0,0 +1,18 @@ + + + shiny-incidenceRates + + zip + + false + + + ./apps/IncidenceRate + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/service/ShinyService.java b/src/main/java/org/ohdsi/webapi/service/ShinyService.java index 700e7364cd..286cec8023 100644 --- a/src/main/java/org/ohdsi/webapi/service/ShinyService.java +++ b/src/main/java/org/ohdsi/webapi/service/ShinyService.java @@ -2,21 +2,17 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import org.glassfish.jersey.media.multipart.ContentDisposition; import org.ohdsi.webapi.shiny.ApplicationBrief; import org.ohdsi.webapi.shiny.ConflictPositConnectException; import org.ohdsi.webapi.shiny.PackagingStrategies; import org.ohdsi.webapi.shiny.PackagingStrategy; import org.ohdsi.webapi.shiny.PositConnectClient; -import org.ohdsi.webapi.shiny.PositConnectClientException; import org.ohdsi.webapi.shiny.ShinyPackagingService; import org.ohdsi.webapi.shiny.ShinyPublishedEntity; import org.ohdsi.webapi.shiny.ShinyPublishedRepository; import org.ohdsi.webapi.shiny.TemporaryFile; import org.ohdsi.webapi.shiro.PermissionManager; import org.ohdsi.webapi.shiro.Entities.UserRepository; -import org.ohdsi.webapi.shiro.annotations.DataSourceAccess; -import org.ohdsi.webapi.shiro.annotations.SourceKey; import org.ohdsi.webapi.shiro.management.Security; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,21 +20,10 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; import javax.inject.Inject; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; import javax.ws.rs.NotFoundException; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.HttpHeaders; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.io.IOException; -import java.nio.file.Files; import java.sql.Date; import java.text.MessageFormat; import java.time.Instant; diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index ee6d3d2133..0f756bf90a 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -1,15 +1,9 @@ package org.ohdsi.webapi.shiny; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.util.Date; -import org.apache.commons.codec.digest.DigestUtils; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; @@ -27,13 +21,12 @@ import javax.ws.rs.InternalServerErrorException; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; -import java.util.function.Consumer; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; import java.util.stream.Stream; @Service @@ -41,12 +34,17 @@ public class CohortCountsShinyPackagingService implements ShinyPackagingService { private static final Logger log = LoggerFactory.getLogger(CohortCountsShinyPackagingService.class); - private static String SHINY_COHORT_COUNTS = "/shiny/shiny-cohortCounts.zip"; - private final ObjectMapper objectMapper = new ObjectMapper(); + private static final String SHINY_COHORT_COUNTS = "/shiny/shiny-cohortCounts.zip"; + @Autowired private CohortDefinitionService cohortDefinitionService; @Autowired private CohortDefinitionRepository cohortDefinitionRepository; + @Autowired + private FileWriter fileWriter; + @Autowired + private ManifestUtils manifestUtils; + @Value("${shiny.atlas.url}") private String atlasUrl; @@ -67,19 +65,19 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr if (!Files.exists(manifestPath)) { throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); } - JsonNode manifest = parseManifest(manifestPath); + JsonNode manifest = manifestUtils.parseManifest(manifestPath); InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 0); //by event InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 1); //by person Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); Stream.of( - writeInclusionRuleReport(dataDir, byEventReport, sourceKey + "_by_event.json"), - writeInclusionRuleReport(dataDir, byPersonReport, sourceKey + "_by_person.json"), - writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)), - writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())) - ).forEach(addDataToManifest(manifest, path)); - writeManifest(manifest, manifestPath); + fileWriter.writeObjectAsJsonFile(dataDir, byEventReport, sourceKey + "_by_event.json"), + fileWriter.writeObjectAsJsonFile(dataDir, byPersonReport, sourceKey + "_by_person.json"), + fileWriter.writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)), + fileWriter.writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())) + ).forEach(manifestUtils.addDataToManifest(manifest, path)); + fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); @@ -90,48 +88,6 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr }); } - private Consumer addDataToManifest(JsonNode manifest, Path root) { - return file -> { - JsonNode node = manifest.get("files"); - if (node.isObject()) { - ObjectNode filesNode = (ObjectNode) node; - Path relative = root.relativize(file); - ObjectNode item = filesNode.putObject(relative.toString().replace("\\", "/")); - item.put("checksum", checksum(file)); - } else { - log.error("Wrong manifest.json, there is no files section"); - throw new InternalServerErrorException(); - } - }; - } - - private String checksum(Path path) { - try(InputStream in = Files.newInputStream(path)) { - return DigestUtils.md5Hex(in); - } catch (IOException e) { - log.error("Failed to calculate checksum", e); - throw new InternalServerErrorException(); - } - } - - private JsonNode parseManifest(Path path) { - try(InputStream in = Files.newInputStream(path)) { - return objectMapper.readTree(in); - } catch (IOException e) { - log.error("Failed to parse manifest", e); - throw new InternalServerErrorException(); - } - } - - private void writeManifest(JsonNode manifest, Path path) { - try { - objectMapper.writeValue(path.toFile(), manifest); - } catch (IOException e) { - log.error("Failed to write manifest.json", e); - throw new InternalServerErrorException(); - } - } - @Override public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); @@ -141,27 +97,4 @@ public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { brief.setDescription(cohort.getDescription()); return brief; } - - private Path writeTextFile(Path path, Consumer writer) { - try(OutputStream out = Files.newOutputStream(path); PrintWriter printWriter = new PrintWriter(out)) { - writer.accept(printWriter); - return path; - } catch (IOException e) { - log.error("Filed to write file", e); - throw new InternalServerErrorException(); - } - } - - private Path writeInclusionRuleReport(Path parentDir, InclusionRuleReport report, String filename) { - try { - Path file = Files.createFile(parentDir.resolve(filename)); - try (OutputStream out = Files.newOutputStream(file)) { - objectMapper.writeValue(out, report); - } - return file; - } catch (IOException e) { - log.error("Failed to package Cohort Counts Shiny application", e); - throw new InternalServerErrorException(); - } - } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java new file mode 100644 index 0000000000..e0bd189621 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java @@ -0,0 +1,56 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Consumer; + +@Component +public class FileWriter { + + private static final Logger LOG = LoggerFactory.getLogger(FileWriter.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + + public Path writeTextFile(Path path, Consumer writer) { + try (OutputStream out = Files.newOutputStream(path); PrintWriter printWriter = new PrintWriter(out)) { + writer.accept(printWriter); + return path; + } catch (IOException e) { + LOG.error("Filed to write file", e); + throw new InternalServerErrorException(); + } + } + + public Path writeObjectAsJsonFile(Path parentDir, Object object, String filename) { + try { + Path file = Files.createFile(parentDir.resolve(filename)); + try (OutputStream out = Files.newOutputStream(file)) { + objectMapper.writeValue(out, object); + } + return file; + } catch (IOException e) { + LOG.error("Failed to package Cohort Counts Shiny application", e); + throw new InternalServerErrorException(); + } + } + + public void writeJsonNodeToFile(JsonNode object, Path path) { + try { + objectMapper.writeValue(path.toFile(), object); + } catch (IOException e) { + LOG.error("Failed to write json file", e); + throw new InternalServerErrorException(); + } + } + + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java new file mode 100644 index 0000000000..08a4ae3064 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -0,0 +1,162 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; +import org.ohdsi.webapi.ircalc.AnalysisReport; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; +import org.ohdsi.webapi.service.IRAnalysisService; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.Date; +import java.util.List; +import java.util.stream.Stream; + +@Service +@ConditionalOnBean(ShinyService.class) +public class IncidenceRatesShinyPackagingService implements ShinyPackagingService { + + private static final Logger LOG = LoggerFactory.getLogger(IncidenceRatesShinyPackagingService.class); + + private static final String SHINY_INCIDENCE_RATES_APP_PATH = "/shiny/shiny-incidenceRates.zip"; + private static final String COHORT_TYPE_TARGET = "target"; + private static final String COHORT_TYPE_OUTCOME = "outcome"; + + + @Autowired + private FileWriter fileWriter; + @Autowired + private ManifestUtils manifestUtils; + @Autowired + private IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; + @Autowired + private IRAnalysisService irAnalysisService; + @Autowired + private ObjectMapper objectMapper; + + @Value("${shiny.atlas.url}") + private String atlasUrl; + @Value("${shiny.repo.link}") + private String repoLink; + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.INCIDENCE; + } + + @Override + public TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingStrategy packaging) { + return TempFileUtils.doInDirectory(path -> { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(analysisId); + ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", analysisId)); + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_INCIDENCE_RATES_APP_PATH, "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = manifestUtils.parseManifest(manifestPath); + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + IncidenceRateAnalysisExportExpression expression = objectMapper.readValue( + analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); + String csvWithCohortDetails = prepareCsvWithCohorts(expression); + + Stream analysisReportPaths = streamAnalysisReportsForAllCohortCombinations(expression, analysisId, sourceKey) + .map(analysisReport -> fileWriter.writeObjectAsJsonFile(dataDir, analysisReport, String.format( + "%s_targetId%s_outcomeId%s.json", sourceKey, analysisReport.summary.targetId, analysisReport.summary.outcomeId))); + + Stream additionalMetadataFilesPaths = Stream.of( + fileWriter.writeTextFile(dataDir.resolve("cohorts.csv"), pw -> pw.print(csvWithCohortDetails)), + fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/iranalysis/%s", atlasUrl, analysisId)), + fileWriter.writeTextFile(dataDir.resolve("repo_link.txt"), pw -> pw.print(repoLink)) + ); + + Stream.concat(analysisReportPaths, additionalMetadataFilesPaths) + .forEach(manifestUtils.addDataToManifest(manifest, path)); + + fileWriter.writeJsonNodeToFile(manifest, manifestPath); + Path appArchive = packaging.apply(path); + return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), + CommonFilenameUtils.sanitizeFilename(analysis.getName())), appArchive); + } catch (IOException e) { + LOG.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + private Stream streamAnalysisReportsForAllCohortCombinations(IncidenceRateAnalysisExportExpression expression, Integer analysisId, String sourceKey) { + List targetCohorts = expression.targetCohorts; + List outcomeCohorts = expression.outcomeCohorts; + return targetCohorts.stream() + .map(CohortDTO::getId) + .flatMap(targetCohortId -> streamAnalysisReportsForOneCohortCombination(targetCohortId, outcomeCohorts, analysisId, sourceKey)); + } + + private Stream streamAnalysisReportsForOneCohortCombination(Integer targetCohortId, List outcomeCohorts, Integer analysisId, String sourceKey) { + return outcomeCohorts.stream() + .map(outcomeCohort -> irAnalysisService.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId())); + } + + @Override + public ApplicationBrief getBrief(Integer analysisId, String sourceKey) { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(analysisId); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(MessageFormat.format("incidence_rates_analysis_{0}_{1}", analysisId, sourceKey)); + applicationBrief.setTitle(analysis.getName()); + applicationBrief.setDescription(analysis.getDescription()); + return applicationBrief; + } + + + private String prepareCsvWithCohorts(IncidenceRateAnalysisExportExpression expression) { + final String[] HEADER = {"cohort_id", "cohort_name", "type"}; + List targetCohorts = expression.targetCohorts; + List outcomeCohorts = expression.outcomeCohorts; + try (StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, + CSVFormat.Builder.create() + .setQuoteMode(QuoteMode.NON_NUMERIC) + .setHeader(HEADER) + .build())) { + + for (CohortDTO targetCohort : targetCohorts) { + csvPrinter.printRecord(targetCohort.getId(), targetCohort.getName(), COHORT_TYPE_TARGET); + } + for (CohortDTO outcomeCohort : outcomeCohorts) { + csvPrinter.printRecord(outcomeCohort.getId(), outcomeCohort.getName(), COHORT_TYPE_OUTCOME); + } + return stringWriter.toString(); + } catch (IOException e) { + LOG.error("Failed to create a CSV file with Cohort details", e); + throw new InternalServerErrorException(); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java b/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java new file mode 100644 index 0000000000..b39dd368c2 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ManifestUtils.java @@ -0,0 +1,57 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import javax.ws.rs.InternalServerErrorException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Consumer; + +@Component +public class ManifestUtils { + private final ObjectMapper objectMapper = new ObjectMapper(); + private static final Logger LOG = LoggerFactory.getLogger(ManifestUtils.class); + + public JsonNode parseManifest(Path path) { + try (InputStream in = Files.newInputStream(path)) { + return objectMapper.readTree(in); + } catch (IOException e) { + LOG.error("Failed to parse manifest", e); + throw new InternalServerErrorException(); + } + } + + public Consumer addDataToManifest(JsonNode manifest, Path root) { + return file -> { + JsonNode node = manifest.get("files"); + if (node.isObject()) { + ObjectNode filesNode = (ObjectNode) node; + Path relative = root.relativize(file); + ObjectNode item = filesNode.putObject(relative.toString().replace("\\", "/")); + item.put("checksum", checksum(file)); + } else { + LOG.error("Wrong manifest.json, there is no files section"); + throw new InternalServerErrorException(); + } + }; + } + + private String checksum(Path path) { + try (InputStream in = Files.newInputStream(path)) { + return DigestUtils.md5Hex(in); + } catch (IOException e) { + LOG.error("Failed to calculate checksum", e); + throw new InternalServerErrorException(); + } + } + + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java index c1e58ad12c..2f8fd35a0d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -1,6 +1,9 @@ package org.ohdsi.webapi.shiny; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.ohdsi.webapi.shiny.ApplicationBrief; +import org.ohdsi.webapi.shiny.PackagingStrategy; +import org.ohdsi.webapi.shiny.TemporaryFile; import java.nio.file.Path; diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties index bdc96251da..08b306c66e 100644 --- a/src/main/resources/application-shiny.properties +++ b/src/main/resources/application-shiny.properties @@ -1,5 +1,6 @@ flyway.locations=${flyway.locations},classpath:shiny/migration shiny.atlas.url=${shiny.atlas.url} +shiny.repo.link=${shiny.repo.link} shiny.connect.api_key=${shiny.connect.api_key} shiny.connect.url=${shiny.connect.url} \ No newline at end of file diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java new file mode 100644 index 0000000000..668fdb5faa --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -0,0 +1,96 @@ +package org.ohdsi.webapi.shiny; + + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.ircalc.AnalysisReport; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisDetails; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; +import org.ohdsi.webapi.service.IRAnalysisService; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class IncidenceRatesShinyPackagingServiceTest { + + @Mock + private IncidenceRateAnalysisRepository repository; + @Spy + private ManifestUtils manifestUtils; + @Spy + private FileWriter fileWriter; + @Mock + private IRAnalysisService irAnalysisService; + @Spy + private ObjectMapper objectMapper; + + @InjectMocks + private IncidenceRatesShinyPackagingService sut; + + + private final Integer analysisId = 1; + private final String sourceKey = "sourceKey"; + + @Test + public void shouldGetBrief() { + IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); + + when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); + ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); + assertEquals(brief.getName(), "incidence_rates_analysis_" + analysisId + "_" + sourceKey); + assertEquals(brief.getTitle(), incidenceRateAnalysis.getName()); + assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); + } + + @Test + public void shouldPackageApp() { + IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); + PackagingStrategy packagingStrategy = mock(PackagingStrategy.class); + when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); + when(irAnalysisService.getAnalysisReport(anyInt(), anyString(), anyInt(), anyInt())) + .thenReturn(createAnalysisReport(1, 2)) + .thenReturn(createAnalysisReport(3, 4)) + .thenReturn(createAnalysisReport(5, 6)) + .thenReturn(createAnalysisReport(7, 8)); + TemporaryFile result = sut.packageApp(analysisId, sourceKey, packagingStrategy); + assertNotNull(result); + } + + @Test + public void shouldReturnIncidenceType() { + assertEquals(sut.getType(), CommonAnalysisType.INCIDENCE); + } + + private IncidenceRateAnalysis createIncidenceRateAnalysis() { + IncidenceRateAnalysis incidenceRateAnalysis = new IncidenceRateAnalysis(); + + IncidenceRateAnalysisDetails incidenceRateAnalysisDetails = new IncidenceRateAnalysisDetails(incidenceRateAnalysis); + incidenceRateAnalysisDetails.setExpression("{\"ConceptSets\":[],\"targetIds\":[11,7],\"outcomeIds\":[12,6],\"timeAtRisk\":{\"start\":{\"DateField\":\"StartDate\",\"Offset\":0},\"end\":{\"DateField\":\"EndDate\",\"Offset\":0}},\"studyWindow\":null,\"strata\":[{\"name\":\"Male\",\"description\":null,\"expression\":{\"Type\":\"ALL\",\"Count\":null,\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":null,\"Gender\":[{\"CONCEPT_ID\":8507,\"CONCEPT_NAME\":\"MALE\",\"STANDARD_CONCEPT\":null,\"STANDARD_CONCEPT_CAPTION\":\"Unknown\",\"INVALID_REASON\":null,\"INVALID_REASON_CAPTION\":\"Unknown\",\"CONCEPT_CODE\":\"M\",\"DOMAIN_ID\":\"Gender\",\"VOCABULARY_ID\":\"Gender\",\"CONCEPT_CLASS_ID\":null}],\"Race\":null,\"Ethnicity\":null,\"OccurrenceStartDate\":null,\"OccurrenceEndDate\":null}],\"Groups\":[]}},{\"name\":\"Female\",\"description\":null,\"expression\":{\"Type\":\"ALL\",\"Count\":null,\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":null,\"Gender\":[{\"CONCEPT_ID\":8532,\"CONCEPT_NAME\":\"FEMALE\",\"STANDARD_CONCEPT\":null,\"STANDARD_CONCEPT_CAPTION\":\"Unknown\",\"INVALID_REASON\":null,\"INVALID_REASON_CAPTION\":\"Unknown\",\"CONCEPT_CODE\":\"F\",\"DOMAIN_ID\":\"Gender\",\"VOCABULARY_ID\":\"Gender\",\"CONCEPT_CLASS_ID\":null}],\"Race\":null,\"Ethnicity\":null,\"OccurrenceStartDate\":null,\"OccurrenceEndDate\":null}],\"Groups\":[]}}],\"targetCohorts\":[{\"id\":11,\"name\":\"All population-IR\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"ConditionOccurrence\":{\"ConditionTypeExclude\":false}},{\"DrugExposure\":{\"DrugTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":0,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}},{\"id\":7,\"name\":\"Test Cohort 4\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expressionType\":\"SIMPLE_EXPRESSION\",\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"DrugExposure\":{\"CodesetId\":0,\"DrugTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":30,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"celecoxib\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":1118084,\"CONCEPT_NAME\":\"celecoxib\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"140587\",\"DOMAIN_ID\":\"Drug\",\"VOCABULARY_ID\":\"RxNorm\",\"CONCEPT_CLASS_ID\":\"Ingredient\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":true}]}},{\"id\":1,\"name\":\"Major gastrointestinal (GI) bleeding\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":4280942,\"CONCEPT_NAME\":\"Acute gastrojejunal ulcer with perforation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"66636001\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":28779,\"CONCEPT_NAME\":\"Bleeding esophageal varices\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"17709002\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":198798,\"CONCEPT_NAME\":\"Dieulafoy's vascular malformation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"109558001\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":4112183,\"CONCEPT_NAME\":\"Esophageal varices with bleeding, associated with another disorder\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"195475003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":194382,\"CONCEPT_NAME\":\"External hemorrhoids\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"23913003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":false,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":192671,\"CONCEPT_NAME\":\"Gastrointestinal hemorrhage\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"74474003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":196436,\"CONCEPT_NAME\":\"Internal hemorrhoids\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"90458007\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":false,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":4338225,\"CONCEPT_NAME\":\"Peptic ulcer with perforation\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"88169003\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false},{\"concept\":{\"CONCEPT_ID\":194158,\"CONCEPT_NAME\":\"Perinatal gastrointestinal hemorrhage\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"48729005\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"All\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[{\"name\":\"No prior GI\",\"expression\":{\"Type\":\"ALL\",\"CriteriaList\":[{\"Criteria\":{\"ConditionOccurrence\":{\"CodesetId\":1}},\"StartWindow\":{\"Start\":{\"Coeff\":-1},\"End\":{\"Days\":0,\"Coeff\":1},\"UseIndexEnd\":false,\"UseEventEnd\":false},\"RestrictVisit\":false,\"IgnoreObservationPeriod\":false,\"Occurrence\":{\"Type\":1,\"Count\":0,\"IsDistinct\":false}}],\"DemographicCriteriaList\":[],\"Groups\":[]}}],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{\"StartDate\":\"2010-04-01\",\"EndDate\":\"2010-12-01\"}}}],\"outcomeCohorts\":[{\"id\":12,\"name\":\"Diabetes-IR\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"ConditionOccurrence\":{\"CodesetId\":0,\"First\":true,\"ConditionTypeExclude\":false}}],\"ObservationWindow\":{\"PriorDays\":365,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"First\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"Diabetes-IR\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":201826,\"CONCEPT_NAME\":\"Type 2 diabetes mellitus\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"44054006\",\"DOMAIN_ID\":\"Condition\",\"VOCABULARY_ID\":\"SNOMED\",\"CONCEPT_CLASS_ID\":\"Clinical Finding\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"First\"},\"InclusionRules\":[{\"name\":\"Age over 18\",\"expression\":{\"Type\":\"ALL\",\"CriteriaList\":[],\"DemographicCriteriaList\":[{\"Age\":{\"Value\":18,\"Op\":\"gte\"}}],\"Groups\":[]}}],\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}},{\"id\":6,\"name\":\"TEST COHORT 2\",\"hasWriteAccess\":false,\"hasReadAccess\":false,\"expressionType\":\"SIMPLE_EXPRESSION\",\"expression\":{\"cdmVersionRange\":\">=5.0.0\",\"PrimaryCriteria\":{\"CriteriaList\":[{\"DrugEra\":{\"CodesetId\":0}}],\"ObservationWindow\":{\"PriorDays\":0,\"PostDays\":0},\"PrimaryCriteriaLimit\":{\"Type\":\"All\"}},\"ConceptSets\":[{\"id\":0,\"name\":\"Simvastatin1\",\"expression\":{\"items\":[{\"concept\":{\"CONCEPT_ID\":1539403,\"CONCEPT_NAME\":\"Simvastatin\",\"STANDARD_CONCEPT\":\"S\",\"STANDARD_CONCEPT_CAPTION\":\"Standard\",\"INVALID_REASON\":\"V\",\"INVALID_REASON_CAPTION\":\"Valid\",\"CONCEPT_CODE\":\"36567\",\"DOMAIN_ID\":\"Drug\",\"VOCABULARY_ID\":\"RxNorm\",\"CONCEPT_CLASS_ID\":\"Ingredient\"},\"isExcluded\":false,\"includeDescendants\":true,\"includeMapped\":false}]}}],\"QualifiedLimit\":{\"Type\":\"First\"},\"ExpressionLimit\":{\"Type\":\"All\"},\"InclusionRules\":[],\"EndStrategy\":{\"DateOffset\":{\"DateField\":\"EndDate\",\"Offset\":0}},\"CensoringCriteria\":[],\"CollapseSettings\":{\"CollapseType\":\"ERA\",\"EraPad\":0},\"CensorWindow\":{}}}]}"); + + incidenceRateAnalysis.setId(analysisId); + incidenceRateAnalysis.setName("Analysis Name"); + incidenceRateAnalysis.setDescription("Analysis Description"); + incidenceRateAnalysis.setDetails(incidenceRateAnalysisDetails); + return incidenceRateAnalysis; + } + + private AnalysisReport createAnalysisReport(int targetId, int outcomeId) { + AnalysisReport analysisReport = new AnalysisReport(); + analysisReport.summary = new AnalysisReport.Summary(); + analysisReport.summary.targetId = targetId; + analysisReport.summary.outcomeId = outcomeId; + return analysisReport; + } +} From fbdc8db74956783060805c742e381d9ef1d6a778 Mon Sep 17 00:00:00 2001 From: fedor_glushchenko Date: Thu, 6 Jun 2024 17:32:02 +0300 Subject: [PATCH 12/32] ATL-46: File name changed --- .../ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index 0f756bf90a..bb49c98a9e 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -79,8 +79,7 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr ).forEach(manifestUtils.addDataToManifest(manifest, path)); fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), - CommonFilenameUtils.sanitizeFilename(cohort.getName())), appArchive); + return new TemporaryFile(String.format("Cohort_%s_%s.zip", cohortId, sourceKey), appArchive); } catch (IOException e) { log.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); From d41dbd30588306bdd004c906f745d4e7f643b14e Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Mon, 10 Jun 2024 12:43:21 +0200 Subject: [PATCH 13/32] [IncidenceRate App] Datasource to be read from text file --- .../shiny/IncidenceRatesShinyPackagingService.java | 9 +++++---- .../shiny/IncidenceRatesShinyPackagingServiceTest.java | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index 08a4ae3064..e5ee164202 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -13,7 +13,7 @@ import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; -import org.ohdsi.webapi.service.IRAnalysisService; +import org.ohdsi.webapi.service.IRAnalysisResource; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; import org.ohdsi.webapi.util.TempFileUtils; @@ -55,7 +55,7 @@ public class IncidenceRatesShinyPackagingService implements ShinyPackagingServic @Autowired private IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; @Autowired - private IRAnalysisService irAnalysisService; + private IRAnalysisResource irAnalysisResource; @Autowired private ObjectMapper objectMapper; @@ -95,7 +95,8 @@ public TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingS Stream additionalMetadataFilesPaths = Stream.of( fileWriter.writeTextFile(dataDir.resolve("cohorts.csv"), pw -> pw.print(csvWithCohortDetails)), fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/iranalysis/%s", atlasUrl, analysisId)), - fileWriter.writeTextFile(dataDir.resolve("repo_link.txt"), pw -> pw.print(repoLink)) + fileWriter.writeTextFile(dataDir.resolve("repo_link.txt"), pw -> pw.print(repoLink)), + fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) ); Stream.concat(analysisReportPaths, additionalMetadataFilesPaths) @@ -122,7 +123,7 @@ private Stream streamAnalysisReportsForAllCohortCombinations(Inc private Stream streamAnalysisReportsForOneCohortCombination(Integer targetCohortId, List outcomeCohorts, Integer analysisId, String sourceKey) { return outcomeCohorts.stream() - .map(outcomeCohort -> irAnalysisService.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId())); + .map(outcomeCohort -> irAnalysisResource.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId())); } @Override diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index 668fdb5faa..ccad0fb191 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -13,7 +13,7 @@ import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisDetails; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; -import org.ohdsi.webapi.service.IRAnalysisService; +import org.ohdsi.webapi.service.IRAnalysisResource; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -32,7 +32,7 @@ public class IncidenceRatesShinyPackagingServiceTest { @Spy private FileWriter fileWriter; @Mock - private IRAnalysisService irAnalysisService; + private IRAnalysisResource irAnalysisResource; @Spy private ObjectMapper objectMapper; @@ -59,7 +59,7 @@ public void shouldPackageApp() { IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); PackagingStrategy packagingStrategy = mock(PackagingStrategy.class); when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); - when(irAnalysisService.getAnalysisReport(anyInt(), anyString(), anyInt(), anyInt())) + when(irAnalysisResource.getAnalysisReport(anyInt(), anyString(), anyInt(), anyInt())) .thenReturn(createAnalysisReport(1, 2)) .thenReturn(createAnalysisReport(3, 4)) .thenReturn(createAnalysisReport(5, 6)) From 535294e25695f122f286b7019a39ffd6e704e4bb Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Tue, 18 Jun 2024 18:51:01 +0200 Subject: [PATCH 14/32] [ATL-53] Added datasource export via file to the data folder for the R app --- .../webapi/shiny/CohortCountsShinyPackagingService.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index bb49c98a9e..44d7e09c22 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; @@ -24,9 +23,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.util.Date; import java.util.stream.Stream; @Service @@ -75,7 +71,8 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr fileWriter.writeObjectAsJsonFile(dataDir, byEventReport, sourceKey + "_by_event.json"), fileWriter.writeObjectAsJsonFile(dataDir, byPersonReport, sourceKey + "_by_person.json"), fileWriter.writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)), - fileWriter.writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())) + fileWriter.writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())), + fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) ).forEach(manifestUtils.addDataToManifest(manifest, path)); fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); From 8e9d15e3445373ac4494bc54b96b0d9882de3d3a Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Wed, 5 Jun 2024 18:33:15 +0200 Subject: [PATCH 15/32] [ATL-49] Implemented Cohort Characterizations Shiny App Publishing --- pom.xml | 17 ++ .../shiny-cohortCharacterizations.xml | 18 ++ ...terizationAnalysisHeaderToFieldMapper.java | 43 ++++ ...CharacterizationShinyPackagingService.java | 214 ++++++++++++++++++ .../org/ohdsi/webapi/shiny/FileWriter.java | 2 +- .../webapi/shiny/ShinyPackagingService.java | 5 - .../org/ohdsi/webapi/util/TempFileUtils.java | 12 +- .../shiny/cc-header-field-mapping.csv | 31 +++ 8 files changed, 333 insertions(+), 9 deletions(-) create mode 100644 src/main/assembly/shiny-cohortCharacterizations.xml create mode 100644 src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java create mode 100644 src/main/resources/shiny/cc-header-field-mapping.csv diff --git a/pom.xml b/pom.xml index 522c4e4df2..78ae96706d 100644 --- a/pom.xml +++ b/pom.xml @@ -1897,6 +1897,7 @@ shiny-cohortCounts.zip shiny-incidenceRates.zip + shiny-cohortCharacterizations.zip @@ -1939,6 +1940,22 @@ shiny-incidenceRates + + build-cohortCharacterizations-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-cohortCharacterizations.xml + + shiny-cohortCharacterizations + + diff --git a/src/main/assembly/shiny-cohortCharacterizations.xml b/src/main/assembly/shiny-cohortCharacterizations.xml new file mode 100644 index 0000000000..3599327163 --- /dev/null +++ b/src/main/assembly/shiny-cohortCharacterizations.xml @@ -0,0 +1,18 @@ + + + shiny-cohortCharacterizations + + zip + + false + + + ./apps/cohortCharacterization + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java new file mode 100644 index 0000000000..838e036b93 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationAnalysisHeaderToFieldMapper.java @@ -0,0 +1,43 @@ +package org.ohdsi.webapi.shiny; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.core.io.Resource; +import org.springframework.stereotype.Service; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +@Service +public class CohortCharacterizationAnalysisHeaderToFieldMapper { + + private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationAnalysisHeaderToFieldMapper.class); + private final Map headerFieldMapping; + + public CohortCharacterizationAnalysisHeaderToFieldMapper(@Value("classpath:shiny/cc-header-field-mapping.csv") Resource resource) throws IOException { + this.headerFieldMapping = new HashMap<>(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(resource.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + String[] parts = line.split(",", 2); // Split line into two parts + if (parts.length >= 2) { // Ensure that line has header and field + String header = parts[0]; + String field = parts[1]; + headerFieldMapping.put(header, field); + } else { + LOG.warn("ignoring a line due to unexpected count of parameters (!=2): " + line); + } + } + } + } + + public Map getHeaderFieldMapping() { + return headerFieldMapping; + } + +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java new file mode 100644 index 0000000000..a8e811ea0e --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -0,0 +1,214 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Iterables; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.tuple.Pair; +import org.ohdsi.analysis.CohortMetadata; +import org.ohdsi.analysis.WithId; +import org.ohdsi.analysis.cohortcharacterization.design.CohortCharacterization; +import org.ohdsi.webapi.cohortcharacterization.CcService; +import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; +import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest; +import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; +import org.ohdsi.webapi.cohortcharacterization.report.ExportItem; +import org.ohdsi.webapi.cohortcharacterization.report.Report; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortCharacterizationShinyPackagingService implements ShinyPackagingService { + private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); + private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; + private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_PATH = "/shiny/shiny-cohortCharacterizations.zip"; + @Value("${shiny.atlas.url}") + private String atlasUrl; + @Autowired + private CcService ccService; + @Autowired + private FileWriter fileWriter; + @Autowired + private ManifestUtils manifestUtils; + @Autowired + private CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper; + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT_CHARACTERIZATION; + } + + @Override + @Transactional + public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { + return TempFileUtils.doInDirectory(path -> { + CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); + ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_CHARACTERIZATIONS_APP_PATH, "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = manifestUtils.parseManifest(manifestPath); + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + + Stream generatedCsvPaths = generationResults.getReports() + .stream() + .map(this::convertReportToCSV) + .map(contentsByFilename -> fileWriter.writeTextFile(dataDir.resolve(contentsByFilename.getLeft()), pw -> pw.print(contentsByFilename.getRight()))); + + Stream additionalMetadataFilesPaths = Stream.of( + fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())) + ); + + Stream.concat(generatedCsvPaths, additionalMetadataFilesPaths) + .forEach(manifestUtils.addDataToManifest(manifest, path)); + + fileWriter.writeJsonNodeToFile(manifest, manifestPath); + Path appArchive = packaging.apply(path); + return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), + CommonFilenameUtils.sanitizeFilename(cohortCharacterization.getName())), appArchive); + } catch (IOException e) { + LOG.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + //Pair.left == CSV filename + //Pair.right == CSV contents + private Pair convertReportToCSV(Report report) { + boolean isComparativeAnalysis = report.isComparative; + String analysisName = report.analysisName; + String fileNameFormat = "Export %s(%s).csv"; + String fileName = String.format(fileNameFormat, isComparativeAnalysis ? "comparison " : "", analysisName); + List exportItems = report.items.stream() + .sorted() + .collect(Collectors.toList()); + + String[] header = Iterables.getOnlyElement(report.header); + + String outCsv = prepareCsv(header, exportItems); + return Pair.of(fileName, outCsv); + } + + private String prepareCsv(String[] headers, List exportItems) { + try (StringWriter stringWriter = new StringWriter(); + CSVPrinter csvPrinter = new CSVPrinter(stringWriter, + CSVFormat.Builder + .create() + .setQuoteMode(QuoteMode.NON_NUMERIC) + .setHeader(headers) + .build())) { + for (ExportItem item : exportItems) { + List record = new ArrayList<>(); + for (String header : headers) { + String fieldName = cohortCharacterizationAnalysisHeaderToFieldMapper.getHeaderFieldMapping().get(header); // get the corresponding Java field name + Field field; + try { + if (fieldName != null) { + field = findFieldInClassHierarchy(item.getClass(), fieldName); + if (field != null) { + field.setAccessible(true); + record.add(String.valueOf(field.get(item))); + } else { + record.add(null); + } + } + } catch (IllegalAccessException ex) { + LOG.error("Error occurred while accessing field value", ex); + record.add(""); + } + } + csvPrinter.printRecord(record); + } + return stringWriter.toString(); + } catch (IOException e) { + LOG.error("Failed to create a CSV file with Cohort Characterization analysis details", e); + throw new InternalServerErrorException(); + } + } + + private Field findFieldInClassHierarchy(Class clazz, String fieldName) { + if (clazz == null) { + return null; + } + Field field; + try { + field = clazz.getDeclaredField(fieldName); + } catch (NoSuchFieldException ex) { + field = findFieldInClassHierarchy(clazz.getSuperclass(), fieldName); + } + return field; + } + + private GenerationResults fetchGenerationResults(Integer generationId, CohortCharacterization cohortCharacterization) { + ExecutionResultRequest executionResultRequest = new ExecutionResultRequest(); + List cohortIds = cohortCharacterization.getCohorts() + .stream() + .map(CohortMetadata::getId) + .collect(Collectors.toList()); + List analysisIds = cohortCharacterization.getFeatureAnalyses() + .stream() + .map(WithId::getId) + .map(Number::intValue) + .collect(Collectors.toList()); + List domainIds = cohortCharacterization.getFeatureAnalyses() + .stream() + .map(featureAnalysis -> featureAnalysis.getDomain().getName().toUpperCase()) + .distinct() + .collect(Collectors.toList()); + executionResultRequest.setAnalysisIds(analysisIds); + executionResultRequest.setCohortIds(cohortIds); + executionResultRequest.setDomainIds(domainIds); + executionResultRequest.setShowEmptyResults(Boolean.TRUE); + executionResultRequest.setThresholdValuePct(DEFAULT_THRESHOLD_VALUE); + return ccService.findResult(Long.valueOf(generationId), executionResultRequest); + } + + @Override + @Transactional + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(MessageFormat.format("cohort_characterization_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setTitle(cohortCharacterizationEntity.getName()); + applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); + return applicationBrief; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java index e0bd189621..377fd6cf20 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java +++ b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java @@ -25,7 +25,7 @@ public Path writeTextFile(Path path, Consumer writer) { writer.accept(printWriter); return path; } catch (IOException e) { - LOG.error("Filed to write file", e); + LOG.error("Failed to write file", e); throw new InternalServerErrorException(); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java index 2f8fd35a0d..267d547c8e 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -1,11 +1,6 @@ package org.ohdsi.webapi.shiny; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import org.ohdsi.webapi.shiny.ApplicationBrief; -import org.ohdsi.webapi.shiny.PackagingStrategy; -import org.ohdsi.webapi.shiny.TemporaryFile; - -import java.nio.file.Path; public interface ShinyPackagingService { CommonAnalysisType getType(); diff --git a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java index e3182976a0..091211cd6a 100644 --- a/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java +++ b/src/main/java/org/ohdsi/webapi/util/TempFileUtils.java @@ -2,9 +2,12 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.springframework.core.NestedIOException; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.function.Function; @@ -15,7 +18,10 @@ public static File copyResourceToTempFile(String resource, String prefix, String File tempFile = File.createTempFile(prefix, suffix); try(InputStream in = TempFileUtils.class.getResourceAsStream(resource)) { - try(OutputStream out = new FileOutputStream(tempFile)) { + try(OutputStream out = Files.newOutputStream(tempFile.toPath())) { + if(in == null) { + throw new IOException("File not found: " + resource); + } IOUtils.copy(in, out); } } diff --git a/src/main/resources/shiny/cc-header-field-mapping.csv b/src/main/resources/shiny/cc-header-field-mapping.csv new file mode 100644 index 0000000000..a91f645f91 --- /dev/null +++ b/src/main/resources/shiny/cc-header-field-mapping.csv @@ -0,0 +1,31 @@ +Analysis ID,analysisId +Analysis name,analysisName +Strata ID,strataId +Strata name,strataName +Cohort ID,cohortId +Cohort name,cohortName +Covariate ID,covariateId +Covariate name,covariateName +Covariate short name,covariateShortName +Count,count +Percent,pct +Value field, +Missing Means Zero,missingMeansZero +Avg,avg +StdDev,stdDev +Min,min +P10,p10 +P25,p25 +Median,median +P75,p75 +P90,p90 +Max,max +Target cohort ID,targetCohortId +Target cohort name,targetCohortName +Comparator cohort ID,comparatorCohortId +Comparator cohort name,comparatorCohortName +Target count,targetCount +Target percent,targetPct +Comparator count,comparatorCount +Comparator percent,comparatorPct +Std. Diff Of Mean,diff From 88c417c97e5ea21470aa9bb0884b75235189f4ee Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Thu, 27 Jun 2024 19:49:23 +0200 Subject: [PATCH 16/32] Added SourceKey to Shiny apps Titles --- .../shiny/CohortCharacterizationShinyPackagingService.java | 2 +- .../ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java | 2 +- .../ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index a8e811ea0e..c5d299d4dc 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -207,7 +207,7 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); ApplicationBrief applicationBrief = new ApplicationBrief(); applicationBrief.setName(MessageFormat.format("cohort_characterization_analysis_{0}_{1}", generationId, sourceKey)); - applicationBrief.setTitle(cohortCharacterizationEntity.getName()); + applicationBrief.setTitle(String.format("%s (%s)", cohortCharacterizationEntity.getName(), sourceKey)); applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); return applicationBrief; } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index 44d7e09c22..8cf3632d62 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -89,7 +89,7 @@ public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); ApplicationBrief brief = new ApplicationBrief(); brief.setName(MessageFormat.format("cohort_{0}_{1}", cohort.getId(), sourceKey)); - brief.setTitle(cohort.getName()); + brief.setTitle(String.format("%s (%s)", cohort.getName(), sourceKey)); brief.setDescription(cohort.getDescription()); return brief; } diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index e5ee164202..09df4a96f1 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -131,7 +131,7 @@ public ApplicationBrief getBrief(Integer analysisId, String sourceKey) { IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(analysisId); ApplicationBrief applicationBrief = new ApplicationBrief(); applicationBrief.setName(MessageFormat.format("incidence_rates_analysis_{0}_{1}", analysisId, sourceKey)); - applicationBrief.setTitle(analysis.getName()); + applicationBrief.setTitle(String.format("%s (%s)", analysis.getName(), sourceKey)); applicationBrief.setDescription(analysis.getDescription()); return applicationBrief; } From a763994f164999c2f5a87b36d3627705fdd9ea03 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Wed, 26 Jun 2024 13:12:04 +0200 Subject: [PATCH 17/32] [ATL-50] Introduced Cohort Pathway Shiny app download and publishing --- pom.xml | 16 + src/main/assembly/shiny-cohortPathways.xml | 18 + .../ohdsi/webapi/pathway/PathwayService.java | 4 + .../webapi/pathway/PathwayServiceImpl.java | 1217 +++++++++-------- .../CohortPathwaysShinyPackagingService.java | 87 ++ .../org/ohdsi/webapi/shiny/FileWriter.java | 2 +- .../webapi/pathway/PathwayServiceTest.java | 46 + ...hortPathwaysShinyPackagingServiceTest.java | 72 + ...cidenceRatesShinyPackagingServiceTest.java | 2 +- 9 files changed, 858 insertions(+), 606 deletions(-) create mode 100644 src/main/assembly/shiny-cohortPathways.xml create mode 100644 src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java create mode 100644 src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java create mode 100644 src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java diff --git a/pom.xml b/pom.xml index 78ae96706d..06aae0fb0f 100644 --- a/pom.xml +++ b/pom.xml @@ -1956,6 +1956,22 @@ shiny-cohortCharacterizations + + build-cohortPathways-archive + + single + + generate-resources + + false + ${shiny.app.directory} + ${shiny.output.directory} + + src/main/assembly/shiny-cohortPathways.xml + + shiny-cohortPathways + + diff --git a/src/main/assembly/shiny-cohortPathways.xml b/src/main/assembly/shiny-cohortPathways.xml new file mode 100644 index 0000000000..3dc1e0fba2 --- /dev/null +++ b/src/main/assembly/shiny-cohortPathways.xml @@ -0,0 +1,18 @@ + + + shiny-incidenceRates + + zip + + false + + + ./apps/cohortPathways + + data/** + + / + + + \ No newline at end of file diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java index dcbd7785a4..3b465c593b 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java @@ -14,6 +14,7 @@ import org.ohdsi.webapi.versioning.dto.VersionUpdateDTO; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; +import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.Map; @@ -69,4 +70,7 @@ public interface PathwayService extends HasTags { PathwayVersion saveVersion(int id); List listByTags(TagNameListRequestDTO requestDTO); + + @Transactional + PathwayAnalysisDTO getByGenerationId(Integer id); } diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java index dd4e62987f..230811525e 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java @@ -30,6 +30,7 @@ import org.ohdsi.webapi.pathway.dto.internal.PathwayCode; import org.ohdsi.webapi.pathway.repository.PathwayAnalysisEntityRepository; import org.ohdsi.webapi.pathway.repository.PathwayAnalysisGenerationRepository; +import org.ohdsi.webapi.security.PermissionService; import org.ohdsi.webapi.service.AbstractDaoService; import org.ohdsi.webapi.service.CohortDefinitionService; import org.ohdsi.webapi.service.JobService; @@ -63,9 +64,11 @@ import org.springframework.batch.core.job.builder.SimpleJobBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.convert.ConversionService; import org.springframework.core.convert.support.GenericConversionService; import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; @@ -96,613 +99,619 @@ import static org.ohdsi.webapi.Constants.Params.JOB_NAME; import static org.ohdsi.webapi.Constants.Params.PATHWAY_ANALYSIS_ID; import static org.ohdsi.webapi.Constants.Params.SOURCE_ID; -import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; -import org.ohdsi.webapi.security.PermissionService; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.PageImpl; @Service @Transactional public class PathwayServiceImpl extends AbstractDaoService implements PathwayService, GeneratesNotification { - private final PathwayAnalysisEntityRepository pathwayAnalysisRepository; - private final PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; - private final SourceService sourceService; - private final JobTemplate jobTemplate; - private final EntityManager entityManager; - private final DesignImportService designImportService; - private final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository; - private final UserRepository userRepository; - private final GenerationUtils generationUtils; - private final JobService jobService; - private final GenericConversionService genericConversionService; - private final StepBuilderFactory stepBuilderFactory; - private final CohortDefinitionService cohortDefinitionService; - private final VersionService versionService; - - private PermissionService permissionService; - - @Value("${security.defaultGlobalReadPermissions}") - private boolean defaultGlobalReadPermissions; - - private final List STEP_COLUMNS = Arrays.asList(new String[]{"step_1", "step_2", "step_3", "step_4", "step_5", "step_6", "step_7", "step_8", "step_9", "step_10"}); - - private final EntityGraph defaultEntityGraph = EntityUtils.fromAttributePaths( - "targetCohorts.cohortDefinition", - "eventCohorts.cohortDefinition", - "createdBy", - "modifiedBy" - ); - - @Autowired - public PathwayServiceImpl( - PathwayAnalysisEntityRepository pathwayAnalysisRepository, - PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository, - SourceService sourceService, - ConversionService conversionService, - JobTemplate jobTemplate, - EntityManager entityManager, - Security security, - DesignImportService designImportService, - AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository, - UserRepository userRepository, - GenerationUtils generationUtils, - JobService jobService, - @Qualifier("conversionService") GenericConversionService genericConversionService, - StepBuilderFactory stepBuilderFactory, - CohortDefinitionService cohortDefinitionService, - VersionService versionService, - PermissionService permissionService) { - - this.pathwayAnalysisRepository = pathwayAnalysisRepository; - this.pathwayAnalysisGenerationRepository = pathwayAnalysisGenerationRepository; - this.sourceService = sourceService; - this.jobTemplate = jobTemplate; - this.entityManager = entityManager; - this.jobService = jobService; - this.genericConversionService = genericConversionService; - this.security = security; - this.designImportService = designImportService; - this.analysisGenerationInfoEntityRepository = analysisGenerationInfoEntityRepository; - this.userRepository = userRepository; - this.generationUtils = generationUtils; - this.stepBuilderFactory = stepBuilderFactory; - this.cohortDefinitionService = cohortDefinitionService; - this.versionService = versionService; - this.permissionService = permissionService; - - SerializedPathwayAnalysisToPathwayAnalysisConverter.setConversionService(conversionService); - } - - @Override - public PathwayAnalysisEntity create(PathwayAnalysisEntity toSave) { - - PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); - - copyProps(toSave, newAnalysis); - - toSave.getTargetCohorts().forEach(tc -> { - tc.setId(null); - tc.setPathwayAnalysis(newAnalysis); - newAnalysis.getTargetCohorts().add(tc); - }); - - toSave.getEventCohorts().forEach(ec -> { - ec.setId(null); - ec.setPathwayAnalysis(newAnalysis); - newAnalysis.getEventCohorts().add(ec); - }); - - newAnalysis.setCreatedBy(getCurrentUser()); - newAnalysis.setCreatedDate(new Date()); - // Fields with information about modifications have to be reseted - newAnalysis.setModifiedBy(null); - newAnalysis.setModifiedDate(null); - return save(newAnalysis); - } - - @Override - public PathwayAnalysisEntity importAnalysis(PathwayAnalysisEntity toImport) { - - PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); - - copyProps(toImport, newAnalysis); - - Stream.concat(toImport.getTargetCohorts().stream(), toImport.getEventCohorts().stream()).forEach(pc -> { - CohortDefinition cohortDefinition = designImportService.persistCohortOrGetExisting(pc.getCohortDefinition()); - pc.setId(null); - pc.setName(cohortDefinition.getName()); - pc.setCohortDefinition(cohortDefinition); - pc.setPathwayAnalysis(newAnalysis); - if (pc instanceof PathwayTargetCohort) { - newAnalysis.getTargetCohorts().add((PathwayTargetCohort) pc); - } else { - newAnalysis.getEventCohorts().add((PathwayEventCohort) pc); - } - }); - - newAnalysis.setCreatedBy(getCurrentUser()); - newAnalysis.setCreatedDate(new Date()); - - return save(newAnalysis); - } - - @Override - public Page getPage(final Pageable pageable) { - List pathwayList = pathwayAnalysisRepository.findAll(defaultEntityGraph) - .stream().filter(!defaultGlobalReadPermissions ? entity -> permissionService.hasReadAccess(entity) : entity -> true) - .collect(Collectors.toList()); - return getPageFromResults(pageable, pathwayList); - } - - private Page getPageFromResults(Pageable pageable, List results) { - // Calculate the start and end indices for the current page - int startIndex = pageable.getPageNumber() * pageable.getPageSize(); - int endIndex = Math.min(startIndex + pageable.getPageSize(), results.size()); - - return new PageImpl<>(results.subList(startIndex, endIndex), pageable, results.size()); - } - - @Override - public int getCountPAWithSameName(Integer id, String name) { - - return pathwayAnalysisRepository.getCountPAWithSameName(id, name); - } - - @Override - public PathwayAnalysisEntity getById(Integer id) { - - PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(id, defaultEntityGraph); - if (Objects.nonNull(entity)) { - entity.getTargetCohorts().forEach(tc -> Hibernate.initialize(tc.getCohortDefinition().getDetails())); - entity.getEventCohorts().forEach(ec -> Hibernate.initialize(ec.getCohortDefinition().getDetails())); - } - return entity; - } - - private List getNamesLike(String name) { - - return pathwayAnalysisRepository.findAllByNameStartsWith(name).stream().map(PathwayAnalysisEntity::getName).collect(Collectors.toList()); - } - - @Override - public String getNameForCopy(String dtoName) { - return NameUtils.getNameForCopy(dtoName, this::getNamesLike, pathwayAnalysisRepository.findByName(dtoName)); - } - - @Override - public String getNameWithSuffix(String dtoName) { - return NameUtils.getNameWithSuffix(dtoName, this::getNamesLike); - } - - @Override - public PathwayAnalysisEntity update(PathwayAnalysisEntity forUpdate) { - - PathwayAnalysisEntity existing = getById(forUpdate.getId()); - - copyProps(forUpdate, existing); - updateCohorts(existing, existing.getTargetCohorts(), forUpdate.getTargetCohorts()); - updateCohorts(existing, existing.getEventCohorts(), forUpdate.getEventCohorts()); - - existing.setModifiedBy(getCurrentUser()); - existing.setModifiedDate(new Date()); - - return save(existing); - } - - private void updateCohorts(PathwayAnalysisEntity analysis, Set existing, Set forUpdate) { - - Set removedCohorts = existing - .stream() - .filter(ec -> !forUpdate.contains(ec)) - .collect(Collectors.toSet()); - existing.removeAll(removedCohorts); - forUpdate.forEach(updatedCohort -> existing.stream() - .filter(ec -> ec.equals(updatedCohort)) - .findFirst() - .map(ec -> { - ec.setName(updatedCohort.getName()); - return ec; - }) - .orElseGet(() -> { - updatedCohort.setId(null); - updatedCohort.setPathwayAnalysis(analysis); - existing.add(updatedCohort); - return updatedCohort; - })); - } - - @Override - public void delete(Integer id) { - - pathwayAnalysisRepository.delete(id); - } - - @Override - public Map getEventCohortCodes(PathwayAnalysisEntity pathwayAnalysis) { - - Integer index = 0; - - List sortedEventCohortsCopy = pathwayAnalysis.getEventCohorts() - .stream() - .sorted(Comparator.comparing(PathwayEventCohort::getName)) - .collect(Collectors.toList()); - - Map cohortDefIdToIndexMap = new HashMap<>(); - - for (PathwayEventCohort eventCohort : sortedEventCohortsCopy) { - cohortDefIdToIndexMap.put(eventCohort.getCohortDefinition().getId(), index++); - } - - return cohortDefIdToIndexMap; - } - - @Override - @DataSourceAccess - public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, @SourceId Integer sourceId, String cohortTable, String sessionId) { - - Map eventCohortCodes = getEventCohortCodes(pathwayAnalysis); - Source source = sourceService.findBySourceId(sourceId); - final StringJoiner joiner = new StringJoiner("\n\n"); - - String analysisSql = ResourceHelper.GetResourceAsString("/resources/pathway/runPathwayAnalysis.sql"); - String eventCohortInputSql = ResourceHelper.GetResourceAsString("/resources/pathway/eventCohortInput.sql"); - - String tempTableQualifier = SourceUtils.getTempQualifier(source); - String resultsTableQualifier = SourceUtils.getResultsQualifier(source); - - String eventCohortIdIndexSql = eventCohortCodes.entrySet() - .stream() - .map(ec -> { - String[] params = new String[]{"cohort_definition_id", "event_cohort_index"}; - String[] values = new String[]{ec.getKey().toString(), ec.getValue().toString()}; - return SqlRender.renderSql(eventCohortInputSql, params, values); - }) - .collect(Collectors.joining(" UNION ALL ")); - - pathwayAnalysis.getTargetCohorts().forEach(tc -> { - - String[] params = new String[]{ - GENERATION_ID, - "event_cohort_id_index_map", - "temp_database_schema", - "target_database_schema", - "target_cohort_table", - "pathway_target_cohort_id", - "max_depth", - "combo_window", - "allow_repeats", - "isHive" - }; - String[] values = new String[]{ - generationId.toString(), - eventCohortIdIndexSql, - tempTableQualifier, - resultsTableQualifier, - cohortTable, - tc.getCohortDefinition().getId().toString(), - pathwayAnalysis.getMaxDepth().toString(), - MoreObjects.firstNonNull(pathwayAnalysis.getCombinationWindow(), 1).toString(), - String.valueOf(pathwayAnalysis.isAllowRepeats()), - String.valueOf(Objects.equals(DBMSType.HIVE.getOhdsiDB(), source.getSourceDialect())) - }; - - String renderedSql = SqlRender.renderSql(analysisSql, params, values); - String translatedSql = SqlTranslate.translateSql(renderedSql, source.getSourceDialect(), sessionId, SourceUtils.getTempQualifier(source)); - - joiner.add(translatedSql); - }); - - return joiner.toString(); - } - - @Override - public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, Integer sourceId) { - - return buildAnalysisSql(generationId, pathwayAnalysis, sourceId, "cohort", SessionUtils.sessionId()); - } - - @Override - @DataSourceAccess - public JobExecutionResource generatePathways(final Integer pathwayAnalysisId, final @SourceId Integer sourceId) { - - PathwayService pathwayService = this; - - PathwayAnalysisEntity pathwayAnalysis = getById(pathwayAnalysisId); - Source source = getSourceRepository().findBySourceId(sourceId); - - JobParametersBuilder builder = new JobParametersBuilder(); - builder.addString(JOB_NAME, String.format("Generating Pathway Analysis %d using %s (%s)", pathwayAnalysisId, source.getSourceName(), source.getSourceKey())); - builder.addString(SOURCE_ID, String.valueOf(source.getSourceId())); - builder.addString(PATHWAY_ANALYSIS_ID, pathwayAnalysis.getId().toString()); - builder.addString(JOB_AUTHOR, getCurrentUserLogin()); - - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - - SimpleJobBuilder generateAnalysisJob = generationUtils.buildJobForCohortBasedAnalysisTasklet( - GENERATE_PATHWAY_ANALYSIS, - source, - builder, - jdbcTemplate, - chunkContext -> { - Integer analysisId = Integer.valueOf(chunkContext.getStepContext().getJobParameters().get(PATHWAY_ANALYSIS_ID).toString()); - PathwayAnalysisEntity analysis = pathwayService.getById(analysisId); - return Stream.concat(analysis.getTargetCohorts().stream(), analysis.getEventCohorts().stream()) - .map(PathwayCohort::getCohortDefinition) - .collect(Collectors.toList()); - }, - new GeneratePathwayAnalysisTasklet( - getSourceJdbcTemplate(source), - getTransactionTemplate(), - pathwayService, - analysisGenerationInfoEntityRepository, - userRepository, - sourceService - ) - ); - TransactionalTasklet statisticsTasklet = new PathwayStatisticsTasklet(getSourceJdbcTemplate(source), getTransactionTemplate(), source, this, genericConversionService); - Step generateStatistics = stepBuilderFactory.get(GENERATE_PATHWAY_ANALYSIS + ".generateStatistics") - .tasklet(statisticsTasklet) - .build(); - - generateAnalysisJob.next(generateStatistics); - - final JobParameters jobParameters = builder.toJobParameters(); - - return jobService.runJob(generateAnalysisJob.build(), jobParameters); - } - - @Override - @DataSourceAccess - public void cancelGeneration(Integer pathwayAnalysisId, @SourceId Integer sourceId) { - - PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(pathwayAnalysisId, defaultEntityGraph); - String sourceKey = getSourceRepository().findBySourceId(sourceId).getSourceKey(); - entity.getTargetCohorts().forEach(tc -> cohortDefinitionService.cancelGenerateCohort(tc.getId(), sourceKey)); - entity.getEventCohorts().forEach(ec -> cohortDefinitionService.cancelGenerateCohort(ec.getId(), sourceKey)); - jobService.cancelJobExecution(j -> { - JobParameters jobParameters = j.getJobParameters(); - String jobName = j.getJobInstance().getJobName(); - return Objects.equals(jobParameters.getString(PATHWAY_ANALYSIS_ID), Integer.toString(pathwayAnalysisId)) - && Objects.equals(jobParameters.getString(SOURCE_ID), String.valueOf(sourceId)) - && Objects.equals(GENERATE_PATHWAY_ANALYSIS, jobName); - }); - } - - @Override - public List getPathwayGenerations(final Integer pathwayAnalysisId) { - - return pathwayAnalysisGenerationRepository.findAllByPathwayAnalysisId(pathwayAnalysisId, EntityUtils.fromAttributePaths("source")); - } - - @Override - public PathwayAnalysisGenerationEntity getGeneration(Long generationId) { - - return pathwayAnalysisGenerationRepository.findOne(generationId, EntityUtils.fromAttributePaths("source")); - } - - @Override - @DataSourceAccess - public PathwayAnalysisResult getResultingPathways(final @PathwayAnalysisGenerationId Long generationId) { - - PathwayAnalysisGenerationEntity generation = getGeneration(generationId); - Source source = generation.getSource(); - return queryGenerationResults(source, generationId); - } - - private final RowMapper codeRowMapper = (final ResultSet resultSet, final int arg1) -> { - return new PathwayCode(resultSet.getLong("code"), resultSet.getString("name"), resultSet.getInt("is_combo") != 0); - }; - - private final RowMapper pathwayStatsRowMapper = (final ResultSet rs, final int arg1) -> { - CohortPathways cp = new CohortPathways(); - cp.setCohortId(rs.getInt("target_cohort_id")); - cp.setTargetCohortCount(rs.getInt("target_cohort_count")); - cp.setTotalPathwaysCount(rs.getInt("pathways_count")); - return cp; - }; - - private final ResultSetExtractor>> pathwayExtractor = (final ResultSet rs) -> { - Map> cohortMap = new HashMap<>(); // maps a cohortId to a list of pathways (which is stored as a Map - - while (rs.next()) { - int cohortId = rs.getInt("target_cohort_id"); - if (!cohortMap.containsKey(cohortId)) { - cohortMap.put(cohortId, new HashMap<>()); - } - Map pathList = cohortMap.get(cohortId); - - // build path - List path = new ArrayList<>(); - for (String stepCol : STEP_COLUMNS) { - String step = rs.getString(stepCol); - - if (step == null) break; // cancel for-loop when we encounter a column with a null value - - path.add(step); - } - pathList.put(StringUtils.join(path, "-"), rs.getInt("count_value")); // for a given cohort, a path must be unique, so no need to check - } - return cohortMap; - }; - - @Override - @DataSourceAccess - public String findDesignByGenerationId(@PathwayAnalysisGenerationId final Long id) { - final AnalysisGenerationInfoEntity entity = analysisGenerationInfoEntityRepository.findById(id) - .orElseThrow(() -> new IllegalArgumentException("Analysis with id: " + id + " cannot be found")); - return entity.getDesign(); - } - - @Override - public void assignTag(Integer id, int tagId) { - PathwayAnalysisEntity entity = getById(id); - checkOwnerOrAdminOrGranted(entity); - assignTag(entity, tagId); - } - - @Override - public void unassignTag(Integer id, int tagId) { - PathwayAnalysisEntity entity = getById(id); - checkOwnerOrAdminOrGranted(entity); - unassignTag(entity, tagId); - } - - @Override - public List getVersions(long id) { - List versions = versionService.getVersions(VersionType.PATHWAY, id); - return versions.stream() - .map(v -> genericConversionService.convert(v, VersionDTO.class)) - .collect(Collectors.toList()); - } - - @Override - public PathwayVersionFullDTO getVersion(int id, int version) { - checkVersion(id, version, false); - PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - return genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); - } - - @Override - public VersionDTO updateVersion(int id, int version, VersionUpdateDTO updateDTO) { - checkVersion(id, version); - updateDTO.setAssetId(id); - updateDTO.setVersion(version); - PathwayVersion updated = versionService.update(VersionType.PATHWAY, updateDTO); - - return genericConversionService.convert(updated, VersionDTO.class); - } - - @Override - public void deleteVersion(int id, int version) { - checkVersion(id, version); - versionService.delete(VersionType.PATHWAY, id, version); - } - - @Override - public PathwayAnalysisDTO copyAssetFromVersion(int id, int version) { - checkVersion(id, version, false); - PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - PathwayVersionFullDTO fullDTO = genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); - - PathwayAnalysisDTO dto = fullDTO.getEntityDTO(); - dto.setId(null); - dto.setTags(null); - dto.setName(NameUtils.getNameForCopy(dto.getName(), this::getNamesLike, - pathwayAnalysisRepository.findByName(dto.getName()))); - PathwayAnalysisEntity pathwayAnalysis = genericConversionService.convert(dto, PathwayAnalysisEntity.class); - PathwayAnalysisEntity saved = create(pathwayAnalysis); - return genericConversionService.convert(saved, PathwayAnalysisDTO.class); - } - - @Override - public List listByTags(TagNameListRequestDTO requestDTO) { - List names = requestDTO.getNames().stream() - .map(name -> name.toLowerCase(Locale.ROOT)) - .collect(Collectors.toList()); - List entities = pathwayAnalysisRepository.findByTags(names); - return listByTags(entities, names, PathwayAnalysisDTO.class); - } - - private void checkVersion(int id, int version) { - checkVersion(id, version, true); - } - - private void checkVersion(int id, int version, boolean checkOwnerShip) { - Version pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayVersion, - String.format("There is no pathway analysis version with id = %d.", version)); - - PathwayAnalysisEntity entity = this.pathwayAnalysisRepository.findOne(id); - if (checkOwnerShip) { - checkOwnerOrAdminOrGranted(entity); - } - } - - public PathwayVersion saveVersion(int id) { - PathwayAnalysisEntity def = this.pathwayAnalysisRepository.findOne(id); - PathwayVersion version = genericConversionService.convert(def, PathwayVersion.class); - - UserEntity user = Objects.nonNull(def.getModifiedBy()) ? def.getModifiedBy() : def.getCreatedBy(); - Date versionDate = Objects.nonNull(def.getModifiedDate()) ? def.getModifiedDate() : def.getCreatedDate(); - version.setCreatedBy(user); - version.setCreatedDate(versionDate); - return versionService.create(VersionType.PATHWAY, version); - } - - private PathwayAnalysisResult queryGenerationResults(Source source, Long generationId) { - - // load code lookup - PreparedStatementRenderer pathwayCodesPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayCodeLookup.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - List pathwayCodes = getSourceJdbcTemplate(source).query(pathwayCodesPsr.getSql(), pathwayCodesPsr.getOrderedParams(), codeRowMapper); - - // fetch cohort stats, paths will be populated after - PreparedStatementRenderer pathwayStatsPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayStats.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - List cohortStats = getSourceJdbcTemplate(source).query(pathwayStatsPsr.getSql(), pathwayStatsPsr.getOrderedParams(), pathwayStatsRowMapper); - - // load cohort paths, and assign back to cohortStats - PreparedStatementRenderer pathwayResultsPsr = new PreparedStatementRenderer( - source, "/resources/pathway/getPathwayResults.sql", "target_database_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - new String[]{GENERATION_ID}, - new Object[]{generationId} - ); - Map> pathwayResults = - getSourceJdbcTemplate(source).query(pathwayResultsPsr.getSql(), pathwayResultsPsr.getOrderedParams(), pathwayExtractor); - - cohortStats.stream().forEach((cp) -> { - cp.setPathwaysCounts(pathwayResults.get(cp.getCohortId())); - }); - - PathwayAnalysisResult result = new PathwayAnalysisResult(); - result.setCodes(new HashSet<>(pathwayCodes)); - result.setCohortPathwaysList(new HashSet<>(cohortStats)); - - return result; - } - - private void copyProps(PathwayAnalysisEntity from, PathwayAnalysisEntity to) { - - to.setName(from.getName()); - to.setDescription(from.getDescription()); - to.setMaxDepth(from.getMaxDepth()); - to.setMinCellCount(from.getMinCellCount()); - to.setCombinationWindow(from.getCombinationWindow()); - to.setAllowRepeats(from.isAllowRepeats()); - } - - private int getAnalysisHashCode(PathwayAnalysisEntity pathwayAnalysis) { - - SerializedPathwayAnalysisToPathwayAnalysisConverter designConverter = new SerializedPathwayAnalysisToPathwayAnalysisConverter(); - return designConverter.convertToDatabaseColumn(pathwayAnalysis).hashCode(); - } - - private PathwayAnalysisEntity save(PathwayAnalysisEntity pathwayAnalysis) { - - pathwayAnalysis = pathwayAnalysisRepository.saveAndFlush(pathwayAnalysis); - entityManager.refresh(pathwayAnalysis); - pathwayAnalysis = getById(pathwayAnalysis.getId()); - pathwayAnalysis.setHashCode(getAnalysisHashCode(pathwayAnalysis)); - return pathwayAnalysis; - } - - @Override - public String getJobName() { - return GENERATE_PATHWAY_ANALYSIS; - } - - @Override - public String getExecutionFoldingKey() { - return PATHWAY_ANALYSIS_ID; - } + private final PathwayAnalysisEntityRepository pathwayAnalysisRepository; + private final PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; + private final SourceService sourceService; + private final JobTemplate jobTemplate; + private final EntityManager entityManager; + private final DesignImportService designImportService; + private final AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository; + private final UserRepository userRepository; + private final GenerationUtils generationUtils; + private final JobService jobService; + private final GenericConversionService genericConversionService; + private final StepBuilderFactory stepBuilderFactory; + private final CohortDefinitionService cohortDefinitionService; + private final VersionService versionService; + + private PermissionService permissionService; + + @Value("${security.defaultGlobalReadPermissions}") + private boolean defaultGlobalReadPermissions; + + private final List STEP_COLUMNS = Arrays.asList(new String[]{"step_1", "step_2", "step_3", "step_4", "step_5", "step_6", "step_7", "step_8", "step_9", "step_10"}); + + private final EntityGraph defaultEntityGraph = EntityUtils.fromAttributePaths( + "targetCohorts.cohortDefinition", + "eventCohorts.cohortDefinition", + "createdBy", + "modifiedBy" + ); + + @Autowired + public PathwayServiceImpl( + PathwayAnalysisEntityRepository pathwayAnalysisRepository, + PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository, + SourceService sourceService, + ConversionService conversionService, + JobTemplate jobTemplate, + EntityManager entityManager, + Security security, + DesignImportService designImportService, + AnalysisGenerationInfoEntityRepository analysisGenerationInfoEntityRepository, + UserRepository userRepository, + GenerationUtils generationUtils, + JobService jobService, + @Qualifier("conversionService") GenericConversionService genericConversionService, + StepBuilderFactory stepBuilderFactory, + CohortDefinitionService cohortDefinitionService, + VersionService versionService, + PermissionService permissionService) { + + this.pathwayAnalysisRepository = pathwayAnalysisRepository; + this.pathwayAnalysisGenerationRepository = pathwayAnalysisGenerationRepository; + this.sourceService = sourceService; + this.jobTemplate = jobTemplate; + this.entityManager = entityManager; + this.jobService = jobService; + this.genericConversionService = genericConversionService; + this.security = security; + this.designImportService = designImportService; + this.analysisGenerationInfoEntityRepository = analysisGenerationInfoEntityRepository; + this.userRepository = userRepository; + this.generationUtils = generationUtils; + this.stepBuilderFactory = stepBuilderFactory; + this.cohortDefinitionService = cohortDefinitionService; + this.versionService = versionService; + this.permissionService = permissionService; + + SerializedPathwayAnalysisToPathwayAnalysisConverter.setConversionService(conversionService); + } + + @Override + public PathwayAnalysisEntity create(PathwayAnalysisEntity toSave) { + + PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); + + copyProps(toSave, newAnalysis); + + toSave.getTargetCohorts().forEach(tc -> { + tc.setId(null); + tc.setPathwayAnalysis(newAnalysis); + newAnalysis.getTargetCohorts().add(tc); + }); + + toSave.getEventCohorts().forEach(ec -> { + ec.setId(null); + ec.setPathwayAnalysis(newAnalysis); + newAnalysis.getEventCohorts().add(ec); + }); + + newAnalysis.setCreatedBy(getCurrentUser()); + newAnalysis.setCreatedDate(new Date()); + // Fields with information about modifications have to be reseted + newAnalysis.setModifiedBy(null); + newAnalysis.setModifiedDate(null); + return save(newAnalysis); + } + + @Override + public PathwayAnalysisEntity importAnalysis(PathwayAnalysisEntity toImport) { + + PathwayAnalysisEntity newAnalysis = new PathwayAnalysisEntity(); + + copyProps(toImport, newAnalysis); + + Stream.concat(toImport.getTargetCohorts().stream(), toImport.getEventCohorts().stream()).forEach(pc -> { + CohortDefinition cohortDefinition = designImportService.persistCohortOrGetExisting(pc.getCohortDefinition()); + pc.setId(null); + pc.setName(cohortDefinition.getName()); + pc.setCohortDefinition(cohortDefinition); + pc.setPathwayAnalysis(newAnalysis); + if (pc instanceof PathwayTargetCohort) { + newAnalysis.getTargetCohorts().add((PathwayTargetCohort) pc); + } else { + newAnalysis.getEventCohorts().add((PathwayEventCohort) pc); + } + }); + + newAnalysis.setCreatedBy(getCurrentUser()); + newAnalysis.setCreatedDate(new Date()); + + return save(newAnalysis); + } + + @Override + public Page getPage(final Pageable pageable) { + List pathwayList = pathwayAnalysisRepository.findAll(defaultEntityGraph) + .stream().filter(!defaultGlobalReadPermissions ? entity -> permissionService.hasReadAccess(entity) : entity -> true) + .collect(Collectors.toList()); + return getPageFromResults(pageable, pathwayList); + } + + private Page getPageFromResults(Pageable pageable, List results) { + // Calculate the start and end indices for the current page + int startIndex = pageable.getPageNumber() * pageable.getPageSize(); + int endIndex = Math.min(startIndex + pageable.getPageSize(), results.size()); + + return new PageImpl<>(results.subList(startIndex, endIndex), pageable, results.size()); + } + + @Override + public int getCountPAWithSameName(Integer id, String name) { + + return pathwayAnalysisRepository.getCountPAWithSameName(id, name); + } + + @Override + public PathwayAnalysisEntity getById(Integer id) { + + PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(id, defaultEntityGraph); + if (Objects.nonNull(entity)) { + entity.getTargetCohorts().forEach(tc -> Hibernate.initialize(tc.getCohortDefinition().getDetails())); + entity.getEventCohorts().forEach(ec -> Hibernate.initialize(ec.getCohortDefinition().getDetails())); + } + return entity; + } + + private List getNamesLike(String name) { + + return pathwayAnalysisRepository.findAllByNameStartsWith(name).stream().map(PathwayAnalysisEntity::getName).collect(Collectors.toList()); + } + + @Override + public String getNameForCopy(String dtoName) { + return NameUtils.getNameForCopy(dtoName, this::getNamesLike, pathwayAnalysisRepository.findByName(dtoName)); + } + + @Override + public String getNameWithSuffix(String dtoName) { + return NameUtils.getNameWithSuffix(dtoName, this::getNamesLike); + } + + @Override + public PathwayAnalysisEntity update(PathwayAnalysisEntity forUpdate) { + + PathwayAnalysisEntity existing = getById(forUpdate.getId()); + + copyProps(forUpdate, existing); + updateCohorts(existing, existing.getTargetCohorts(), forUpdate.getTargetCohorts()); + updateCohorts(existing, existing.getEventCohorts(), forUpdate.getEventCohorts()); + + existing.setModifiedBy(getCurrentUser()); + existing.setModifiedDate(new Date()); + + return save(existing); + } + + private void updateCohorts(PathwayAnalysisEntity analysis, Set existing, Set forUpdate) { + + Set removedCohorts = existing + .stream() + .filter(ec -> !forUpdate.contains(ec)) + .collect(Collectors.toSet()); + existing.removeAll(removedCohorts); + forUpdate.forEach(updatedCohort -> existing.stream() + .filter(ec -> ec.equals(updatedCohort)) + .findFirst() + .map(ec -> { + ec.setName(updatedCohort.getName()); + return ec; + }) + .orElseGet(() -> { + updatedCohort.setId(null); + updatedCohort.setPathwayAnalysis(analysis); + existing.add(updatedCohort); + return updatedCohort; + })); + } + + @Override + public void delete(Integer id) { + + pathwayAnalysisRepository.delete(id); + } + + @Override + public Map getEventCohortCodes(PathwayAnalysisEntity pathwayAnalysis) { + + Integer index = 0; + + List sortedEventCohortsCopy = pathwayAnalysis.getEventCohorts() + .stream() + .sorted(Comparator.comparing(PathwayEventCohort::getName)) + .collect(Collectors.toList()); + + Map cohortDefIdToIndexMap = new HashMap<>(); + + for (PathwayEventCohort eventCohort : sortedEventCohortsCopy) { + cohortDefIdToIndexMap.put(eventCohort.getCohortDefinition().getId(), index++); + } + + return cohortDefIdToIndexMap; + } + + @Override + @DataSourceAccess + public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, @SourceId Integer sourceId, String cohortTable, String sessionId) { + + Map eventCohortCodes = getEventCohortCodes(pathwayAnalysis); + Source source = sourceService.findBySourceId(sourceId); + final StringJoiner joiner = new StringJoiner("\n\n"); + + String analysisSql = ResourceHelper.GetResourceAsString("/resources/pathway/runPathwayAnalysis.sql"); + String eventCohortInputSql = ResourceHelper.GetResourceAsString("/resources/pathway/eventCohortInput.sql"); + + String tempTableQualifier = SourceUtils.getTempQualifier(source); + String resultsTableQualifier = SourceUtils.getResultsQualifier(source); + + String eventCohortIdIndexSql = eventCohortCodes.entrySet() + .stream() + .map(ec -> { + String[] params = new String[]{"cohort_definition_id", "event_cohort_index"}; + String[] values = new String[]{ec.getKey().toString(), ec.getValue().toString()}; + return SqlRender.renderSql(eventCohortInputSql, params, values); + }) + .collect(Collectors.joining(" UNION ALL ")); + + pathwayAnalysis.getTargetCohorts().forEach(tc -> { + + String[] params = new String[]{ + GENERATION_ID, + "event_cohort_id_index_map", + "temp_database_schema", + "target_database_schema", + "target_cohort_table", + "pathway_target_cohort_id", + "max_depth", + "combo_window", + "allow_repeats", + "isHive" + }; + String[] values = new String[]{ + generationId.toString(), + eventCohortIdIndexSql, + tempTableQualifier, + resultsTableQualifier, + cohortTable, + tc.getCohortDefinition().getId().toString(), + pathwayAnalysis.getMaxDepth().toString(), + MoreObjects.firstNonNull(pathwayAnalysis.getCombinationWindow(), 1).toString(), + String.valueOf(pathwayAnalysis.isAllowRepeats()), + String.valueOf(Objects.equals(DBMSType.HIVE.getOhdsiDB(), source.getSourceDialect())) + }; + + String renderedSql = SqlRender.renderSql(analysisSql, params, values); + String translatedSql = SqlTranslate.translateSql(renderedSql, source.getSourceDialect(), sessionId, SourceUtils.getTempQualifier(source)); + + joiner.add(translatedSql); + }); + + return joiner.toString(); + } + + @Override + public String buildAnalysisSql(Long generationId, PathwayAnalysisEntity pathwayAnalysis, Integer sourceId) { + + return buildAnalysisSql(generationId, pathwayAnalysis, sourceId, "cohort", SessionUtils.sessionId()); + } + + @Override + @DataSourceAccess + public JobExecutionResource generatePathways(final Integer pathwayAnalysisId, final @SourceId Integer sourceId) { + + PathwayService pathwayService = this; + + PathwayAnalysisEntity pathwayAnalysis = getById(pathwayAnalysisId); + Source source = getSourceRepository().findBySourceId(sourceId); + + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Generating Pathway Analysis %d using %s (%s)", pathwayAnalysisId, source.getSourceName(), source.getSourceKey())); + builder.addString(SOURCE_ID, String.valueOf(source.getSourceId())); + builder.addString(PATHWAY_ANALYSIS_ID, pathwayAnalysis.getId().toString()); + builder.addString(JOB_AUTHOR, getCurrentUserLogin()); + + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + + SimpleJobBuilder generateAnalysisJob = generationUtils.buildJobForCohortBasedAnalysisTasklet( + GENERATE_PATHWAY_ANALYSIS, + source, + builder, + jdbcTemplate, + chunkContext -> { + Integer analysisId = Integer.valueOf(chunkContext.getStepContext().getJobParameters().get(PATHWAY_ANALYSIS_ID).toString()); + PathwayAnalysisEntity analysis = pathwayService.getById(analysisId); + return Stream.concat(analysis.getTargetCohorts().stream(), analysis.getEventCohorts().stream()) + .map(PathwayCohort::getCohortDefinition) + .collect(Collectors.toList()); + }, + new GeneratePathwayAnalysisTasklet( + getSourceJdbcTemplate(source), + getTransactionTemplate(), + pathwayService, + analysisGenerationInfoEntityRepository, + userRepository, + sourceService + ) + ); + TransactionalTasklet statisticsTasklet = new PathwayStatisticsTasklet(getSourceJdbcTemplate(source), getTransactionTemplate(), source, this, genericConversionService); + Step generateStatistics = stepBuilderFactory.get(GENERATE_PATHWAY_ANALYSIS + ".generateStatistics") + .tasklet(statisticsTasklet) + .build(); + + generateAnalysisJob.next(generateStatistics); + + final JobParameters jobParameters = builder.toJobParameters(); + + return jobService.runJob(generateAnalysisJob.build(), jobParameters); + } + + @Override + @DataSourceAccess + public void cancelGeneration(Integer pathwayAnalysisId, @SourceId Integer sourceId) { + + PathwayAnalysisEntity entity = pathwayAnalysisRepository.findOne(pathwayAnalysisId, defaultEntityGraph); + String sourceKey = getSourceRepository().findBySourceId(sourceId).getSourceKey(); + entity.getTargetCohorts().forEach(tc -> cohortDefinitionService.cancelGenerateCohort(tc.getId(), sourceKey)); + entity.getEventCohorts().forEach(ec -> cohortDefinitionService.cancelGenerateCohort(ec.getId(), sourceKey)); + jobService.cancelJobExecution(j -> { + JobParameters jobParameters = j.getJobParameters(); + String jobName = j.getJobInstance().getJobName(); + return Objects.equals(jobParameters.getString(PATHWAY_ANALYSIS_ID), Integer.toString(pathwayAnalysisId)) + && Objects.equals(jobParameters.getString(SOURCE_ID), String.valueOf(sourceId)) + && Objects.equals(GENERATE_PATHWAY_ANALYSIS, jobName); + }); + } + + @Override + public List getPathwayGenerations(final Integer pathwayAnalysisId) { + return pathwayAnalysisGenerationRepository.findAllByPathwayAnalysisId(pathwayAnalysisId, EntityUtils.fromAttributePaths("source")); + } + + @Override + public PathwayAnalysisGenerationEntity getGeneration(Long generationId) { + return pathwayAnalysisGenerationRepository.findOne(generationId, EntityUtils.fromAttributePaths("source")); + } + + @Override + @DataSourceAccess + public PathwayAnalysisResult getResultingPathways(final @PathwayAnalysisGenerationId Long generationId) { + + PathwayAnalysisGenerationEntity generation = getGeneration(generationId); + Source source = generation.getSource(); + return queryGenerationResults(source, generationId); + } + + private final RowMapper codeRowMapper = (final ResultSet resultSet, final int arg1) -> { + return new PathwayCode(resultSet.getLong("code"), resultSet.getString("name"), resultSet.getInt("is_combo") != 0); + }; + + private final RowMapper pathwayStatsRowMapper = (final ResultSet rs, final int arg1) -> { + CohortPathways cp = new CohortPathways(); + cp.setCohortId(rs.getInt("target_cohort_id")); + cp.setTargetCohortCount(rs.getInt("target_cohort_count")); + cp.setTotalPathwaysCount(rs.getInt("pathways_count")); + return cp; + }; + + private final ResultSetExtractor>> pathwayExtractor = (final ResultSet rs) -> { + Map> cohortMap = new HashMap<>(); // maps a cohortId to a list of pathways (which is stored as a Map + + while (rs.next()) { + int cohortId = rs.getInt("target_cohort_id"); + if (!cohortMap.containsKey(cohortId)) { + cohortMap.put(cohortId, new HashMap<>()); + } + Map pathList = cohortMap.get(cohortId); + + // build path + List path = new ArrayList<>(); + for (String stepCol : STEP_COLUMNS) { + String step = rs.getString(stepCol); + + if (step == null) break; // cancel for-loop when we encounter a column with a null value + + path.add(step); + } + pathList.put(StringUtils.join(path, "-"), rs.getInt("count_value")); // for a given cohort, a path must be unique, so no need to check + } + return cohortMap; + }; + + @Override + @DataSourceAccess + public String findDesignByGenerationId(@PathwayAnalysisGenerationId final Long id) { + final AnalysisGenerationInfoEntity entity = analysisGenerationInfoEntityRepository.findById(id) + .orElseThrow(() -> new IllegalArgumentException("Analysis with id: " + id + " cannot be found")); + return entity.getDesign(); + } + + @Override + public void assignTag(Integer id, int tagId) { + PathwayAnalysisEntity entity = getById(id); + checkOwnerOrAdminOrGranted(entity); + assignTag(entity, tagId); + } + + @Override + public void unassignTag(Integer id, int tagId) { + PathwayAnalysisEntity entity = getById(id); + checkOwnerOrAdminOrGranted(entity); + unassignTag(entity, tagId); + } + + @Override + public List getVersions(long id) { + List versions = versionService.getVersions(VersionType.PATHWAY, id); + return versions.stream() + .map(v -> genericConversionService.convert(v, VersionDTO.class)) + .collect(Collectors.toList()); + } + + @Override + public PathwayVersionFullDTO getVersion(int id, int version) { + checkVersion(id, version, false); + PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + return genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); + } + + @Override + public VersionDTO updateVersion(int id, int version, VersionUpdateDTO updateDTO) { + checkVersion(id, version); + updateDTO.setAssetId(id); + updateDTO.setVersion(version); + PathwayVersion updated = versionService.update(VersionType.PATHWAY, updateDTO); + + return genericConversionService.convert(updated, VersionDTO.class); + } + + @Override + public void deleteVersion(int id, int version) { + checkVersion(id, version); + versionService.delete(VersionType.PATHWAY, id, version); + } + + @Override + public PathwayAnalysisDTO copyAssetFromVersion(int id, int version) { + checkVersion(id, version, false); + PathwayVersion pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + PathwayVersionFullDTO fullDTO = genericConversionService.convert(pathwayVersion, PathwayVersionFullDTO.class); + + PathwayAnalysisDTO dto = fullDTO.getEntityDTO(); + dto.setId(null); + dto.setTags(null); + dto.setName(NameUtils.getNameForCopy(dto.getName(), this::getNamesLike, + pathwayAnalysisRepository.findByName(dto.getName()))); + PathwayAnalysisEntity pathwayAnalysis = genericConversionService.convert(dto, PathwayAnalysisEntity.class); + PathwayAnalysisEntity saved = create(pathwayAnalysis); + return genericConversionService.convert(saved, PathwayAnalysisDTO.class); + } + + @Override + public List listByTags(TagNameListRequestDTO requestDTO) { + List names = requestDTO.getNames().stream() + .map(name -> name.toLowerCase(Locale.ROOT)) + .collect(Collectors.toList()); + List entities = pathwayAnalysisRepository.findByTags(names); + return listByTags(entities, names, PathwayAnalysisDTO.class); + } + + private void checkVersion(int id, int version) { + checkVersion(id, version, true); + } + + private void checkVersion(int id, int version, boolean checkOwnerShip) { + Version pathwayVersion = versionService.getById(VersionType.PATHWAY, id, version); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayVersion, + String.format("There is no pathway analysis version with id = %d.", version)); + + PathwayAnalysisEntity entity = this.pathwayAnalysisRepository.findOne(id); + if (checkOwnerShip) { + checkOwnerOrAdminOrGranted(entity); + } + } + + public PathwayVersion saveVersion(int id) { + PathwayAnalysisEntity def = this.pathwayAnalysisRepository.findOne(id); + PathwayVersion version = genericConversionService.convert(def, PathwayVersion.class); + + UserEntity user = Objects.nonNull(def.getModifiedBy()) ? def.getModifiedBy() : def.getCreatedBy(); + Date versionDate = Objects.nonNull(def.getModifiedDate()) ? def.getModifiedDate() : def.getCreatedDate(); + version.setCreatedBy(user); + version.setCreatedDate(versionDate); + return versionService.create(VersionType.PATHWAY, version); + } + + private PathwayAnalysisResult queryGenerationResults(Source source, Long generationId) { + + // load code lookup + PreparedStatementRenderer pathwayCodesPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayCodeLookup.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + List pathwayCodes = getSourceJdbcTemplate(source).query(pathwayCodesPsr.getSql(), pathwayCodesPsr.getOrderedParams(), codeRowMapper); + + // fetch cohort stats, paths will be populated after + PreparedStatementRenderer pathwayStatsPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayStats.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + List cohortStats = getSourceJdbcTemplate(source).query(pathwayStatsPsr.getSql(), pathwayStatsPsr.getOrderedParams(), pathwayStatsRowMapper); + + // load cohort paths, and assign back to cohortStats + PreparedStatementRenderer pathwayResultsPsr = new PreparedStatementRenderer( + source, "/resources/pathway/getPathwayResults.sql", "target_database_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[]{GENERATION_ID}, + new Object[]{generationId} + ); + Map> pathwayResults = + getSourceJdbcTemplate(source).query(pathwayResultsPsr.getSql(), pathwayResultsPsr.getOrderedParams(), pathwayExtractor); + + cohortStats.stream().forEach((cp) -> { + cp.setPathwaysCounts(pathwayResults.get(cp.getCohortId())); + }); + + PathwayAnalysisResult result = new PathwayAnalysisResult(); + result.setCodes(new HashSet<>(pathwayCodes)); + result.setCohortPathwaysList(new HashSet<>(cohortStats)); + + return result; + } + + private void copyProps(PathwayAnalysisEntity from, PathwayAnalysisEntity to) { + + to.setName(from.getName()); + to.setDescription(from.getDescription()); + to.setMaxDepth(from.getMaxDepth()); + to.setMinCellCount(from.getMinCellCount()); + to.setCombinationWindow(from.getCombinationWindow()); + to.setAllowRepeats(from.isAllowRepeats()); + } + + private int getAnalysisHashCode(PathwayAnalysisEntity pathwayAnalysis) { + + SerializedPathwayAnalysisToPathwayAnalysisConverter designConverter = new SerializedPathwayAnalysisToPathwayAnalysisConverter(); + return designConverter.convertToDatabaseColumn(pathwayAnalysis).hashCode(); + } + + private PathwayAnalysisEntity save(PathwayAnalysisEntity pathwayAnalysis) { + + pathwayAnalysis = pathwayAnalysisRepository.saveAndFlush(pathwayAnalysis); + entityManager.refresh(pathwayAnalysis); + pathwayAnalysis = getById(pathwayAnalysis.getId()); + pathwayAnalysis.setHashCode(getAnalysisHashCode(pathwayAnalysis)); + return pathwayAnalysis; + } + + @Override + public String getJobName() { + return GENERATE_PATHWAY_ANALYSIS; + } + + @Override + public String getExecutionFoldingKey() { + return PATHWAY_ANALYSIS_ID; + } + + + @Override + @Transactional + public PathwayAnalysisDTO getByGenerationId(final Integer id) { + PathwayAnalysisGenerationEntity pathwayAnalysisGenerationEntity = getGeneration(id.longValue()); + PathwayAnalysisEntity pathwayAnalysis = pathwayAnalysisGenerationEntity.getPathwayAnalysis(); + Map eventCodes = getEventCohortCodes(pathwayAnalysis); + PathwayAnalysisDTO dto = genericConversionService.convert(pathwayAnalysis, PathwayAnalysisDTO.class); + dto.getEventCohorts().forEach(ec -> ec.setCode(eventCodes.get(ec.getId()))); + return dto; + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java new file mode 100644 index 0000000000..f74a1719ae --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -0,0 +1,87 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; +import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.util.ExceptionUtils; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.text.MessageFormat; +import java.util.stream.Stream; + +@Service +@ConditionalOnBean(ShinyService.class) +public class CohortPathwaysShinyPackagingService implements ShinyPackagingService { + + private static final Logger log = LoggerFactory.getLogger(CohortPathwaysShinyPackagingService.class); + private static final String SHINY_COHORT_PATHWAYS = "/shiny/shiny-cohortPathways.zip"; + + @Autowired + private PathwayService pathwayService; + @Autowired + private FileWriter fileWriter; + @Autowired + private ManifestUtils manifestUtils; + + @Override + public CommonAnalysisType getType() { + return CommonAnalysisType.COHORT_PATHWAY; + } + + @Override + public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { + return TempFileUtils.doInDirectory(path -> { + PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); + PathwayAnalysisResult pathwayAnalysisResult = pathwayService.getResultingPathways(generationId.longValue()); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysis, String.format("There is no pathway analysis definition with generation id = %d.", generationId)); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysisResult, String.format("There is no pathway analysis result definition with generation id = %d.", generationId)); + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_PATHWAYS, "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = manifestUtils.parseManifest(manifestPath); + + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + Stream.of( + fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysis, "pathwayAnalysis.json"), + fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysisResult, "pathwayAnalysisResult.json"), + fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) + ).forEach(manifestUtils.addDataToManifest(manifest, path)); + fileWriter.writeJsonNodeToFile(manifest, manifestPath); + Path appArchive = packaging.apply(path); + return new TemporaryFile(String.format("CohortPathways_%s_%s.zip", generationId, sourceKey), appArchive); + } catch (IOException e) { + log.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + @Override + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); + ApplicationBrief applicationBrief = new ApplicationBrief(); + applicationBrief.setName(MessageFormat.format("cohort_pathways_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setTitle(String.format("%s (%s)", pathwayAnalysis.getName(), sourceKey)); + applicationBrief.setDescription(pathwayAnalysis.getDescription()); + return applicationBrief; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java index 377fd6cf20..2300f50b4d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java +++ b/src/main/java/org/ohdsi/webapi/shiny/FileWriter.java @@ -38,7 +38,7 @@ public Path writeObjectAsJsonFile(Path parentDir, Object object, String filename } return file; } catch (IOException e) { - LOG.error("Failed to package Cohort Counts Shiny application", e); + LOG.error("Failed to package Shiny application", e); throw new InternalServerErrorException(); } } diff --git a/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java b/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java new file mode 100644 index 0000000000..f2d4a0de79 --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/pathway/PathwayServiceTest.java @@ -0,0 +1,46 @@ +package org.ohdsi.webapi.pathway; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisEntity; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.repository.PathwayAnalysisGenerationRepository; +import org.springframework.core.convert.support.GenericConversionService; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyLong; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class PathwayServiceTest { + + @Mock + private PathwayAnalysisGenerationRepository pathwayAnalysisGenerationRepository; + @Mock + private GenericConversionService genericConversionService; + @Mock + private PathwayAnalysisGenerationEntity pathwayAnalysisGenerationEntity; + @Mock(answer = Answers.RETURNS_DEEP_STUBS) + private PathwayAnalysisDTO pathwayAnalysisDTO; + @Mock + private PathwayAnalysisEntity pathwayAnalysisEntity; + @InjectMocks + private PathwayServiceImpl sut; + + @Test + public void shouldGetByGenerationId() { + when(pathwayAnalysisGenerationRepository.findOne(anyLong(), any())).thenReturn(pathwayAnalysisGenerationEntity); + when(pathwayAnalysisGenerationEntity.getPathwayAnalysis()).thenReturn(pathwayAnalysisEntity); + when(genericConversionService.convert(eq(pathwayAnalysisEntity), eq(PathwayAnalysisDTO.class))).thenReturn(pathwayAnalysisDTO); + PathwayAnalysisDTO result = sut.getByGenerationId(1); + assertEquals(result, pathwayAnalysisDTO); + } + +} diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java new file mode 100644 index 0000000000..c802a08e01 --- /dev/null +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -0,0 +1,72 @@ +package org.ohdsi.webapi.shiny; + +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class CohortPathwaysShinyPackagingServiceTest { + + private static final int GENERATION_ID = 1; + private static final String SOURCE_KEY = "SynPuf110k"; + + @Mock + private PathwayService pathwayService; + @Spy + private ManifestUtils manifestUtils; + @Spy + private FileWriter fileWriter; + + @InjectMocks + private CohortPathwaysShinyPackagingService sut; + + @Test + public void shouldGetBrief() { + when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(createPathwayAnalysisDTO()); + when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); + + ApplicationBrief brief = sut.getBrief(GENERATION_ID, SOURCE_KEY); + assertEquals(brief.getName(), "cohort_pathways_analysis_" + GENERATION_ID + "_" + SOURCE_KEY); + assertEquals(brief.getTitle(), "pathwayAnalysis (SynPuf110k)"); + assertEquals(brief.getDescription(), "desc"); + } + + @Test + public void shouldPackageApp() { + when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(createPathwayAnalysisDTO()); + when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); + PackagingStrategy packagingStrategy = mock(PackagingStrategy.class); + TemporaryFile result = sut.packageApp(GENERATION_ID, SOURCE_KEY, packagingStrategy); + assertNotNull(result); + } + + @Test + public void shouldReturnIncidenceType() { + assertEquals(sut.getType(), CommonAnalysisType.COHORT_PATHWAY); + } + + + private PathwayAnalysisResult createPathwayAnalysisResult() { + return new PathwayAnalysisResult(); + } + + private PathwayAnalysisDTO createPathwayAnalysisDTO() { + PathwayAnalysisDTO pathwayAnalysisDTO = new PathwayAnalysisDTO(); + pathwayAnalysisDTO.setName("pathwayAnalysis"); + pathwayAnalysisDTO.setDescription("desc"); + return pathwayAnalysisDTO; + } +} diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index ccad0fb191..0cb053296d 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -50,7 +50,7 @@ public void shouldGetBrief() { when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); assertEquals(brief.getName(), "incidence_rates_analysis_" + analysisId + "_" + sourceKey); - assertEquals(brief.getTitle(), incidenceRateAnalysis.getName()); + assertEquals(brief.getTitle(), "Analysis Name (sourceKey)"); assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); } From 1b444a4fd98875a5aa7451f5040d12d2e23b3704 Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Mon, 1 Jul 2024 10:37:44 +0200 Subject: [PATCH 18/32] Explicitly using the dot instead of the underscore as there were issues while configuring application Docker container with an --env-file --- src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java | 4 ++-- .../java/org/ohdsi/webapi/shiny/PositConnectProperties.java | 2 ++ src/main/resources/application-shiny.properties | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java index 5677a6381d..ddc93642fc 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -139,8 +139,8 @@ private Call call(Request.Builder request, String token) { public void afterPropertiesSet() throws Exception { if (properties != null) { if (StringUtils.isBlank(properties.getApiKey())) { - log.error("Set Posit Connect API Key to property \"shiny.connect.api_key\""); - throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api_key\""); + log.error("Set Posit Connect API Key to property \"shiny.connect.api.key\""); + throw new BeanInitializationException("Set Posit Connect API Key to property \"shiny.connect.api.key\""); } if (StringUtils.isBlank(properties.getUrl())) { log.error("Set Posit Connect URL to property \"shiny.connect.url\""); diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java index 6260449f8d..3c71e2f18f 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java @@ -1,9 +1,11 @@ package org.ohdsi.webapi.shiny; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.properties.ConfigurationProperties; @ConfigurationProperties(prefix = "shiny.connect") public class PositConnectProperties { + @Value("${shiny.connect.api.key}") private String apiKey; private String url; diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties index 08b306c66e..fbd6440afa 100644 --- a/src/main/resources/application-shiny.properties +++ b/src/main/resources/application-shiny.properties @@ -2,5 +2,5 @@ flyway.locations=${flyway.locations},classpath:shiny/migration shiny.atlas.url=${shiny.atlas.url} shiny.repo.link=${shiny.repo.link} -shiny.connect.api_key=${shiny.connect.api_key} +shiny.connect.api.key=${shiny.connect.api.key} shiny.connect.url=${shiny.connect.url} \ No newline at end of file From 2ae926efcb6fb0f1b808ff16b0e1d0b301506f90 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Mon, 1 Jul 2024 17:51:48 +0200 Subject: [PATCH 19/32] Unified Shiny apps names --- ...CharacterizationShinyPackagingService.java | 15 +++++---- .../CohortCountsShinyPackagingService.java | 26 ++++++++++------ .../CohortPathwaysShinyPackagingService.java | 9 ++++-- .../IncidenceRatesShinyPackagingService.java | 31 +++++++++---------- .../webapi/shiny/ShinyPackagingService.java | 6 ++-- ...hortPathwaysShinyPackagingServiceTest.java | 3 +- ...cidenceRatesShinyPackagingServiceTest.java | 2 +- 7 files changed, 52 insertions(+), 40 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index c5d299d4dc..3e6f0f38d2 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Iterables; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; @@ -37,11 +36,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.time.Instant; import java.util.ArrayList; -import java.util.Comparator; -import java.util.Date; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -52,6 +47,7 @@ public class CohortCharacterizationShinyPackagingService implements ShinyPackagi private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_PATH = "/shiny/shiny-cohortCharacterizations.zip"; + private static final String APP_NAME_FORMAT = "Characterization_%s_gv%s_%s"; @Value("${shiny.atlas.url}") private String atlasUrl; @Autowired @@ -100,8 +96,7 @@ public TemporaryFile packageApp(Integer generationId, String sourceKey, Packagin fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), - CommonFilenameUtils.sanitizeFilename(cohortCharacterization.getName())), appArchive); + return new TemporaryFile(String.format("%s.zip", prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)), appArchive); } catch (IOException e) { LOG.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); @@ -207,8 +202,12 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); ApplicationBrief applicationBrief = new ApplicationBrief(); applicationBrief.setName(MessageFormat.format("cohort_characterization_analysis_{0}_{1}", generationId, sourceKey)); - applicationBrief.setTitle(String.format("%s (%s)", cohortCharacterizationEntity.getName(), sourceKey)); + applicationBrief.setTitle(prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)); applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); return applicationBrief; } + + private String prepareAppTitle(Long studyAssetId, Integer generationId, String sourceKey) { + return String.format(APP_NAME_FORMAT, studyAssetId, generationId, sourceKey); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index 8cf3632d62..db628f1eab 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -32,6 +32,8 @@ public class CohortCountsShinyPackagingService implements ShinyPackagingService private static final Logger log = LoggerFactory.getLogger(CohortCountsShinyPackagingService.class); private static final String SHINY_COHORT_COUNTS = "/shiny/shiny-cohortCounts.zip"; + private static final String APP_NAME_FORMAT = "Cohort_%s_%s"; + @Autowired private CohortDefinitionService cohortDefinitionService; @Autowired @@ -50,10 +52,10 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStrategy packaging) { + public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { return TempFileUtils.doInDirectory(path -> { - CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); - ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", cohortId)); + CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); try { File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_COUNTS, "shiny", ".zip"); CommonFileUtils.unzipFiles(templateArchive, path.toFile()); @@ -63,20 +65,20 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr } JsonNode manifest = manifestUtils.parseManifest(manifestPath); - InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 0); //by event - InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(cohortId, sourceKey, 1); //by person + InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event + InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); Stream.of( fileWriter.writeObjectAsJsonFile(dataDir, byEventReport, sourceKey + "_by_event.json"), fileWriter.writeObjectAsJsonFile(dataDir, byPersonReport, sourceKey + "_by_person.json"), - fileWriter.writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, cohortId)), + fileWriter.writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, generationId)), fileWriter.writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())), fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) ).forEach(manifestUtils.addDataToManifest(manifest, path)); fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("Cohort_%s_%s.zip", cohortId, sourceKey), appArchive); + return new TemporaryFile(String.format("%s.zip", prepareAppTitle(generationId, sourceKey)), appArchive); } catch (IOException e) { log.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); @@ -85,12 +87,16 @@ public TemporaryFile packageApp(Integer cohortId, String sourceKey, PackagingStr } @Override - public ApplicationBrief getBrief(Integer cohortId, String sourceKey) { - CohortDefinition cohort = cohortDefinitionRepository.findOne(cohortId); + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); ApplicationBrief brief = new ApplicationBrief(); brief.setName(MessageFormat.format("cohort_{0}_{1}", cohort.getId(), sourceKey)); - brief.setTitle(String.format("%s (%s)", cohort.getName(), sourceKey)); + brief.setTitle(prepareAppTitle(generationId, sourceKey)); brief.setDescription(cohort.getDescription()); return brief; } + + private String prepareAppTitle(Integer generationId, String sourceKey) { + return String.format(APP_NAME_FORMAT, generationId, sourceKey); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index f74a1719ae..83d1a5c9fa 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -30,6 +30,7 @@ public class CohortPathwaysShinyPackagingService implements ShinyPackagingServic private static final Logger log = LoggerFactory.getLogger(CohortPathwaysShinyPackagingService.class); private static final String SHINY_COHORT_PATHWAYS = "/shiny/shiny-cohortPathways.zip"; + private static final String APP_NAME_FORMAT = "Pathway_%s_gv%s_%s"; @Autowired private PathwayService pathwayService; @Autowired @@ -67,7 +68,7 @@ public TemporaryFile packageApp(Integer generationId, String sourceKey, Packagin ).forEach(manifestUtils.addDataToManifest(manifest, path)); fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("CohortPathways_%s_%s.zip", generationId, sourceKey), appArchive); + return new TemporaryFile(String.format("%s.zip", prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)), appArchive); } catch (IOException e) { log.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); @@ -80,8 +81,12 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); applicationBrief.setName(MessageFormat.format("cohort_pathways_analysis_{0}_{1}", generationId, sourceKey)); - applicationBrief.setTitle(String.format("%s (%s)", pathwayAnalysis.getName(), sourceKey)); + applicationBrief.setTitle(prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)); applicationBrief.setDescription(pathwayAnalysis.getDescription()); return applicationBrief; } + + private String prepareAppTitle(Integer studyAssetId, Integer generationId, String sourceKey) { + return String.format(APP_NAME_FORMAT, studyAssetId, generationId, sourceKey); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index 09df4a96f1..20c10595a1 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.commons.utils.CommonFilenameUtils; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; @@ -31,9 +30,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.text.MessageFormat; -import java.text.SimpleDateFormat; -import java.time.Instant; -import java.util.Date; import java.util.List; import java.util.stream.Stream; @@ -46,7 +42,7 @@ public class IncidenceRatesShinyPackagingService implements ShinyPackagingServic private static final String SHINY_INCIDENCE_RATES_APP_PATH = "/shiny/shiny-incidenceRates.zip"; private static final String COHORT_TYPE_TARGET = "target"; private static final String COHORT_TYPE_OUTCOME = "outcome"; - + private static final String APP_NAME_FORMAT = "Incidence_%s_%s"; @Autowired private FileWriter fileWriter; @@ -70,10 +66,10 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingStrategy packaging) { + public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { return TempFileUtils.doInDirectory(path -> { - IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(analysisId); - ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", analysisId)); + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); try { File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_INCIDENCE_RATES_APP_PATH, "shiny", ".zip"); CommonFileUtils.unzipFiles(templateArchive, path.toFile()); @@ -88,13 +84,13 @@ public TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingS analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); String csvWithCohortDetails = prepareCsvWithCohorts(expression); - Stream analysisReportPaths = streamAnalysisReportsForAllCohortCombinations(expression, analysisId, sourceKey) + Stream analysisReportPaths = streamAnalysisReportsForAllCohortCombinations(expression, generationId, sourceKey) .map(analysisReport -> fileWriter.writeObjectAsJsonFile(dataDir, analysisReport, String.format( "%s_targetId%s_outcomeId%s.json", sourceKey, analysisReport.summary.targetId, analysisReport.summary.outcomeId))); Stream additionalMetadataFilesPaths = Stream.of( fileWriter.writeTextFile(dataDir.resolve("cohorts.csv"), pw -> pw.print(csvWithCohortDetails)), - fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/iranalysis/%s", atlasUrl, analysisId)), + fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/iranalysis/%s", atlasUrl, generationId)), fileWriter.writeTextFile(dataDir.resolve("repo_link.txt"), pw -> pw.print(repoLink)), fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) ); @@ -104,8 +100,7 @@ public TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingS fileWriter.writeJsonNodeToFile(manifest, manifestPath); Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s_%s_%s.zip", sourceKey, new SimpleDateFormat("yyyy_MM_dd").format(Date.from(Instant.now())), - CommonFilenameUtils.sanitizeFilename(analysis.getName())), appArchive); + return new TemporaryFile(String.format("%s.zip", prepareAppTitle(generationId, sourceKey)), appArchive); } catch (IOException e) { LOG.error("Failed to prepare Shiny application", e); throw new InternalServerErrorException(); @@ -127,11 +122,11 @@ private Stream streamAnalysisReportsForOneCohortCombination(Inte } @Override - public ApplicationBrief getBrief(Integer analysisId, String sourceKey) { - IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(analysisId); + public ApplicationBrief getBrief(Integer generationId, String sourceKey) { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("incidence_rates_analysis_{0}_{1}", analysisId, sourceKey)); - applicationBrief.setTitle(String.format("%s (%s)", analysis.getName(), sourceKey)); + applicationBrief.setName(MessageFormat.format("incidence_rates_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setTitle(prepareAppTitle(generationId, sourceKey)); applicationBrief.setDescription(analysis.getDescription()); return applicationBrief; } @@ -160,4 +155,8 @@ private String prepareCsvWithCohorts(IncidenceRateAnalysisExportExpression expre throw new InternalServerErrorException(); } } + + private String prepareAppTitle(Integer generationId, String sourceKey) { + return String.format(APP_NAME_FORMAT, generationId, sourceKey); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java index 267d547c8e..6e1c852384 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyPackagingService.java @@ -4,6 +4,8 @@ public interface ShinyPackagingService { CommonAnalysisType getType(); - TemporaryFile packageApp(Integer analysisId, String sourceKey, PackagingStrategy packaging); - ApplicationBrief getBrief(Integer analysisId, String sourceKey); + + TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging); + + ApplicationBrief getBrief(Integer generationId, String sourceKey); } diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java index c802a08e01..d5ef350edd 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -40,7 +40,7 @@ public void shouldGetBrief() { ApplicationBrief brief = sut.getBrief(GENERATION_ID, SOURCE_KEY); assertEquals(brief.getName(), "cohort_pathways_analysis_" + GENERATION_ID + "_" + SOURCE_KEY); - assertEquals(brief.getTitle(), "pathwayAnalysis (SynPuf110k)"); + assertEquals(brief.getTitle(), "Pathway_8_gv1_SynPuf110k"); assertEquals(brief.getDescription(), "desc"); } @@ -65,6 +65,7 @@ private PathwayAnalysisResult createPathwayAnalysisResult() { private PathwayAnalysisDTO createPathwayAnalysisDTO() { PathwayAnalysisDTO pathwayAnalysisDTO = new PathwayAnalysisDTO(); + pathwayAnalysisDTO.setId(8); pathwayAnalysisDTO.setName("pathwayAnalysis"); pathwayAnalysisDTO.setDescription("desc"); return pathwayAnalysisDTO; diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index 0cb053296d..feecfe8f8a 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -50,7 +50,7 @@ public void shouldGetBrief() { when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); assertEquals(brief.getName(), "incidence_rates_analysis_" + analysisId + "_" + sourceKey); - assertEquals(brief.getTitle(), "Analysis Name (sourceKey)"); + assertEquals(brief.getTitle(), "Incidence_1_sourceKey"); assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); } From 760403c5ab8ab991bbe122ef55d16fd1cf3f97c7 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 12 Jul 2024 16:21:08 +0200 Subject: [PATCH 20/32] [ATL-48] Fixed issue with Incidence Rates app publishing for a No Records scenario --- .../shiny/IncidenceRatesShinyPackagingService.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index 20c10595a1..a22cab6470 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -118,7 +118,15 @@ private Stream streamAnalysisReportsForAllCohortCombinations(Inc private Stream streamAnalysisReportsForOneCohortCombination(Integer targetCohortId, List outcomeCohorts, Integer analysisId, String sourceKey) { return outcomeCohorts.stream() - .map(outcomeCohort -> irAnalysisResource.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId())); + .map(outcomeCohort -> { + AnalysisReport analysisReport = irAnalysisResource.getAnalysisReport(analysisId, sourceKey, targetCohortId, outcomeCohort.getId()); + if (analysisReport.summary == null) { + analysisReport.summary = new AnalysisReport.Summary(); + analysisReport.summary.targetId = targetCohortId; + analysisReport.summary.outcomeId = outcomeCohort.getId(); + } + return analysisReport; + }); } @Override From d99a1e2de626f1430d42add5c5aeaf4f03c29f6d Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Thu, 22 Aug 2024 12:23:38 +0200 Subject: [PATCH 21/32] [ATL-50] Changed interface of the Cohort Pathways Shiny app to contain the same metadata as used by Atlas --- .../webapi/pathway/PathwayController.java | 33 +--------------- .../ohdsi/webapi/pathway/PathwayService.java | 5 ++- .../webapi/pathway/PathwayServiceImpl.java | 38 +++++++++++++++++++ .../CohortPathwaysShinyPackagingService.java | 10 ++--- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java index 4d7cd55dc6..02d93b1625 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayController.java @@ -430,38 +430,7 @@ public String getGenerationDesign( public PathwayPopulationResultsDTO getGenerationResults( @PathParam("generationId") final Long generationId ) { - - PathwayAnalysisResult resultingPathways = pathwayService.getResultingPathways(generationId); - - List eventCodeDtos = resultingPathways.getCodes() - .stream() - .map(entry -> { - PathwayCodeDTO dto = new PathwayCodeDTO(); - dto.setCode(entry.getCode()); - dto.setName(entry.getName()); - dto.setIsCombo(entry.isCombo()); - return dto; - }) - .collect(Collectors.toList()); - - List pathwayDtos = resultingPathways.getCohortPathwaysList() - .stream() - .map(cohortResults -> { - if (cohortResults.getPathwaysCounts() == null) { - return null; - } - - List eventDTOs = cohortResults.getPathwaysCounts() - .entrySet() - .stream() - .map(entry -> new PathwayPopulationEventDTO(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - return new TargetCohortPathwaysDTO(cohortResults.getCohortId(), cohortResults.getTargetCohortCount(), cohortResults.getTotalPathwaysCount(), eventDTOs); - }) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - - return new PathwayPopulationResultsDTO(eventCodeDtos, pathwayDtos); + return pathwayService.getGenerationResults(generationId); } private PathwayAnalysisDTO reloadAndConvert(Integer id) { diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java index 3b465c593b..a3137ab189 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayService.java @@ -4,6 +4,7 @@ import org.ohdsi.webapi.pathway.domain.PathwayAnalysisEntity; import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.PathwayVersionFullDTO; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import org.ohdsi.webapi.shiro.annotations.PathwayAnalysisGenerationId; @@ -14,7 +15,6 @@ import org.ohdsi.webapi.versioning.dto.VersionUpdateDTO; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.transaction.annotation.Transactional; import java.util.List; import java.util.Map; @@ -71,6 +71,7 @@ public interface PathwayService extends HasTags { List listByTags(TagNameListRequestDTO requestDTO); - @Transactional PathwayAnalysisDTO getByGenerationId(Integer id); + + PathwayPopulationResultsDTO getGenerationResults(Long generationId); } diff --git a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java index 230811525e..65dd22155f 100644 --- a/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java +++ b/src/main/java/org/ohdsi/webapi/pathway/PathwayServiceImpl.java @@ -24,7 +24,11 @@ import org.ohdsi.webapi.pathway.domain.PathwayEventCohort; import org.ohdsi.webapi.pathway.domain.PathwayTargetCohort; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayCodeDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationEventDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.PathwayVersionFullDTO; +import org.ohdsi.webapi.pathway.dto.TargetCohortPathwaysDTO; import org.ohdsi.webapi.pathway.dto.internal.CohortPathways; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import org.ohdsi.webapi.pathway.dto.internal.PathwayCode; @@ -714,4 +718,38 @@ public PathwayAnalysisDTO getByGenerationId(final Integer id) { dto.getEventCohorts().forEach(ec -> ec.setCode(eventCodes.get(ec.getId()))); return dto; } + @Override + public PathwayPopulationResultsDTO getGenerationResults(Long generationId) { + PathwayAnalysisResult resultingPathways = getResultingPathways(generationId); + + List eventCodeDtos = resultingPathways.getCodes() + .stream() + .map(entry -> { + PathwayCodeDTO dto = new PathwayCodeDTO(); + dto.setCode(entry.getCode()); + dto.setName(entry.getName()); + dto.setIsCombo(entry.isCombo()); + return dto; + }) + .collect(Collectors.toList()); + + List pathwayDtos = resultingPathways.getCohortPathwaysList() + .stream() + .map(cohortResults -> { + if (cohortResults.getPathwaysCounts() == null) { + return null; + } + + List eventDTOs = cohortResults.getPathwaysCounts() + .entrySet() + .stream() + .map(entry -> new PathwayPopulationEventDTO(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + return new TargetCohortPathwaysDTO(cohortResults.getCohortId(), cohortResults.getTargetCohortCount(), cohortResults.getTotalPathwaysCount(), eventDTOs); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + return new PathwayPopulationResultsDTO(eventCodeDtos, pathwayDtos); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 83d1a5c9fa..01379187bc 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -5,7 +5,7 @@ import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.ohdsi.webapi.pathway.PathwayService; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; -import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; import org.ohdsi.webapi.util.TempFileUtils; @@ -47,9 +47,9 @@ public CommonAnalysisType getType() { public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { return TempFileUtils.doInDirectory(path -> { PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); - PathwayAnalysisResult pathwayAnalysisResult = pathwayService.getResultingPathways(generationId.longValue()); + PathwayPopulationResultsDTO pathwayPopulationResultsDTO = pathwayService.getGenerationResults(generationId.longValue()); ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysis, String.format("There is no pathway analysis definition with generation id = %d.", generationId)); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysisResult, String.format("There is no pathway analysis result definition with generation id = %d.", generationId)); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayPopulationResultsDTO, String.format("There is no pathway population result with generation id = %d.", generationId)); try { File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_PATHWAYS, "shiny", ".zip"); CommonFileUtils.unzipFiles(templateArchive, path.toFile()); @@ -62,8 +62,8 @@ public TemporaryFile packageApp(Integer generationId, String sourceKey, Packagin Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); Stream.of( - fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysis, "pathwayAnalysis.json"), - fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysisResult, "pathwayAnalysisResult.json"), + fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysis, "PathwayDefinitionsMetaData.json"), + fileWriter.writeObjectAsJsonFile(dataDir, pathwayPopulationResultsDTO, "PathwayAnalysisDTOs.json"), fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) ).forEach(manifestUtils.addDataToManifest(manifest, path)); fileWriter.writeJsonNodeToFile(manifest, manifestPath); From 16ca5141f6a8bb0c51a4ba890167c386022f9e07 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 23 Aug 2024 14:04:32 +0200 Subject: [PATCH 22/32] Added a configurable timeout for OkHttp client (posit shiny apps publishing connect/read/write timeout, no changes/limits to the call timeout), Improved shiny apps code design, eliminated duplicate logic, implemented properties propagation into R apps using a single file app.properties --- ...CharacterizationShinyPackagingService.java | 93 ++++------ .../CohortCountsShinyPackagingService.java | 92 ++++------ .../CohortPathwaysShinyPackagingService.java | 82 ++++----- .../shiny/CommonShinyPackagingService.java | 164 ++++++++++++++++++ .../IncidenceRatesShinyPackagingService.java | 111 +++++------- .../webapi/shiny/PositConnectClient.java | 9 + .../webapi/shiny/PositConnectProperties.java | 10 ++ .../resources/application-shiny.properties | 3 +- ...hortPathwaysShinyPackagingServiceTest.java | 20 ++- ...cidenceRatesShinyPackagingServiceTest.java | 57 ++++-- 10 files changed, 386 insertions(+), 255 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index 3e6f0f38d2..aa91d18475 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -1,9 +1,8 @@ package org.ohdsi.webapi.shiny; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Iterables; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; @@ -19,7 +18,6 @@ import org.ohdsi.webapi.cohortcharacterization.report.Report; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; -import org.ohdsi.webapi.util.TempFileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -29,79 +27,64 @@ import org.springframework.transaction.annotation.Transactional; import javax.ws.rs.InternalServerErrorException; -import java.io.File; import java.io.IOException; import java.io.StringWriter; import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Path; import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import java.util.stream.Stream; @Service @ConditionalOnBean(ShinyService.class) -public class CohortCharacterizationShinyPackagingService implements ShinyPackagingService { +public class CohortCharacterizationShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; - private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_PATH = "/shiny/shiny-cohortCharacterizations.zip"; + private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCharacterizations.zip"; private static final String APP_NAME_FORMAT = "Characterization_%s_gv%s_%s"; - @Value("${shiny.atlas.url}") - private String atlasUrl; - @Autowired - private CcService ccService; - @Autowired - private FileWriter fileWriter; - @Autowired - private ManifestUtils manifestUtils; + + private final CcService ccService; + + private final CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper; + @Autowired - private CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper; + public CohortCharacterizationShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, + CcService ccService, + CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + this.ccService = ccService; + this.cohortCharacterizationAnalysisHeaderToFieldMapper = cohortCharacterizationAnalysisHeaderToFieldMapper; + } @Override public CommonAnalysisType getType() { return CommonAnalysisType.COHORT_CHARACTERIZATION; } + + @Override + public String getAppTemplateFilePath() { + return SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH; + } + @Override @Transactional - public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { - return TempFileUtils.doInDirectory(path -> { - CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); - GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); - ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); - try { - File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_CHARACTERIZATIONS_APP_PATH, "shiny", ".zip"); - CommonFileUtils.unzipFiles(templateArchive, path.toFile()); - Path manifestPath = path.resolve("manifest.json"); - if (!Files.exists(manifestPath)) { - throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); - } - JsonNode manifest = manifestUtils.parseManifest(manifestPath); - Path dataDir = path.resolve("data"); - Files.createDirectory(dataDir); - - Stream generatedCsvPaths = generationResults.getReports() - .stream() - .map(this::convertReportToCSV) - .map(contentsByFilename -> fileWriter.writeTextFile(dataDir.resolve(contentsByFilename.getLeft()), pw -> pw.print(contentsByFilename.getRight()))); - - Stream additionalMetadataFilesPaths = Stream.of( - fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())) - ); - - Stream.concat(generatedCsvPaths, additionalMetadataFilesPaths) - .forEach(manifestUtils.addDataToManifest(manifest, path)); - - fileWriter.writeJsonNodeToFile(manifest, manifestPath); - Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s.zip", prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)), appArchive); - } catch (IOException e) { - LOG.error("Failed to prepare Shiny application", e); - throw new InternalServerErrorException(); - } - }); + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); + ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); + + dataConsumers.getAppProperties().accept("atlas_link", String.format("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())); + + generationResults.getReports() + .stream() + .map(this::convertReportToCSV) + .forEach(csvDataByFilename -> dataConsumers.getTextFiles().accept(csvDataByFilename.getKey(), csvDataByFilename.getValue())); } //Pair.left == CSV filename @@ -201,7 +184,7 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("cohort_characterization_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(MessageFormat.format("cca_{0}_{1}", generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)); applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); return applicationBrief; diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index db628f1eab..fc2d95c6d5 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -1,50 +1,39 @@ package org.ohdsi.webapi.shiny; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; import org.ohdsi.webapi.service.CohortDefinitionService; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; -import org.ohdsi.webapi.util.TempFileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; -import javax.ws.rs.InternalServerErrorException; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.text.MessageFormat; -import java.util.stream.Stream; @Service @ConditionalOnBean(ShinyService.class) -public class CohortCountsShinyPackagingService implements ShinyPackagingService { +public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final String SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCounts.zip"; + private static final String APP_TITLE_FORMAT = "Cohort_%s_%s"; + private final CohortDefinitionService cohortDefinitionService; + private final CohortDefinitionRepository cohortDefinitionRepository; - private static final Logger log = LoggerFactory.getLogger(CohortCountsShinyPackagingService.class); - private static final String SHINY_COHORT_COUNTS = "/shiny/shiny-cohortCounts.zip"; - - private static final String APP_NAME_FORMAT = "Cohort_%s_%s"; - - @Autowired - private CohortDefinitionService cohortDefinitionService; - @Autowired - private CohortDefinitionRepository cohortDefinitionRepository; @Autowired - private FileWriter fileWriter; - @Autowired - private ManifestUtils manifestUtils; - - @Value("${shiny.atlas.url}") - private String atlasUrl; + public CohortCountsShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, CohortDefinitionService cohortDefinitionService, CohortDefinitionRepository cohortDefinitionRepository) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + this.cohortDefinitionService = cohortDefinitionService; + this.cohortDefinitionRepository = cohortDefinitionRepository; + } @Override public CommonAnalysisType getType() { @@ -52,38 +41,23 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { - return TempFileUtils.doInDirectory(path -> { - CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); - ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); - try { - File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_COUNTS, "shiny", ".zip"); - CommonFileUtils.unzipFiles(templateArchive, path.toFile()); - Path manifestPath = path.resolve("manifest.json"); - if (!Files.exists(manifestPath)) { - throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); - } - JsonNode manifest = manifestUtils.parseManifest(manifestPath); + public String getAppTemplateFilePath() { + return SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH; + } + + @Override + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); + + dataConsumers.getAppProperties().accept("cohort_link", String.format("%s/#/cohortdefinition/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept("cohort_name", cohort.getName()); + + InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event + InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person - InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event - InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person - Path dataDir = path.resolve("data"); - Files.createDirectory(dataDir); - Stream.of( - fileWriter.writeObjectAsJsonFile(dataDir, byEventReport, sourceKey + "_by_event.json"), - fileWriter.writeObjectAsJsonFile(dataDir, byPersonReport, sourceKey + "_by_person.json"), - fileWriter.writeTextFile(dataDir.resolve("cohort_link.txt"), pw -> pw.printf("%s/#/cohortdefinition/%s", atlasUrl, generationId)), - fileWriter.writeTextFile(dataDir.resolve("cohort_name.txt"), pw -> pw.print(cohort.getName())), - fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) - ).forEach(manifestUtils.addDataToManifest(manifest, path)); - fileWriter.writeJsonNodeToFile(manifest, manifestPath); - Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s.zip", prepareAppTitle(generationId, sourceKey)), appArchive); - } catch (IOException e) { - log.error("Failed to prepare Shiny application", e); - throw new InternalServerErrorException(); - } - }); + dataConsumers.getJsonObjects().accept(sourceKey + "_by_event.json", byEventReport); + dataConsumers.getJsonObjects().accept(sourceKey + "_by_person.json", byPersonReport); } @Override @@ -97,6 +71,6 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { } private String prepareAppTitle(Integer generationId, String sourceKey) { - return String.format(APP_NAME_FORMAT, generationId, sourceKey); + return String.format(APP_TITLE_FORMAT, generationId, sourceKey); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 01379187bc..071237e0af 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -1,42 +1,37 @@ package org.ohdsi.webapi.shiny; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.ohdsi.webapi.pathway.PathwayService; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; -import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; +import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; -import org.ohdsi.webapi.util.TempFileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; -import javax.ws.rs.InternalServerErrorException; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.text.MessageFormat; -import java.util.stream.Stream; @Service @ConditionalOnBean(ShinyService.class) -public class CohortPathwaysShinyPackagingService implements ShinyPackagingService { +public class CohortPathwaysShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final String SHINY_COHORT_PATHWAYS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortPathways.zip"; + private static final String APP_TITLE_FORMAT = "Pathway_%s_gv%s_%s"; - private static final Logger log = LoggerFactory.getLogger(CohortPathwaysShinyPackagingService.class); - private static final String SHINY_COHORT_PATHWAYS = "/shiny/shiny-cohortPathways.zip"; + private final PathwayService pathwayService; - private static final String APP_NAME_FORMAT = "Pathway_%s_gv%s_%s"; @Autowired - private PathwayService pathwayService; - @Autowired - private FileWriter fileWriter; - @Autowired - private ManifestUtils manifestUtils; + public CohortPathwaysShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, PathwayService pathwayService) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + this.pathwayService = pathwayService; + } @Override public CommonAnalysisType getType() { @@ -44,49 +39,32 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { - return TempFileUtils.doInDirectory(path -> { - PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); - PathwayPopulationResultsDTO pathwayPopulationResultsDTO = pathwayService.getGenerationResults(generationId.longValue()); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysis, String.format("There is no pathway analysis definition with generation id = %d.", generationId)); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayPopulationResultsDTO, String.format("There is no pathway population result with generation id = %d.", generationId)); - try { - File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_COHORT_PATHWAYS, "shiny", ".zip"); - CommonFileUtils.unzipFiles(templateArchive, path.toFile()); - Path manifestPath = path.resolve("manifest.json"); - if (!Files.exists(manifestPath)) { - throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); - } - JsonNode manifest = manifestUtils.parseManifest(manifestPath); + public String getAppTemplateFilePath() { + return SHINY_COHORT_PATHWAYS_APP_TEMPLATE_FILE_PATH; + } + + @Override + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); + PathwayAnalysisResult pathwayAnalysisResult = pathwayService.getResultingPathways(generationId.longValue()); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysis, String.format("There is no pathway analysis definition with generation id = %d.", generationId)); + ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysisResult, String.format("There is no pathway analysis result definition with generation id = %d.", generationId)); - Path dataDir = path.resolve("data"); - Files.createDirectory(dataDir); - Stream.of( - fileWriter.writeObjectAsJsonFile(dataDir, pathwayAnalysis, "PathwayDefinitionsMetaData.json"), - fileWriter.writeObjectAsJsonFile(dataDir, pathwayPopulationResultsDTO, "PathwayAnalysisDTOs.json"), - fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) - ).forEach(manifestUtils.addDataToManifest(manifest, path)); - fileWriter.writeJsonNodeToFile(manifest, manifestPath); - Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s.zip", prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)), appArchive); - } catch (IOException e) { - log.error("Failed to prepare Shiny application", e); - throw new InternalServerErrorException(); - } - }); + dataConsumers.getJsonObjects().accept("pathwayAnalysis.json", pathwayAnalysis); + dataConsumers.getJsonObjects().accept("pathwayAnalysisResult.json", pathwayAnalysisResult); } @Override public ApplicationBrief getBrief(Integer generationId, String sourceKey) { PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("cohort_pathways_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(MessageFormat.format("cpa_{0}_{1}", generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)); applicationBrief.setDescription(pathwayAnalysis.getDescription()); return applicationBrief; } private String prepareAppTitle(Integer studyAssetId, Integer generationId, String sourceKey) { - return String.format(APP_NAME_FORMAT, studyAssetId, generationId, sourceKey); + return String.format(APP_TITLE_FORMAT, studyAssetId, generationId, sourceKey); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java new file mode 100644 index 0000000000..e58ee552ad --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java @@ -0,0 +1,164 @@ +package org.ohdsi.webapi.shiny; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.ohdsi.webapi.util.TempFileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.ws.rs.InternalServerErrorException; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiConsumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public abstract class CommonShinyPackagingService { + private static final Logger LOG = LoggerFactory.getLogger(CommonShinyPackagingService.class); + protected final String atlasUrl; + protected String repoLink; + protected final FileWriter fileWriter; + protected final ManifestUtils manifestUtils; + protected final ObjectMapper objectMapper; + + private final Map applicationProperties = new HashMap<>(); + private final Map jsonObjectsToSave = new HashMap<>(); + private final Map textFilesToSave = new HashMap<>(); + + public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper) { + this.atlasUrl = atlasUrl; + this.repoLink = repoLink; + this.fileWriter = fileWriter; + this.manifestUtils = manifestUtils; + this.objectMapper = objectMapper; + } + + public abstract CommonAnalysisType getType(); + + + public abstract ApplicationBrief getBrief(Integer generationId, String sourceKey); + + public abstract String getAppTemplateFilePath(); + + public abstract void populateAppData( + Integer generationId, + String sourceKey, + ShinyAppDataConsumers shinyAppDataConsumers + ); + + public String getAtlasUrl() { + return atlasUrl; + } + + public String getRepoLink() { + return repoLink; + } + + public void setRepoLink(String repoLink) { + this.repoLink = repoLink; + } + + public FileWriter getFileWriter() { + return fileWriter; + } + + public ManifestUtils getManifestUtils() { + return manifestUtils; + } + + public ObjectMapper getObjectMapper() { + return objectMapper; + } + + public Map getApplicationProperties() { + return applicationProperties; + } + + public Map getJsonObjectsToSave() { + return jsonObjectsToSave; + } + + public Map getTextFilesToSave() { + return textFilesToSave; + } + + class ShinyAppDataConsumers { + private final BiConsumer appProperties = getApplicationProperties()::put; + private final BiConsumer textFiles = getTextFilesToSave()::put; + private final BiConsumer jsonObjects = getJsonObjectsToSave()::put; + + public BiConsumer getAppProperties() { + return appProperties; + } + + public BiConsumer getTextFiles() { + return textFiles; + } + + public BiConsumer getJsonObjects() { + return jsonObjects; + } + } + + + public final TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { + return TempFileUtils.doInDirectory(path -> { + try { + File templateArchive = TempFileUtils.copyResourceToTempFile(getAppTemplateFilePath(), "shiny", ".zip"); + CommonFileUtils.unzipFiles(templateArchive, path.toFile()); + Path manifestPath = path.resolve("manifest.json"); + if (!Files.exists(manifestPath)) { + throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); + } + JsonNode manifest = getManifestUtils().parseManifest(manifestPath); + + Path dataDir = path.resolve("data"); + Files.createDirectory(dataDir); + + //Default properties common for each shiny app + getApplicationProperties().put("repo_link", getRepoLink()); + getApplicationProperties().put("atlas_url", getAtlasUrl()); + getApplicationProperties().put("datasource", sourceKey); + + populateAppData(generationId, sourceKey, new ShinyAppDataConsumers()); + + Stream textFilesPaths = getTextFilesToSave().entrySet() + .stream() + .map(entry -> getFileWriter().writeTextFile(dataDir.resolve(entry.getKey()), pw -> pw.print(entry.getValue()))); + + Stream jsonFilesPaths = getJsonObjectsToSave().entrySet() + .stream() + .map(entry -> getFileWriter().writeObjectAsJsonFile(dataDir, entry.getValue(), entry.getKey())); + + Stream appPropertiesFilePath = Stream.of( + getFileWriter().writeTextFile(dataDir.resolve("app.properties"), pw -> pw.print(prepareAppProperties(applicationProperties))) + ); + + Stream.of(textFilesPaths, jsonFilesPaths, appPropertiesFilePath) + .flatMap(Function.identity()) + .forEach(getManifestUtils().addDataToManifest(manifest, path)); + + getFileWriter().writeJsonNodeToFile(manifest, manifestPath); + Path appArchive = packaging.apply(path); + ApplicationBrief applicationBrief = getBrief(generationId, sourceKey); + return new TemporaryFile(String.format("%s.zip", applicationBrief.getTitle()), appArchive); + } catch (IOException e) { + LOG.error("Failed to prepare Shiny application", e); + throw new InternalServerErrorException(); + } + }); + } + + private String prepareAppProperties(Map appProperties) { + return getApplicationProperties().entrySet().stream() + .map(entry -> String.format("%s=%s\n", entry.getKey(), entry.getValue())) + .collect(Collectors.joining()); + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index a22cab6470..c61766362d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -1,9 +1,8 @@ package org.ohdsi.webapi.shiny; -import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; -import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; @@ -15,7 +14,6 @@ import org.ohdsi.webapi.service.IRAnalysisResource; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; -import org.ohdsi.webapi.util.TempFileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -24,41 +22,36 @@ import org.springframework.stereotype.Service; import javax.ws.rs.InternalServerErrorException; -import java.io.File; import java.io.IOException; import java.io.StringWriter; -import java.nio.file.Files; -import java.nio.file.Path; import java.text.MessageFormat; import java.util.List; import java.util.stream.Stream; @Service @ConditionalOnBean(ShinyService.class) -public class IncidenceRatesShinyPackagingService implements ShinyPackagingService { - +public class IncidenceRatesShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { private static final Logger LOG = LoggerFactory.getLogger(IncidenceRatesShinyPackagingService.class); - - private static final String SHINY_INCIDENCE_RATES_APP_PATH = "/shiny/shiny-incidenceRates.zip"; + private static final String SHINY_INCIDENCE_RATES_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-incidenceRates.zip"; private static final String COHORT_TYPE_TARGET = "target"; private static final String COHORT_TYPE_OUTCOME = "outcome"; private static final String APP_NAME_FORMAT = "Incidence_%s_%s"; + private final IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; + private final IRAnalysisResource irAnalysisResource; @Autowired - private FileWriter fileWriter; - @Autowired - private ManifestUtils manifestUtils; - @Autowired - private IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; - @Autowired - private IRAnalysisResource irAnalysisResource; - @Autowired - private ObjectMapper objectMapper; - - @Value("${shiny.atlas.url}") - private String atlasUrl; - @Value("${shiny.repo.link}") - private String repoLink; + public IncidenceRatesShinyPackagingService( + @Value("${shiny.atlas.url}") String atlasUrl, + @Value("${shiny.repo.link}") String repoLink, + FileWriter fileWriter, + ManifestUtils manifestUtils, + ObjectMapper objectMapper, + IncidenceRateAnalysisRepository incidenceRateAnalysisRepository, + IRAnalysisResource irAnalysisResource) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + this.incidenceRateAnalysisRepository = incidenceRateAnalysisRepository; + this.irAnalysisResource = irAnalysisResource; + } @Override public CommonAnalysisType getType() { @@ -66,46 +59,33 @@ public CommonAnalysisType getType() { } @Override - public TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { - return TempFileUtils.doInDirectory(path -> { - IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); - ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); - try { - File templateArchive = TempFileUtils.copyResourceToTempFile(SHINY_INCIDENCE_RATES_APP_PATH, "shiny", ".zip"); - CommonFileUtils.unzipFiles(templateArchive, path.toFile()); - Path manifestPath = path.resolve("manifest.json"); - if (!Files.exists(manifestPath)) { - throw new PositConnectClientException("manifest.json is not found in the Shiny Application"); - } - JsonNode manifest = manifestUtils.parseManifest(manifestPath); - Path dataDir = path.resolve("data"); - Files.createDirectory(dataDir); - IncidenceRateAnalysisExportExpression expression = objectMapper.readValue( - analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); - String csvWithCohortDetails = prepareCsvWithCohorts(expression); - - Stream analysisReportPaths = streamAnalysisReportsForAllCohortCombinations(expression, generationId, sourceKey) - .map(analysisReport -> fileWriter.writeObjectAsJsonFile(dataDir, analysisReport, String.format( - "%s_targetId%s_outcomeId%s.json", sourceKey, analysisReport.summary.targetId, analysisReport.summary.outcomeId))); - - Stream additionalMetadataFilesPaths = Stream.of( - fileWriter.writeTextFile(dataDir.resolve("cohorts.csv"), pw -> pw.print(csvWithCohortDetails)), - fileWriter.writeTextFile(dataDir.resolve("atlas_link.txt"), pw -> pw.printf("%s/#/iranalysis/%s", atlasUrl, generationId)), - fileWriter.writeTextFile(dataDir.resolve("repo_link.txt"), pw -> pw.print(repoLink)), - fileWriter.writeTextFile(dataDir.resolve("datasource.txt"), pw -> pw.print(sourceKey)) - ); - - Stream.concat(analysisReportPaths, additionalMetadataFilesPaths) - .forEach(manifestUtils.addDataToManifest(manifest, path)); - - fileWriter.writeJsonNodeToFile(manifest, manifestPath); - Path appArchive = packaging.apply(path); - return new TemporaryFile(String.format("%s.zip", prepareAppTitle(generationId, sourceKey)), appArchive); - } catch (IOException e) { - LOG.error("Failed to prepare Shiny application", e); - throw new InternalServerErrorException(); - } - }); + public String getAppTemplateFilePath() { + return SHINY_INCIDENCE_RATES_APP_TEMPLATE_FILE_PATH; + } + + @Override + public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { + IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); + ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); + try { + dataConsumers.getAppProperties().accept("atlas_link", String.format("%s/#/iranalysis/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept("analysis_name", analysis.getName()); + + IncidenceRateAnalysisExportExpression expression = objectMapper.readValue(analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); + String csvWithCohortDetails = prepareCsvWithCohorts(expression); + + dataConsumers.getTextFiles().accept("cohorts.csv", csvWithCohortDetails); + + streamAnalysisReportsForAllCohortCombinations(expression, generationId, sourceKey) + .forEach(analysisReport -> + dataConsumers.getJsonObjects().accept( + String.format("%s_targetId%s_outcomeId%s.json", sourceKey, analysisReport.summary.targetId, analysisReport.summary.outcomeId), + analysisReport + ) + ); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } } private Stream streamAnalysisReportsForAllCohortCombinations(IncidenceRateAnalysisExportExpression expression, Integer analysisId, String sourceKey) { @@ -133,13 +113,12 @@ private Stream streamAnalysisReportsForOneCohortCombination(Inte public ApplicationBrief getBrief(Integer generationId, String sourceKey) { IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("incidence_rates_analysis_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(MessageFormat.format("ira_{0}_{1}", generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(generationId, sourceKey)); applicationBrief.setDescription(analysis.getDescription()); return applicationBrief; } - private String prepareCsvWithCohorts(IncidenceRateAnalysisExportExpression expression) { final String[] HEADER = {"cohort_id", "cohort_name", "type"}; List targetCohorts = expression.targetCohorts; diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java index ddc93642fc..f20d3c8224 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectClient.java @@ -29,7 +29,9 @@ import java.text.MessageFormat; import java.time.Instant; import java.util.List; +import java.util.Objects; import java.util.UUID; +import java.util.concurrent.TimeUnit; @Service @ConditionalOnBean(ShinyService.class) @@ -131,6 +133,9 @@ private String toJson(T value) { private Call call(Request.Builder request, String token) { OkHttpClient client = new OkHttpClient.Builder() .retryOnConnectionFailure(false) + .connectTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) + .readTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) + .writeTimeout(properties.getTimeoutSeconds(), TimeUnit.SECONDS) .build(); return client.newCall(request.header(HEADER_AUTH, AUTH_PREFIX + " " + token).build()); } @@ -146,6 +151,10 @@ public void afterPropertiesSet() throws Exception { log.error("Set Posit Connect URL to property \"shiny.connect.url\""); throw new BeanInitializationException("Set Posit Connect URL to property \"shiny.connect.url\""); } + if (Objects.isNull(properties.getTimeoutSeconds())) { + log.error("Set Posit Connect HTTP Connect/Read/Write Timeout to property \"shiny.connect.okhttp.timeout.seconds\""); + throw new BeanInitializationException("Set Posit Connect HTTP Connect/Read/Write Timeout to property \"shiny.connect.okhttp.timeout.seconds\""); + } } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java index 3c71e2f18f..a603e35df9 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java +++ b/src/main/java/org/ohdsi/webapi/shiny/PositConnectProperties.java @@ -8,6 +8,8 @@ public class PositConnectProperties { @Value("${shiny.connect.api.key}") private String apiKey; private String url; + @Value("${shiny.connect.okhttp.timeout.seconds}") + private Integer timeoutSeconds; public String getApiKey() { return apiKey; @@ -24,4 +26,12 @@ public String getUrl() { public void setUrl(String url) { this.url = url; } + + public Integer getTimeoutSeconds() { + return timeoutSeconds; + } + + public void setTimeoutSeconds(Integer timeoutSeconds) { + this.timeoutSeconds = timeoutSeconds; + } } diff --git a/src/main/resources/application-shiny.properties b/src/main/resources/application-shiny.properties index fbd6440afa..079030668b 100644 --- a/src/main/resources/application-shiny.properties +++ b/src/main/resources/application-shiny.properties @@ -3,4 +3,5 @@ flyway.locations=${flyway.locations},classpath:shiny/migration shiny.atlas.url=${shiny.atlas.url} shiny.repo.link=${shiny.repo.link} shiny.connect.api.key=${shiny.connect.api.key} -shiny.connect.url=${shiny.connect.url} \ No newline at end of file +shiny.connect.url=${shiny.connect.url} +shiny.connect.okhttp.timeout.seconds=${shiny.connect.okhttp.timeout.seconds} \ No newline at end of file diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java index d5ef350edd..e3c1dee1cd 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -3,8 +3,10 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; import org.ohdsi.webapi.pathway.PathwayService; @@ -12,9 +14,10 @@ import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; -import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -39,18 +42,21 @@ public void shouldGetBrief() { when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); ApplicationBrief brief = sut.getBrief(GENERATION_ID, SOURCE_KEY); - assertEquals(brief.getName(), "cohort_pathways_analysis_" + GENERATION_ID + "_" + SOURCE_KEY); + assertEquals(brief.getName(), "cpa_" + GENERATION_ID + "_" + SOURCE_KEY); assertEquals(brief.getTitle(), "Pathway_8_gv1_SynPuf110k"); assertEquals(brief.getDescription(), "desc"); } @Test - public void shouldPackageApp() { + public void shouldPopulateAppData() { when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(createPathwayAnalysisDTO()); when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); - PackagingStrategy packagingStrategy = mock(PackagingStrategy.class); - TemporaryFile result = sut.packageApp(GENERATION_ID, SOURCE_KEY, packagingStrategy); - assertNotNull(result); + + CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = Mockito.mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); + sut.populateAppData(GENERATION_ID, SOURCE_KEY, dataConsumers); + + verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("pathwayAnalysis.json"), any(PathwayAnalysisDTO.class)); + verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("pathwayAnalysisResult.json"), any(PathwayAnalysisResult.class)); } @Test diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index feecfe8f8a..37398f1baa 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -1,25 +1,31 @@ package org.ohdsi.webapi.shiny; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.Answers; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; +import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; import org.ohdsi.webapi.ircalc.AnalysisReport; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisDetails; +import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; import org.ohdsi.webapi.service.IRAnalysisResource; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @RunWith(MockitoJUnitRunner.class) @@ -39,7 +45,6 @@ public class IncidenceRatesShinyPackagingServiceTest { @InjectMocks private IncidenceRatesShinyPackagingService sut; - private final Integer analysisId = 1; private final String sourceKey = "sourceKey"; @@ -49,23 +54,45 @@ public void shouldGetBrief() { when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); - assertEquals(brief.getName(), "incidence_rates_analysis_" + analysisId + "_" + sourceKey); + assertEquals(brief.getName(), "ira_" + analysisId + "_" + sourceKey); assertEquals(brief.getTitle(), "Incidence_1_sourceKey"); assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); } @Test - public void shouldPackageApp() { - IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); - PackagingStrategy packagingStrategy = mock(PackagingStrategy.class); - when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); - when(irAnalysisResource.getAnalysisReport(anyInt(), anyString(), anyInt(), anyInt())) - .thenReturn(createAnalysisReport(1, 2)) - .thenReturn(createAnalysisReport(3, 4)) - .thenReturn(createAnalysisReport(5, 6)) - .thenReturn(createAnalysisReport(7, 8)); - TemporaryFile result = sut.packageApp(analysisId, sourceKey, packagingStrategy); - assertNotNull(result); + public void shouldPopulateAppDataWithValidData() throws JsonProcessingException { + Integer generationId = 1; + String sourceKey = "source"; + + IncidenceRateAnalysis analysis = Mockito.mock(IncidenceRateAnalysis.class, Answers.RETURNS_DEEP_STUBS.get()); + when(analysis.getDetails().getExpression()).thenReturn("{}"); + when(repository.findOne(generationId)).thenReturn(analysis); + + CohortDTO targetCohort = new CohortDTO(); + targetCohort.setId(101); + targetCohort.setName("Target Cohort"); + + CohortDTO outcomeCohort = new CohortDTO(); + outcomeCohort.setId(201); + outcomeCohort.setName("Outcome Cohort"); + + + IncidenceRateAnalysisExportExpression expression = new IncidenceRateAnalysisExportExpression(); + expression.outcomeCohorts.add(outcomeCohort); + expression.targetCohorts.add(targetCohort); + + when(objectMapper.readValue("{}", IncidenceRateAnalysisExportExpression.class)).thenReturn(expression); + AnalysisReport analysisReport = new AnalysisReport(); + analysisReport.summary = new AnalysisReport.Summary(); + when(irAnalysisResource.getAnalysisReport(1, "source", 101, 201)).thenReturn(analysisReport); + + CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); + + sut.populateAppData(generationId, sourceKey, dataConsumers); + + verify(dataConsumers.getAppProperties(), times(2)).accept(anyString(), anyString()); + verify(dataConsumers.getTextFiles(), times(1)).accept(anyString(), anyString()); + verify(dataConsumers.getJsonObjects(), times(1)).accept(anyString(), any()); } @Test From e6e972ed6de33270f9a9ca84a0d248681ebce20d Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Mon, 12 Aug 2024 22:24:02 +0200 Subject: [PATCH 23/32] Changed MessageFormat to String.format for shiny app names to avoid locale formatting of numbers --- .../shiny/CohortCharacterizationShinyPackagingService.java | 7 +++---- .../webapi/shiny/CohortCountsShinyPackagingService.java | 4 +--- .../webapi/shiny/CohortPathwaysShinyPackagingService.java | 4 +--- .../webapi/shiny/IncidenceRatesShinyPackagingService.java | 3 +-- 4 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index aa91d18475..e51ff01f52 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -30,7 +30,6 @@ import java.io.IOException; import java.io.StringWriter; import java.lang.reflect.Field; -import java.text.MessageFormat; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -41,7 +40,7 @@ public class CohortCharacterizationShinyPackagingService extends CommonShinyPack private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCharacterizations.zip"; - private static final String APP_NAME_FORMAT = "Characterization_%s_gv%s_%s"; + private static final String APP_TITLE_FORMAT = "Characterization_%s_gv%s_%s"; private final CcService ccService; @@ -184,13 +183,13 @@ public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("cca_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT_CHARACTERIZATION.getCode(), generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(cohortCharacterization.getId(), generationId, sourceKey)); applicationBrief.setDescription(cohortCharacterizationEntity.getDescription()); return applicationBrief; } private String prepareAppTitle(Long studyAssetId, Integer generationId, String sourceKey) { - return String.format(APP_NAME_FORMAT, studyAssetId, generationId, sourceKey); + return String.format(APP_TITLE_FORMAT, studyAssetId, generationId, sourceKey); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index fc2d95c6d5..b7df9d37bb 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -13,8 +13,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; -import java.text.MessageFormat; - @Service @ConditionalOnBean(ShinyService.class) public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { @@ -64,7 +62,7 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); ApplicationBrief brief = new ApplicationBrief(); - brief.setName(MessageFormat.format("cohort_{0}_{1}", cohort.getId(), sourceKey)); + brief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT.getCode(), generationId, sourceKey)); brief.setTitle(prepareAppTitle(generationId, sourceKey)); brief.setDescription(cohort.getDescription()); return brief; diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 071237e0af..033eb429e4 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -12,8 +12,6 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; -import java.text.MessageFormat; - @Service @ConditionalOnBean(ShinyService.class) public class CohortPathwaysShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { @@ -58,7 +56,7 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData public ApplicationBrief getBrief(Integer generationId, String sourceKey) { PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("cpa_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT_PATHWAY.getCode(), generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(pathwayAnalysis.getId(), generationId, sourceKey)); applicationBrief.setDescription(pathwayAnalysis.getDescription()); return applicationBrief; diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index c61766362d..544f4c8962 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -24,7 +24,6 @@ import javax.ws.rs.InternalServerErrorException; import java.io.IOException; import java.io.StringWriter; -import java.text.MessageFormat; import java.util.List; import java.util.stream.Stream; @@ -113,7 +112,7 @@ private Stream streamAnalysisReportsForOneCohortCombination(Inte public ApplicationBrief getBrief(Integer generationId, String sourceKey) { IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); ApplicationBrief applicationBrief = new ApplicationBrief(); - applicationBrief.setName(MessageFormat.format("ira_{0}_{1}", generationId, sourceKey)); + applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.INCIDENCE.getCode(), generationId, sourceKey)); applicationBrief.setTitle(prepareAppTitle(generationId, sourceKey)); applicationBrief.setDescription(analysis.getDescription()); return applicationBrief; From 532fd1db9cefa3cc6fb3dfa2dbe4f0924001615b Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Thu, 29 Aug 2024 14:00:51 +0200 Subject: [PATCH 24/32] Unified naming of generation versions for all shiny apps --- ...CharacterizationShinyPackagingService.java | 2 +- .../CohortCountsShinyPackagingService.java | 19 ++++++++++++++----- .../CohortPathwaysShinyPackagingService.java | 2 +- .../IncidenceRatesShinyPackagingService.java | 19 ++++++++++++++----- ...hortPathwaysShinyPackagingServiceTest.java | 4 ++-- ...cidenceRatesShinyPackagingServiceTest.java | 12 ++++++++++-- 6 files changed, 42 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index e51ff01f52..962b20d00c 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -40,7 +40,7 @@ public class CohortCharacterizationShinyPackagingService extends CommonShinyPack private static final Logger LOG = LoggerFactory.getLogger(CohortCharacterizationShinyPackagingService.class); private static final Float DEFAULT_THRESHOLD_VALUE = 0.01f; private static final String SHINY_COHORT_CHARACTERIZATIONS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCharacterizations.zip"; - private static final String APP_TITLE_FORMAT = "Characterization_%s_gv%s_%s"; + private static final String APP_TITLE_FORMAT = "Characterization_%s_gv%sx_%s"; private final CcService ccService; diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index b7df9d37bb..845241d329 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -7,30 +7,35 @@ import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; import org.ohdsi.webapi.service.CohortDefinitionService; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; @Service @ConditionalOnBean(ShinyService.class) public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { private static final String SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCounts.zip"; - private static final String APP_TITLE_FORMAT = "Cohort_%s_%s"; + private static final String APP_TITLE_FORMAT = "Cohort_%s_gv%sx%s_%s"; private final CohortDefinitionService cohortDefinitionService; private final CohortDefinitionRepository cohortDefinitionRepository; + private final SourceRepository sourceRepository; + @Autowired public CohortCountsShinyPackagingService( @Value("${shiny.atlas.url}") String atlasUrl, @Value("${shiny.repo.link}") String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, - ObjectMapper objectMapper, CohortDefinitionService cohortDefinitionService, CohortDefinitionRepository cohortDefinitionRepository) { + ObjectMapper objectMapper, CohortDefinitionService cohortDefinitionService, CohortDefinitionRepository cohortDefinitionRepository, SourceRepository sourceRepository) { super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); this.cohortDefinitionService = cohortDefinitionService; this.cohortDefinitionRepository = cohortDefinitionRepository; + this.sourceRepository = sourceRepository; } @Override @@ -44,6 +49,7 @@ public String getAppTemplateFilePath() { } @Override + @Transactional public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); @@ -61,14 +67,17 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData @Override public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); + Integer assetId = cohort.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); + ApplicationBrief brief = new ApplicationBrief(); brief.setName(String.format("%s_%s_%s", CommonAnalysisType.COHORT.getCode(), generationId, sourceKey)); - brief.setTitle(prepareAppTitle(generationId, sourceKey)); + brief.setTitle(prepareAppTitle(generationId, assetId, sourceId, sourceKey)); brief.setDescription(cohort.getDescription()); return brief; } - private String prepareAppTitle(Integer generationId, String sourceKey) { - return String.format(APP_TITLE_FORMAT, generationId, sourceKey); + private String prepareAppTitle(Integer generationId, Integer assetId, Integer sourceId, String sourceKey) { + return String.format(APP_TITLE_FORMAT, generationId, assetId, sourceId, sourceKey); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 033eb429e4..9a3bc51fe0 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -16,7 +16,7 @@ @ConditionalOnBean(ShinyService.class) public class CohortPathwaysShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { private static final String SHINY_COHORT_PATHWAYS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortPathways.zip"; - private static final String APP_TITLE_FORMAT = "Pathway_%s_gv%s_%s"; + private static final String APP_TITLE_FORMAT = "Pathway_%s_gv%sx_%s"; private final PathwayService pathwayService; diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index 544f4c8962..c7e919f5cd 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -13,6 +13,7 @@ import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; import org.ohdsi.webapi.service.IRAnalysisResource; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,6 +21,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import javax.ws.rs.InternalServerErrorException; import java.io.IOException; @@ -34,10 +36,12 @@ public class IncidenceRatesShinyPackagingService extends CommonShinyPackagingSer private static final String SHINY_INCIDENCE_RATES_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-incidenceRates.zip"; private static final String COHORT_TYPE_TARGET = "target"; private static final String COHORT_TYPE_OUTCOME = "outcome"; - private static final String APP_NAME_FORMAT = "Incidence_%s_%s"; + private static final String APP_NAME_FORMAT = "Incidence_%s_gv%sx%s_%s"; private final IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; private final IRAnalysisResource irAnalysisResource; + private final SourceRepository sourceRepository; + @Autowired public IncidenceRatesShinyPackagingService( @Value("${shiny.atlas.url}") String atlasUrl, @@ -46,10 +50,11 @@ public IncidenceRatesShinyPackagingService( ManifestUtils manifestUtils, ObjectMapper objectMapper, IncidenceRateAnalysisRepository incidenceRateAnalysisRepository, - IRAnalysisResource irAnalysisResource) { + IRAnalysisResource irAnalysisResource, SourceRepository sourceRepository) { super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); this.incidenceRateAnalysisRepository = incidenceRateAnalysisRepository; this.irAnalysisResource = irAnalysisResource; + this.sourceRepository = sourceRepository; } @Override @@ -63,6 +68,7 @@ public String getAppTemplateFilePath() { } @Override + @Transactional public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); @@ -109,11 +115,14 @@ private Stream streamAnalysisReportsForOneCohortCombination(Inte } @Override + @Transactional public ApplicationBrief getBrief(Integer generationId, String sourceKey) { IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); + Integer assetId = analysis.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); ApplicationBrief applicationBrief = new ApplicationBrief(); applicationBrief.setName(String.format("%s_%s_%s", CommonAnalysisType.INCIDENCE.getCode(), generationId, sourceKey)); - applicationBrief.setTitle(prepareAppTitle(generationId, sourceKey)); + applicationBrief.setTitle(prepareAppTitle(generationId, assetId, sourceId, sourceKey)); applicationBrief.setDescription(analysis.getDescription()); return applicationBrief; } @@ -142,7 +151,7 @@ private String prepareCsvWithCohorts(IncidenceRateAnalysisExportExpression expre } } - private String prepareAppTitle(Integer generationId, String sourceKey) { - return String.format(APP_NAME_FORMAT, generationId, sourceKey); + private String prepareAppTitle(Integer generationId, Integer assetId, Integer sourceId, String sourceKey) { + return String.format(APP_NAME_FORMAT, generationId, assetId, sourceId, sourceKey); } } diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java index e3c1dee1cd..fa275eeed7 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -42,8 +42,8 @@ public void shouldGetBrief() { when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); ApplicationBrief brief = sut.getBrief(GENERATION_ID, SOURCE_KEY); - assertEquals(brief.getName(), "cpa_" + GENERATION_ID + "_" + SOURCE_KEY); - assertEquals(brief.getTitle(), "Pathway_8_gv1_SynPuf110k"); + assertEquals(brief.getName(), "txp_" + GENERATION_ID + "_" + SOURCE_KEY); + assertEquals(brief.getTitle(), "Pathway_8_gv1x_SynPuf110k"); assertEquals(brief.getDescription(), "desc"); } diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index 37398f1baa..2c36607fb5 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -19,10 +19,13 @@ import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; import org.ohdsi.webapi.service.IRAnalysisResource; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -39,6 +42,8 @@ public class IncidenceRatesShinyPackagingServiceTest { private FileWriter fileWriter; @Mock private IRAnalysisResource irAnalysisResource; + @Mock + private SourceRepository sourceRepository; @Spy private ObjectMapper objectMapper; @@ -53,9 +58,12 @@ public void shouldGetBrief() { IncidenceRateAnalysis incidenceRateAnalysis = createIncidenceRateAnalysis(); when(repository.findOne(analysisId)).thenReturn(incidenceRateAnalysis); + Source source = new Source(); + source.setSourceId(3); + when(sourceRepository.findBySourceKey("sourceKey")).thenReturn(source); ApplicationBrief brief = sut.getBrief(analysisId, sourceKey); - assertEquals(brief.getName(), "ira_" + analysisId + "_" + sourceKey); - assertEquals(brief.getTitle(), "Incidence_1_sourceKey"); + assertEquals(brief.getName(), "ir_" + analysisId + "_" + sourceKey); + assertEquals(brief.getTitle(), "Incidence_1_gv1x3_sourceKey"); assertEquals(brief.getDescription(), incidenceRateAnalysis.getDescription()); } From 63c9d33217ca350e4976538eec840f555701bba0 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 30 Aug 2024 12:05:18 +0200 Subject: [PATCH 25/32] Added Content-Disposition header to CORS Filter to allow propagating the generated filenames to the UI --- src/main/java/org/ohdsi/webapi/Constants.java | 4 ++++ .../org/ohdsi/webapi/shiro/filters/CorsFilter.java | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/Constants.java b/src/main/java/org/ohdsi/webapi/Constants.java index 2069ed108d..7e1f07a4ad 100644 --- a/src/main/java/org/ohdsi/webapi/Constants.java +++ b/src/main/java/org/ohdsi/webapi/Constants.java @@ -89,9 +89,13 @@ interface Variables { } interface Headers { + String ACCESS_CONTROL_EXPOSE_HEADERS = "Access-Control-Expose-Headers"; String AUTH_PROVIDER = "x-auth-provider"; String USER_LANGAUGE = "User-Language"; String ACTION_LOCATION = "action-location"; + String BEARER = "Bearer"; + String X_AUTH_ERROR = "x-auth-error"; + String CONTENT_DISPOSITION = "Content-Disposition"; } interface SecurityProviders { diff --git a/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java b/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java index 320d64c25a..4c62e23479 100644 --- a/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java +++ b/src/main/java/org/ohdsi/webapi/shiro/filters/CorsFilter.java @@ -12,6 +12,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.List; import java.util.Objects; import java.util.StringJoiner; @@ -64,8 +66,16 @@ protected boolean preHandle(ServletRequest request, ServletResponse response) th // continue processing request // - httpResponse.setHeader("Access-Control-Expose-Headers", "Bearer,x-auth-error," + - Joiner.on(",").join(Constants.Headers.AUTH_PROVIDER, Constants.Headers.USER_LANGAUGE)); + + List exposedHeaders = Arrays.asList( + Constants.Headers.BEARER, + Constants.Headers.X_AUTH_ERROR, + Constants.Headers.AUTH_PROVIDER, + Constants.Headers.USER_LANGAUGE, + Constants.Headers.CONTENT_DISPOSITION + ); + httpResponse.setHeader(Constants.Headers.ACCESS_CONTROL_EXPOSE_HEADERS, Joiner.on(",").join(exposedHeaders)); + return true; } } From 364bcce5344596a8658bb5973bd0b7a09f960c78 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Thu, 5 Sep 2024 16:13:45 +0200 Subject: [PATCH 26/32] Added 'analysis_name' to app.properties for Cohort Characterization Shiny App --- .../shiny/CohortCharacterizationShinyPackagingService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index 962b20d00c..8712b0dde5 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -79,6 +79,7 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); dataConsumers.getAppProperties().accept("atlas_link", String.format("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())); + dataConsumers.getAppProperties().accept("analysis_name", cohortCharacterization.getName()); generationResults.getReports() .stream() From 6d91240640ae9a7c2519209fb1a47880ca99ac4a Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Wed, 18 Sep 2024 11:40:00 +0200 Subject: [PATCH 27/32] Added design JSON and chart data JSON files to Cohort Pathways Shiny app --- .../CohortPathwaysShinyPackagingService.java | 15 ++++++++------- .../CohortPathwaysShinyPackagingServiceTest.java | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 9a3bc51fe0..ab0ec04ffc 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -4,7 +4,7 @@ import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.ohdsi.webapi.pathway.PathwayService; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; -import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.service.ShinyService; import org.ohdsi.webapi.util.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -43,13 +43,14 @@ public String getAppTemplateFilePath() { @Override public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { - PathwayAnalysisDTO pathwayAnalysis = pathwayService.getByGenerationId(generationId); - PathwayAnalysisResult pathwayAnalysisResult = pathwayService.getResultingPathways(generationId.longValue()); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysis, String.format("There is no pathway analysis definition with generation id = %d.", generationId)); - ExceptionUtils.throwNotFoundExceptionIfNull(pathwayAnalysisResult, String.format("There is no pathway analysis result definition with generation id = %d.", generationId)); + String designJSON = pathwayService.findDesignByGenerationId(generationId.longValue()); + PathwayPopulationResultsDTO generationResults = pathwayService.getGenerationResults(generationId.longValue()); + + ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no pathway analysis generation results with generation id = %d.", generationId)); + ExceptionUtils.throwNotFoundExceptionIfNull(designJSON, String.format("There is no pathway analysis design with generation id = %d.", generationId)); - dataConsumers.getJsonObjects().accept("pathwayAnalysis.json", pathwayAnalysis); - dataConsumers.getJsonObjects().accept("pathwayAnalysisResult.json", pathwayAnalysisResult); + dataConsumers.getTextFiles().accept("design.json", designJSON); + dataConsumers.getJsonObjects().accept("chartData.json", generationResults); } @Override diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java index fa275eeed7..f40480d5f0 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -11,10 +11,14 @@ import org.mockito.runners.MockitoJUnitRunner; import org.ohdsi.webapi.pathway.PathwayService; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; +import java.util.Collections; + import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -49,14 +53,18 @@ public void shouldGetBrief() { @Test public void shouldPopulateAppData() { - when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(createPathwayAnalysisDTO()); - when(pathwayService.getResultingPathways(eq((long) GENERATION_ID))).thenReturn(createPathwayAnalysisResult()); + when(pathwayService.findDesignByGenerationId(eq((long) GENERATION_ID))).thenReturn("design json"); + when(pathwayService.getGenerationResults(eq((long) GENERATION_ID))).thenReturn(createPathwayGenerationResults()); CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = Mockito.mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); sut.populateAppData(GENERATION_ID, SOURCE_KEY, dataConsumers); - verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("pathwayAnalysis.json"), any(PathwayAnalysisDTO.class)); - verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("pathwayAnalysisResult.json"), any(PathwayAnalysisResult.class)); + verify(dataConsumers.getTextFiles(), times(1)).accept(eq("design.json"), anyString()); + verify(dataConsumers.getJsonObjects(), times(1)).accept(eq("chartData.json"), any(PathwayPopulationResultsDTO.class)); + } + + private PathwayPopulationResultsDTO createPathwayGenerationResults() { + return new PathwayPopulationResultsDTO(Collections.emptyList(), Collections.emptyList()); } @Test From 35fe82477db3903611b2a947baf0ee2b3954fe3e Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 20 Sep 2024 13:46:10 +0200 Subject: [PATCH 28/32] Fix for eliminating statefulness between Shiny apps generation calls --- .../shiny/CommonShinyPackagingService.java | 53 ++++++++----------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java index e58ee552ad..6bd1631a5f 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java @@ -24,14 +24,11 @@ public abstract class CommonShinyPackagingService { private static final Logger LOG = LoggerFactory.getLogger(CommonShinyPackagingService.class); protected final String atlasUrl; protected String repoLink; + protected final FileWriter fileWriter; protected final ManifestUtils manifestUtils; protected final ObjectMapper objectMapper; - private final Map applicationProperties = new HashMap<>(); - private final Map jsonObjectsToSave = new HashMap<>(); - private final Map textFilesToSave = new HashMap<>(); - public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper) { this.atlasUrl = atlasUrl; this.repoLink = repoLink; @@ -77,37 +74,27 @@ public ObjectMapper getObjectMapper() { return objectMapper; } - public Map getApplicationProperties() { - return applicationProperties; - } - - public Map getJsonObjectsToSave() { - return jsonObjectsToSave; - } - - public Map getTextFilesToSave() { - return textFilesToSave; - } - class ShinyAppDataConsumers { - private final BiConsumer appProperties = getApplicationProperties()::put; - private final BiConsumer textFiles = getTextFilesToSave()::put; - private final BiConsumer jsonObjects = getJsonObjectsToSave()::put; + private final Map applicationProperties = new HashMap<>(); + private final Map jsonObjectsToSave = new HashMap<>(); + private final Map textFilesToSave = new HashMap<>(); + private final BiConsumer appPropertiesConsumer = applicationProperties::put; + private final BiConsumer textFilesConsumer = textFilesToSave::put; + private final BiConsumer jsonObjectsConsumer = jsonObjectsToSave::put; public BiConsumer getAppProperties() { - return appProperties; + return appPropertiesConsumer; } public BiConsumer getTextFiles() { - return textFiles; + return textFilesConsumer; } public BiConsumer getJsonObjects() { - return jsonObjects; + return jsonObjectsConsumer; } } - public final TemporaryFile packageApp(Integer generationId, String sourceKey, PackagingStrategy packaging) { return TempFileUtils.doInDirectory(path -> { try { @@ -122,23 +109,25 @@ public final TemporaryFile packageApp(Integer generationId, String sourceKey, Pa Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); + ShinyAppDataConsumers shinyAppDataConsumers = new ShinyAppDataConsumers(); + //Default properties common for each shiny app - getApplicationProperties().put("repo_link", getRepoLink()); - getApplicationProperties().put("atlas_url", getAtlasUrl()); - getApplicationProperties().put("datasource", sourceKey); + shinyAppDataConsumers.applicationProperties.put("repo_link", getRepoLink()); + shinyAppDataConsumers.applicationProperties.put("atlas_url", getAtlasUrl()); + shinyAppDataConsumers.applicationProperties.put("datasource", sourceKey); - populateAppData(generationId, sourceKey, new ShinyAppDataConsumers()); + populateAppData(generationId, sourceKey, shinyAppDataConsumers); - Stream textFilesPaths = getTextFilesToSave().entrySet() + Stream textFilesPaths = shinyAppDataConsumers.textFilesToSave.entrySet() .stream() .map(entry -> getFileWriter().writeTextFile(dataDir.resolve(entry.getKey()), pw -> pw.print(entry.getValue()))); - Stream jsonFilesPaths = getJsonObjectsToSave().entrySet() + Stream jsonFilesPaths = shinyAppDataConsumers.jsonObjectsToSave.entrySet() .stream() .map(entry -> getFileWriter().writeObjectAsJsonFile(dataDir, entry.getValue(), entry.getKey())); Stream appPropertiesFilePath = Stream.of( - getFileWriter().writeTextFile(dataDir.resolve("app.properties"), pw -> pw.print(prepareAppProperties(applicationProperties))) + getFileWriter().writeTextFile(dataDir.resolve("app.properties"), pw -> pw.print(convertAppPropertiesToString(shinyAppDataConsumers.applicationProperties))) ); Stream.of(textFilesPaths, jsonFilesPaths, appPropertiesFilePath) @@ -156,8 +145,8 @@ public final TemporaryFile packageApp(Integer generationId, String sourceKey, Pa }); } - private String prepareAppProperties(Map appProperties) { - return getApplicationProperties().entrySet().stream() + private String convertAppPropertiesToString(Map appProperties) { + return appProperties.entrySet().stream() .map(entry -> String.format("%s=%s\n", entry.getKey(), entry.getValue())) .collect(Collectors.joining()); } From 4309e6686ee60e86223da9462cabf1a34acb8b46 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Mon, 16 Sep 2024 17:53:58 +0200 Subject: [PATCH 29/32] Added parameters required for the Shiny Apps introduction pages --- .../service/CohortDefinitionService.java | 19 ++- ...CharacterizationShinyPackagingService.java | 55 +++++++- .../CohortCountsShinyPackagingService.java | 81 ++++++++++- .../CohortPathwaysShinyPackagingService.java | 71 +++++++++- .../shiny/CommonShinyPackagingService.java | 57 +++++++- .../IncidenceRatesShinyPackagingService.java | 87 +++++++++++- .../shiny/summary/DataSourceSummary.java | 67 ++++++++++ .../summary/DataSourceSummaryConverter.java | 126 ++++++++++++++++++ ...hortPathwaysShinyPackagingServiceTest.java | 6 + ...cidenceRatesShinyPackagingServiceTest.java | 7 +- 10 files changed, 552 insertions(+), 24 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java create mode 100644 src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index 4de4e872e6..4f496f5f25 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -900,7 +900,7 @@ public CheckResult runDiagnosticsWithTags(CohortDTO cohortDTO) { @Path("/printfriendly/cohort") @Consumes(MediaType.APPLICATION_JSON) public Response cohortPrintFriendly(CohortExpression expression, @DefaultValue("html") @QueryParam("format") String format) { - String markdown = markdownPF.renderCohort(expression); + String markdown = convertCohortExpressionToMarkdown(expression); return printFrindly(markdown, format); } @@ -926,16 +926,23 @@ public Response conceptSetListPrintFriendly(List conceptSetList, @De return printFrindly(markdown, format); } + public String convertCohortExpressionToMarkdown(CohortExpression expression){ + return markdownPF.renderCohort(expression); + } + + public String convertMarkdownToHTML(String markdown){ + Parser parser = Parser.builder().extensions(extensions).build(); + Node document = parser.parse(markdown); + HtmlRenderer renderer = HtmlRenderer.builder().extensions(extensions).build(); + return renderer.render(document); + } + private Response printFrindly(String markdown, String format) { ResponseBuilder res; if ("html".equalsIgnoreCase(format)) { - Parser parser = Parser.builder().extensions(extensions).build(); - Node document = parser.parse(markdown); - HtmlRenderer renderer = HtmlRenderer.builder().extensions(extensions).build(); - String html = renderer.render(document); + String html = convertMarkdownToHTML(markdown); res = Response.ok(html, MediaType.TEXT_HTML); - } else if ("markdown".equals(format)) { res = Response.ok(markdown, MediaType.TEXT_PLAIN); } else { diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index 8712b0dde5..bfd9c57481 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -11,12 +11,17 @@ import org.ohdsi.analysis.WithId; import org.ohdsi.analysis.cohortcharacterization.design.CohortCharacterization; import org.ohdsi.webapi.cohortcharacterization.CcService; +import org.ohdsi.webapi.cohortcharacterization.domain.CcGenerationEntity; import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; import org.ohdsi.webapi.cohortcharacterization.dto.ExecutionResultRequest; import org.ohdsi.webapi.cohortcharacterization.dto.GenerationResults; import org.ohdsi.webapi.cohortcharacterization.report.ExportItem; import org.ohdsi.webapi.cohortcharacterization.report.Report; +import org.ohdsi.webapi.cohortdefinition.CohortDefinition; +import org.ohdsi.webapi.service.CDMResultsService; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,8 +59,11 @@ public CohortCharacterizationShinyPackagingService( ManifestUtils manifestUtils, ObjectMapper objectMapper, CcService ccService, - CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper) { - super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + CohortCharacterizationAnalysisHeaderToFieldMapper cohortCharacterizationAnalysisHeaderToFieldMapper, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); this.ccService = ccService; this.cohortCharacterizationAnalysisHeaderToFieldMapper = cohortCharacterizationAnalysisHeaderToFieldMapper; } @@ -75,6 +83,7 @@ public String getAppTemplateFilePath() { @Transactional public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { CohortCharacterization cohortCharacterization = ccService.findDesignByGenerationId(Long.valueOf(generationId)); + CohortCharacterizationEntity cohortCharacterizationEntity = ccService.findById(cohortCharacterization.getId()); GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); @@ -85,6 +94,48 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData .stream() .map(this::convertReportToCSV) .forEach(csvDataByFilename -> dataConsumers.getTextFiles().accept(csvDataByFilename.getKey(), csvDataByFilename.getValue())); + + CcGenerationEntity generationEntity = ccService.findGenerationById(Long.valueOf(generationId)); + + Long resultsTotalCount = ccService.getCCResultsTotalCount(Long.valueOf(generationId)); + + dataConsumers.getAppProperties().accept("author", getAuthor(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ASSET_ID_KEY, cohortCharacterization.getId().toString()); + dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept("record_count", Long.toString(resultsTotalCount)); + dataConsumers.getAppProperties().accept("author_notes", getDescription(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept("referenced_cohorts", getReferencedCohorts(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept("version_id", Integer.toString(generationId)); + dataConsumers.getAppProperties().accept("generation_id", Integer.toString(generationId)); + } + + private String getReferencedCohorts(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity != null) { + return cohortCharacterizationEntity.getCohortDefinitions().stream().map(CohortDefinition::getName).collect(Collectors.joining("; ")); + } + return VALUE_NOT_AVAILABLE; + } + + + private String getAuthor(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity.getCreatedBy() != null) { + return cohortCharacterizationEntity.getCreatedBy().getLogin(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getGenerationStartTime(CcGenerationEntity generation) { + if (generation != null) { + return dateToString(generation.getStartTime()); + } + return VALUE_NOT_AVAILABLE; + } + + private String getDescription(CohortCharacterizationEntity cohortCharacterizationEntity) { + if (cohortCharacterizationEntity != null && cohortCharacterizationEntity.getDescription() != null) { + return cohortCharacterizationEntity.getDescription(); + } + return VALUE_NOT_AVAILABLE; } //Pair.left == CSV filename diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index 845241d329..51f2d4333d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -2,11 +2,17 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.ohdsi.circe.cohortdefinition.CohortExpression; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfo; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoId; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoRepository; import org.ohdsi.webapi.cohortdefinition.InclusionRuleReport; +import org.ohdsi.webapi.service.CDMResultsService; import org.ohdsi.webapi.service.CohortDefinitionService; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -18,12 +24,13 @@ @Service @ConditionalOnBean(ShinyService.class) public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { + private static final String SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCounts.zip"; private static final String APP_TITLE_FORMAT = "Cohort_%s_gv%sx%s_%s"; private final CohortDefinitionService cohortDefinitionService; private final CohortDefinitionRepository cohortDefinitionRepository; - private final SourceRepository sourceRepository; + private final CohortGenerationInfoRepository cohortGenerationInfoRepository; @Autowired public CohortCountsShinyPackagingService( @@ -31,11 +38,18 @@ public CohortCountsShinyPackagingService( @Value("${shiny.repo.link}") String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, - ObjectMapper objectMapper, CohortDefinitionService cohortDefinitionService, CohortDefinitionRepository cohortDefinitionRepository, SourceRepository sourceRepository) { - super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + ObjectMapper objectMapper, + CohortDefinitionService cohortDefinitionService, + CohortDefinitionRepository cohortDefinitionRepository, + SourceRepository sourceRepository, + CohortGenerationInfoRepository cohortGenerationInfoRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter + ) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); this.cohortDefinitionService = cohortDefinitionService; this.cohortDefinitionRepository = cohortDefinitionRepository; - this.sourceRepository = sourceRepository; + this.cohortGenerationInfoRepository = cohortGenerationInfoRepository; } @Override @@ -54,8 +68,28 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); ExceptionUtils.throwNotFoundExceptionIfNull(cohort, String.format("There is no cohort definition with id = %d.", generationId)); + int sourceId = getSourceRepository().findBySourceKey(sourceKey).getId(); + CohortGenerationInfo cohortGenerationInfo = cohortGenerationInfoRepository.findOne(new CohortGenerationInfoId(cohort.getId(), sourceId)); + + CohortExpression cohortExpression = cohort.getExpression(); + + String cohortSummaryAsMarkdown = cohortDefinitionService.convertCohortExpressionToMarkdown(cohortExpression); + String cohortSummaryAsHTML = cohortDefinitionService.convertMarkdownToHTML(cohortSummaryAsMarkdown); + dataConsumers.getAppProperties().accept("cohort_link", String.format("%s/#/cohortdefinition/%s", atlasUrl, generationId)); dataConsumers.getAppProperties().accept("cohort_name", cohort.getName()); + dataConsumers.getAppProperties().accept("author", getAuthor(cohort)); + dataConsumers.getAppProperties().accept(ASSET_ID_KEY, cohort.getId().toString()); + dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept("record_count", getRecordCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept("person_count", getPersonCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept("author_notes", getDescription(cohort)); + dataConsumers.getAppProperties().accept("referenced_cohorts", cohort.getName()); + dataConsumers.getAppProperties().accept("version_id", getGenerationId(cohortGenerationInfo.getId())); + dataConsumers.getAppProperties().accept("generation_id", getGenerationId(cohortGenerationInfo.getId())); + + dataConsumers.getTextFiles().accept("cohort_summary_markdown.txt", cohortSummaryAsMarkdown); + dataConsumers.getTextFiles().accept("cohort_summary.html", cohortSummaryAsHTML); InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person @@ -64,6 +98,45 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData dataConsumers.getJsonObjects().accept(sourceKey + "_by_person.json", byPersonReport); } + private String getGenerationId(CohortGenerationInfoId id) { + return id == null ? "" : Integer.toString(id.getCohortDefinitionId()).concat("x").concat(Integer.toString(id.getSourceId())); + } + + private String getDescription(CohortDefinition cohort) { + if (cohort != null && cohort.getDescription() != null) { + return cohort.getDescription(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getPersonCount(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getPersonCount() != null) { + return cohortGenerationInfo.getPersonCount().toString(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getRecordCount(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getRecordCount() != null) { + return cohortGenerationInfo.getRecordCount().toString(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getGenerationStartTime(CohortGenerationInfo cohortGenerationInfo) { + if (cohortGenerationInfo != null && cohortGenerationInfo.getStartTime() != null) { + return dateToString(cohortGenerationInfo.getStartTime()); + } + return VALUE_NOT_AVAILABLE; + } + + private String getAuthor(CohortDefinition cohort) { + if (cohort.getCreatedBy() != null) { + return cohort.getCreatedBy().getLogin(); + } + return VALUE_NOT_AVAILABLE; + } + @Override public ApplicationBrief getBrief(Integer generationId, String sourceKey) { CohortDefinition cohort = cohortDefinitionRepository.findOne(generationId); diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index ab0ec04ffc..5b70d4bc28 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -3,15 +3,24 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; +import org.ohdsi.webapi.pathway.dto.PathwayCohortDTO; import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; +import org.ohdsi.webapi.pathway.dto.TargetCohortPathwaysDTO; +import org.ohdsi.webapi.service.CDMResultsService; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; +import java.util.HashSet; +import java.util.Set; + @Service @ConditionalOnBean(ShinyService.class) public class CohortPathwaysShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { @@ -26,8 +35,11 @@ public CohortPathwaysShinyPackagingService( @Value("${shiny.repo.link}") String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, - ObjectMapper objectMapper, PathwayService pathwayService) { - super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + ObjectMapper objectMapper, PathwayService pathwayService, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); this.pathwayService = pathwayService; } @@ -45,12 +57,63 @@ public String getAppTemplateFilePath() { public void populateAppData(Integer generationId, String sourceKey, ShinyAppDataConsumers dataConsumers) { String designJSON = pathwayService.findDesignByGenerationId(generationId.longValue()); PathwayPopulationResultsDTO generationResults = pathwayService.getGenerationResults(generationId.longValue()); - ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no pathway analysis generation results with generation id = %d.", generationId)); ExceptionUtils.throwNotFoundExceptionIfNull(designJSON, String.format("There is no pathway analysis design with generation id = %d.", generationId)); - dataConsumers.getTextFiles().accept("design.json", designJSON); dataConsumers.getJsonObjects().accept("chartData.json", generationResults); + + PathwayAnalysisDTO pathwayAnalysisDTO = pathwayService.getByGenerationId(generationId); + PathwayAnalysisGenerationEntity generationEntity = pathwayService.getGeneration(generationId.longValue()); + + int totalCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTargetCohortCount).sum(); + int personCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTotalPathwaysCount).sum(); + + dataConsumers.getAppProperties().accept("author", getAuthor(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept("asset_name", pathwayAnalysisDTO.getName()); + dataConsumers.getAppProperties().accept(ASSET_ID_KEY, pathwayAnalysisDTO.getId().toString()); + dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept("record_count", Integer.toString(totalCount)); + dataConsumers.getAppProperties().accept("person_count", Integer.toString(personCount)); + dataConsumers.getAppProperties().accept("author_notes", getDescription(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept("referenced_cohorts", prepareReferencedCohorts(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept("version_id", Integer.toString(generationId)); + dataConsumers.getAppProperties().accept("generation_id", Integer.toString(generationId)); + } + + private String getAuthor(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO.getCreatedBy() != null) { + return pathwayAnalysisDTO.getCreatedBy().getLogin(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getDescription(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO != null && pathwayAnalysisDTO.getDescription() != null) { + return pathwayAnalysisDTO.getDescription(); + } + return VALUE_NOT_AVAILABLE; + } + + + private String prepareReferencedCohorts(PathwayAnalysisDTO pathwayAnalysisDTO) { + if (pathwayAnalysisDTO == null) { + return VALUE_NOT_AVAILABLE; + } + Set referencedCohortNames = new HashSet<>(); + for (PathwayCohortDTO eventCohort : pathwayAnalysisDTO.getEventCohorts()) { + referencedCohortNames.add(eventCohort.getName()); + } + for (PathwayCohortDTO targetCohort : pathwayAnalysisDTO.getTargetCohorts()) { + referencedCohortNames.add(targetCohort.getName()); + } + return String.join("; ", referencedCohortNames); + } + + private String getGenerationStartTime(PathwayAnalysisGenerationEntity generationEntity) { + if (generationEntity != null) { + return dateToString(generationEntity.getStartTime()); + } + return VALUE_NOT_AVAILABLE; } @Override diff --git a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java index 6bd1631a5f..07a8f489d8 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java @@ -4,6 +4,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; import com.odysseusinc.arachne.execution_engine_common.util.CommonFileUtils; +import org.ohdsi.webapi.report.CDMDashboard; +import org.ohdsi.webapi.service.CDMResultsService; +import org.ohdsi.webapi.shiny.summary.DataSourceSummary; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.TempFileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +19,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.HashMap; import java.util.Map; import java.util.function.BiConsumer; @@ -21,6 +30,9 @@ import java.util.stream.Stream; public abstract class CommonShinyPackagingService { + protected static final String VALUE_NOT_AVAILABLE = "N/A"; + private static final String DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + private static final Logger LOG = LoggerFactory.getLogger(CommonShinyPackagingService.class); protected final String atlasUrl; protected String repoLink; @@ -28,13 +40,20 @@ public abstract class CommonShinyPackagingService { protected final FileWriter fileWriter; protected final ManifestUtils manifestUtils; protected final ObjectMapper objectMapper; + protected final SourceRepository sourceRepository; + protected final CDMResultsService cdmResultsService; + protected final DataSourceSummaryConverter dataSourceSummaryConverter; + protected static final String ASSET_ID_KEY = "asset_id"; - public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper) { + public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper, SourceRepository sourceRepository, CDMResultsService cdmResultsService, DataSourceSummaryConverter dataSourceSummaryConverter) { this.atlasUrl = atlasUrl; this.repoLink = repoLink; this.fileWriter = fileWriter; this.manifestUtils = manifestUtils; this.objectMapper = objectMapper; + this.sourceRepository = sourceRepository; + this.cdmResultsService = cdmResultsService; + this.dataSourceSummaryConverter = dataSourceSummaryConverter; } public abstract CommonAnalysisType getType(); @@ -74,6 +93,19 @@ public ObjectMapper getObjectMapper() { return objectMapper; } + public SourceRepository getSourceRepository() { + return sourceRepository; + } + + public CDMResultsService getCdmResultsService() { + return cdmResultsService; + } + + public DataSourceSummaryConverter getDataSourceSummaryConverter() { + return dataSourceSummaryConverter; + } + + class ShinyAppDataConsumers { private final Map applicationProperties = new HashMap<>(); private final Map jsonObjectsToSave = new HashMap<>(); @@ -109,12 +141,17 @@ public final TemporaryFile packageApp(Integer generationId, String sourceKey, Pa Path dataDir = path.resolve("data"); Files.createDirectory(dataDir); + Source source = getSourceRepository().findBySourceKey(sourceKey); + ShinyAppDataConsumers shinyAppDataConsumers = new ShinyAppDataConsumers(); //Default properties common for each shiny app shinyAppDataConsumers.applicationProperties.put("repo_link", getRepoLink()); shinyAppDataConsumers.applicationProperties.put("atlas_url", getAtlasUrl()); shinyAppDataConsumers.applicationProperties.put("datasource", sourceKey); + shinyAppDataConsumers.applicationProperties.put("datasource_name", source.getSourceName()); + + populateCDMDataSourceSummaryIfPresent(source, shinyAppDataConsumers); populateAppData(generationId, sourceKey, shinyAppDataConsumers); @@ -145,9 +182,27 @@ public final TemporaryFile packageApp(Integer generationId, String sourceKey, Pa }); } + private void populateCDMDataSourceSummaryIfPresent(Source source, ShinyAppDataConsumers shinyAppDataConsumers) { + DataSourceSummary dataSourceSummary; + try { + CDMDashboard cdmDashboard = getCdmResultsService().getDashboard(source.getSourceKey()); + dataSourceSummary = getDataSourceSummaryConverter().convert(cdmDashboard); + } catch (Exception e) { + LOG.warn("Could not populate datasource summary", e); + dataSourceSummary = getDataSourceSummaryConverter().emptySummary(source.getSourceName()); + } + shinyAppDataConsumers.jsonObjectsToSave.put("datasource_summary.json", dataSourceSummary); + } + private String convertAppPropertiesToString(Map appProperties) { return appProperties.entrySet().stream() .map(entry -> String.format("%s=%s\n", entry.getKey(), entry.getValue())) .collect(Collectors.joining()); } + + protected String dateToString(Date date) { + if (date == null) return null; + DateFormat df = new SimpleDateFormat(DATE_TIME_FORMAT); + return df.format(date); + } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index c7e919f5cd..c7310955d8 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -2,7 +2,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Iterables; import com.odysseusinc.arachne.commons.api.v1.dto.CommonAnalysisType; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; import org.apache.commons.csv.QuoteMode; @@ -11,8 +13,11 @@ import org.ohdsi.webapi.ircalc.IncidenceRateAnalysis; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisExportExpression; import org.ohdsi.webapi.ircalc.IncidenceRateAnalysisRepository; +import org.ohdsi.webapi.service.CDMResultsService; import org.ohdsi.webapi.service.IRAnalysisResource; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.service.dto.AnalysisInfoDTO; +import org.ohdsi.webapi.shiny.summary.DataSourceSummaryConverter; import org.ohdsi.webapi.source.SourceRepository; import org.ohdsi.webapi.util.ExceptionUtils; import org.slf4j.Logger; @@ -26,7 +31,9 @@ import javax.ws.rs.InternalServerErrorException; import java.io.IOException; import java.io.StringWriter; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Stream; @Service @@ -40,8 +47,6 @@ public class IncidenceRatesShinyPackagingService extends CommonShinyPackagingSer private final IncidenceRateAnalysisRepository incidenceRateAnalysisRepository; private final IRAnalysisResource irAnalysisResource; - private final SourceRepository sourceRepository; - @Autowired public IncidenceRatesShinyPackagingService( @Value("${shiny.atlas.url}") String atlasUrl, @@ -50,11 +55,13 @@ public IncidenceRatesShinyPackagingService( ManifestUtils manifestUtils, ObjectMapper objectMapper, IncidenceRateAnalysisRepository incidenceRateAnalysisRepository, - IRAnalysisResource irAnalysisResource, SourceRepository sourceRepository) { - super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper); + IRAnalysisResource irAnalysisResource, + SourceRepository sourceRepository, + CDMResultsService cdmResultsService, + DataSourceSummaryConverter dataSourceSummaryConverter) { + super(atlasUrl, repoLink, fileWriter, manifestUtils, objectMapper, sourceRepository, cdmResultsService, dataSourceSummaryConverter); this.incidenceRateAnalysisRepository = incidenceRateAnalysisRepository; this.irAnalysisResource = irAnalysisResource; - this.sourceRepository = sourceRepository; } @Override @@ -77,6 +84,21 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData dataConsumers.getAppProperties().accept("analysis_name", analysis.getName()); IncidenceRateAnalysisExportExpression expression = objectMapper.readValue(analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); + AnalysisInfoDTO analysisInfoDTO = irAnalysisResource.getAnalysisInfo(analysis.getId(), sourceKey); + + Integer assetId = analysis.getId(); + Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); + + dataConsumers.getAppProperties().accept("author", getAuthor(analysis)); + dataConsumers.getAppProperties().accept(ASSET_ID_KEY, analysis.getId().toString()); + dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(analysis)); + dataConsumers.getAppProperties().accept("record_count", getRecordCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept("person_count", getPersonCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept("author_notes", getDescription(analysis)); + dataConsumers.getAppProperties().accept("referenced_cohorts", prepareReferencedCohorts(expression)); + dataConsumers.getAppProperties().accept("version_id", getGenerationId(assetId, sourceId)); + dataConsumers.getAppProperties().accept("generation_id", getGenerationId(assetId, sourceId)); + String csvWithCohortDetails = prepareCsvWithCohorts(expression); dataConsumers.getTextFiles().accept("cohorts.csv", csvWithCohortDetails); @@ -93,6 +115,61 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData } } + private String getAuthor(IncidenceRateAnalysis analysis) { + if (analysis.getCreatedBy() != null) { + return analysis.getCreatedBy().getLogin(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getGenerationStartTime(IncidenceRateAnalysis analysis) { + if (analysis != null) { + if (CollectionUtils.isNotEmpty(analysis.getExecutionInfoList())) { + return dateToString(Iterables.getLast(analysis.getExecutionInfoList()).getStartTime()); + } + } + return VALUE_NOT_AVAILABLE; + } + + private String getDescription(IncidenceRateAnalysis analysis) { + if (analysis != null && analysis.getDescription() != null) { + return analysis.getDescription(); + } + return VALUE_NOT_AVAILABLE; + } + + private String getPersonCount(AnalysisInfoDTO analysisInfo) { + if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { + return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).cases); + } + return VALUE_NOT_AVAILABLE; + } + + private String getRecordCount(AnalysisInfoDTO analysisInfo) { + if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { + return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).totalPersons); + } + return VALUE_NOT_AVAILABLE; + } + + private String getGenerationId(Integer assetId, Integer sourceId) { + return assetId == null || sourceId == null ? "" : Integer.toString(assetId).concat("x").concat(Integer.toString(sourceId)); + } + + private String prepareReferencedCohorts(IncidenceRateAnalysisExportExpression expression) { + if (expression == null) { + return ""; + } + Set referencedCohortNames = new HashSet<>(); + for (CohortDTO targetCohort : expression.targetCohorts) { + referencedCohortNames.add(targetCohort.getName()); + } + for (CohortDTO outcomeCohort : expression.outcomeCohorts) { + referencedCohortNames.add(outcomeCohort.getName()); + } + return String.join("; ", referencedCohortNames); + } + private Stream streamAnalysisReportsForAllCohortCombinations(IncidenceRateAnalysisExportExpression expression, Integer analysisId, String sourceKey) { List targetCohorts = expression.targetCohorts; List outcomeCohorts = expression.outcomeCohorts; diff --git a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java new file mode 100644 index 0000000000..aa8e8b00e0 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummary.java @@ -0,0 +1,67 @@ +package org.ohdsi.webapi.shiny.summary; + +public class DataSourceSummary { + private String sourceName; + private String numberOfPersons; + private String female; + private String male; + private String ageAtFirstObservation; + private String cumulativeObservation; + private String continuousObservationCoverage; + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + + public void setNumberOfPersons(String numberOfPersons) { + this.numberOfPersons = numberOfPersons; + } + + public void setFemale(String female) { + this.female = female; + } + + public void setMale(String male) { + this.male = male; + } + + public void setAgeAtFirstObservation(String ageAtFirstObservation) { + this.ageAtFirstObservation = ageAtFirstObservation; + } + + public void setCumulativeObservation(String cumulativeObservation) { + this.cumulativeObservation = cumulativeObservation; + } + + public void setContinuousObservationCoverage(String continuousObservationCoverage) { + this.continuousObservationCoverage = continuousObservationCoverage; + } + + public String getSourceName() { + return sourceName; + } + + public String getNumberOfPersons() { + return numberOfPersons; + } + + public String getFemale() { + return female; + } + + public String getMale() { + return male; + } + + public String getAgeAtFirstObservation() { + return ageAtFirstObservation; + } + + public String getCumulativeObservation() { + return cumulativeObservation; + } + + public String getContinuousObservationCoverage() { + return continuousObservationCoverage; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java new file mode 100644 index 0000000000..9f37d65e31 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java @@ -0,0 +1,126 @@ +package org.ohdsi.webapi.shiny.summary; + +import org.ohdsi.webapi.report.CDMAttribute; +import org.ohdsi.webapi.report.CDMDashboard; +import org.ohdsi.webapi.report.ConceptCountRecord; +import org.ohdsi.webapi.report.ConceptDistributionRecord; +import org.ohdsi.webapi.report.CumulativeObservationRecord; +import org.ohdsi.webapi.report.MonthObservationRecord; +import org.ohdsi.webapi.service.ShinyService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; + +import java.text.DecimalFormat; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@ConditionalOnBean(ShinyService.class) +public class DataSourceSummaryConverter { + + private static final String VALUE_NOT_AVAILABLE = "N/A"; + + private double calculateVariance(List values, double mean) { + double variance = 0; + for (double value : values) { + variance += Math.pow(value - mean, 2); + } + return variance / values.size(); + } + + public DataSourceSummary convert(CDMDashboard cdmDashboard) { + DataSourceSummary dataSourceSummary = new DataSourceSummary(); + + if (cdmDashboard.getSummary() != null) { + for (CDMAttribute attribute : cdmDashboard.getSummary()) { + switch (attribute.getAttributeName()) { + case "Source name": + dataSourceSummary.setSourceName(attribute.getAttributeValue()); + break; + case "Number of persons": + double number = Double.parseDouble(attribute.getAttributeValue()); + String formattedNumber = new DecimalFormat("#,###.###M").format(number / 1_000_000); + dataSourceSummary.setNumberOfPersons(formattedNumber); + break; + } + } + } + + if (cdmDashboard.getGender() != null) { + long maleCount = 0; + long femaleCount = 0; + for (ConceptCountRecord record : cdmDashboard.getGender()) { + if (record.getConceptName().equalsIgnoreCase("MALE")) { + maleCount = record.getCountValue(); + } else if (record.getConceptName().equalsIgnoreCase("FEMALE")) { + femaleCount = record.getCountValue(); + } + } + long totalGenderCount = maleCount + femaleCount; + String malePercentage = String.format("%,.1f %%", 100 * (double) maleCount / totalGenderCount); + String femalePercentage = String.format("%,.1f %%", 100 * (double) femaleCount / totalGenderCount); + dataSourceSummary.setMale(String.format("%,d (%s)", maleCount, malePercentage)); + dataSourceSummary.setFemale(String.format("%,d (%s)", femaleCount, femalePercentage)); + } + + if (cdmDashboard.getAgeAtFirstObservation() != null) { + List percents = cdmDashboard.getAgeAtFirstObservation().stream() + .map(ConceptDistributionRecord::getPercentValue) + .collect(Collectors.toList()); + double sum = percents.stream().mapToDouble(Double::doubleValue).sum(); + double mean = sum / percents.size(); + double variance = calculateVariance(percents, mean); + + int minYear = cdmDashboard.getAgeAtFirstObservation().stream() + .min(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) + .orElse(new ConceptDistributionRecord()).getIntervalIndex(); + int maxYear = cdmDashboard.getAgeAtFirstObservation().stream() + .max(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) + .orElse(new ConceptDistributionRecord()).getIntervalIndex(); + dataSourceSummary.setAgeAtFirstObservation(String.format("[%d - %d] (M= %.1f; SD= %.1f)", + minYear, maxYear, mean, Math.sqrt(variance))); + } + + if (cdmDashboard.getCumulativeObservation() != null) { + List percentPersons = cdmDashboard.getCumulativeObservation().stream() + .map(CumulativeObservationRecord::getyPercentPersons) + .collect(Collectors.toList()); + double sum = percentPersons.stream().mapToDouble(Double::doubleValue).sum(); + double mean = sum / percentPersons.size(); + double variance = calculateVariance(percentPersons, mean); + + int minObs = cdmDashboard.getCumulativeObservation().stream() + .min(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) + .orElse(new CumulativeObservationRecord()).getxLengthOfObservation(); + int maxObs = cdmDashboard.getCumulativeObservation().stream() + .max(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) + .orElse(new CumulativeObservationRecord()).getxLengthOfObservation(); + dataSourceSummary.setCumulativeObservation(String.format("[%d - %d] (M= %.1f; SD= %.1f)", + minObs, maxObs, mean, Math.sqrt(variance))); + } + + if (cdmDashboard.getObservedByMonth() != null && !cdmDashboard.getObservedByMonth().isEmpty()) { + MonthObservationRecord startRecord = cdmDashboard.getObservedByMonth().get(0); + MonthObservationRecord endRecord = cdmDashboard.getObservedByMonth() + .get(cdmDashboard.getObservedByMonth().size() - 1); + dataSourceSummary.setContinuousObservationCoverage(String.format("Start: %02d/%02d, End: %02d/%02d", + startRecord.getMonthYear() % 100, startRecord.getMonthYear() / 100, + endRecord.getMonthYear() % 100, endRecord.getMonthYear() / 100)); + } + + return dataSourceSummary; + } + + public DataSourceSummary emptySummary(String dataSourceName) { + DataSourceSummary dataSourceSummary = new DataSourceSummary(); + dataSourceSummary.setSourceName(dataSourceName); + dataSourceSummary.setFemale(VALUE_NOT_AVAILABLE); + dataSourceSummary.setMale(VALUE_NOT_AVAILABLE); + dataSourceSummary.setCumulativeObservation(VALUE_NOT_AVAILABLE); + dataSourceSummary.setAgeAtFirstObservation(VALUE_NOT_AVAILABLE); + dataSourceSummary.setContinuousObservationCoverage(VALUE_NOT_AVAILABLE); + dataSourceSummary.setNumberOfPersons(VALUE_NOT_AVAILABLE); + return dataSourceSummary; + } +} diff --git a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java index f40480d5f0..f01e05bb39 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingServiceTest.java @@ -10,6 +10,7 @@ import org.mockito.Spy; import org.mockito.runners.MockitoJUnitRunner; import org.ohdsi.webapi.pathway.PathwayService; +import org.ohdsi.webapi.pathway.domain.PathwayAnalysisGenerationEntity; import org.ohdsi.webapi.pathway.dto.PathwayAnalysisDTO; import org.ohdsi.webapi.pathway.dto.PathwayPopulationResultsDTO; import org.ohdsi.webapi.pathway.dto.internal.PathwayAnalysisResult; @@ -56,6 +57,11 @@ public void shouldPopulateAppData() { when(pathwayService.findDesignByGenerationId(eq((long) GENERATION_ID))).thenReturn("design json"); when(pathwayService.getGenerationResults(eq((long) GENERATION_ID))).thenReturn(createPathwayGenerationResults()); + PathwayAnalysisDTO pathwayAnalysisDTO = Mockito.mock(PathwayAnalysisDTO.class); + PathwayAnalysisGenerationEntity generationEntity = Mockito.mock(PathwayAnalysisGenerationEntity.class); + when(pathwayService.getByGenerationId(eq(GENERATION_ID))).thenReturn(pathwayAnalysisDTO); + when(pathwayService.getGeneration(eq((long) GENERATION_ID))).thenReturn(generationEntity); + CommonShinyPackagingService.ShinyAppDataConsumers dataConsumers = Mockito.mock(CommonShinyPackagingService.ShinyAppDataConsumers.class, Answers.RETURNS_DEEP_STUBS.get()); sut.populateAppData(GENERATION_ID, SOURCE_KEY, dataConsumers); diff --git a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java index 2c36607fb5..8ca6193ceb 100644 --- a/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java +++ b/src/test/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingServiceTest.java @@ -25,7 +25,6 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; -import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; @@ -72,6 +71,10 @@ public void shouldPopulateAppDataWithValidData() throws JsonProcessingException Integer generationId = 1; String sourceKey = "source"; + Source source = new Source(); + source.setSourceId(3); + when(sourceRepository.findBySourceKey("source")).thenReturn(source); + IncidenceRateAnalysis analysis = Mockito.mock(IncidenceRateAnalysis.class, Answers.RETURNS_DEEP_STUBS.get()); when(analysis.getDetails().getExpression()).thenReturn("{}"); when(repository.findOne(generationId)).thenReturn(analysis); @@ -98,7 +101,7 @@ public void shouldPopulateAppDataWithValidData() throws JsonProcessingException sut.populateAppData(generationId, sourceKey, dataConsumers); - verify(dataConsumers.getAppProperties(), times(2)).accept(anyString(), anyString()); + verify(dataConsumers.getAppProperties(), times(11)).accept(anyString(), anyString()); verify(dataConsumers.getTextFiles(), times(1)).accept(anyString(), anyString()); verify(dataConsumers.getJsonObjects(), times(1)).accept(anyString(), any()); } From c94aa0cc2cf2263f58bf863a69776914d3157ed4 Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Fri, 11 Oct 2024 16:39:49 +0200 Subject: [PATCH 30/32] Exteacted common constants into a separate enum, changed datasource info parameters for Mean, StdDev, variance calculations --- ...CharacterizationShinyPackagingService.java | 28 ++++++------ .../CohortCountsShinyPackagingService.java | 38 ++++++++-------- .../CohortPathwaysShinyPackagingService.java | 28 ++++++------ .../shiny/CommonShinyPackagingService.java | 15 +++---- .../IncidenceRatesShinyPackagingService.java | 28 ++++++------ .../ohdsi/webapi/shiny/ShinyConstants.java | 34 +++++++++++++++ .../summary/DataSourceSummaryConverter.java | 43 ++++++++++--------- 7 files changed, 120 insertions(+), 94 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java index bfd9c57481..cdc7f8886d 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCharacterizationShinyPackagingService.java @@ -87,8 +87,8 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData GenerationResults generationResults = fetchGenerationResults(generationId, cohortCharacterization); ExceptionUtils.throwNotFoundExceptionIfNull(generationResults, String.format("There are no analysis generation results with generationId = %d.", generationId)); - dataConsumers.getAppProperties().accept("atlas_link", String.format("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())); - dataConsumers.getAppProperties().accept("analysis_name", cohortCharacterization.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/cc/characterizations/%s", atlasUrl, cohortCharacterization.getId())); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ANALYSIS_NAME.getValue(), cohortCharacterization.getName()); generationResults.getReports() .stream() @@ -99,21 +99,21 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData Long resultsTotalCount = ccService.getCCResultsTotalCount(Long.valueOf(generationId)); - dataConsumers.getAppProperties().accept("author", getAuthor(cohortCharacterizationEntity)); - dataConsumers.getAppProperties().accept(ASSET_ID_KEY, cohortCharacterization.getId().toString()); - dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(generationEntity)); - dataConsumers.getAppProperties().accept("record_count", Long.toString(resultsTotalCount)); - dataConsumers.getAppProperties().accept("author_notes", getDescription(cohortCharacterizationEntity)); - dataConsumers.getAppProperties().accept("referenced_cohorts", getReferencedCohorts(cohortCharacterizationEntity)); - dataConsumers.getAppProperties().accept("version_id", Integer.toString(generationId)); - dataConsumers.getAppProperties().accept("generation_id", Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), cohortCharacterization.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), Long.toString(resultsTotalCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), getReferencedCohorts(cohortCharacterizationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), Integer.toString(generationId)); } private String getReferencedCohorts(CohortCharacterizationEntity cohortCharacterizationEntity) { if (cohortCharacterizationEntity != null) { return cohortCharacterizationEntity.getCohortDefinitions().stream().map(CohortDefinition::getName).collect(Collectors.joining("; ")); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } @@ -121,21 +121,21 @@ private String getAuthor(CohortCharacterizationEntity cohortCharacterizationEnti if (cohortCharacterizationEntity.getCreatedBy() != null) { return cohortCharacterizationEntity.getCreatedBy().getLogin(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getGenerationStartTime(CcGenerationEntity generation) { if (generation != null) { return dateToString(generation.getStartTime()); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getDescription(CohortCharacterizationEntity cohortCharacterizationEntity) { if (cohortCharacterizationEntity != null && cohortCharacterizationEntity.getDescription() != null) { return cohortCharacterizationEntity.getDescription(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } //Pair.left == CSV filename diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java index 51f2d4333d..92a41d634f 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortCountsShinyPackagingService.java @@ -24,12 +24,10 @@ @Service @ConditionalOnBean(ShinyService.class) public class CohortCountsShinyPackagingService extends CommonShinyPackagingService implements ShinyPackagingService { - private static final String SHINY_COHORT_COUNTS_APP_TEMPLATE_FILE_PATH = "/shiny/shiny-cohortCounts.zip"; private static final String APP_TITLE_FORMAT = "Cohort_%s_gv%sx%s_%s"; private final CohortDefinitionService cohortDefinitionService; private final CohortDefinitionRepository cohortDefinitionRepository; - private final CohortGenerationInfoRepository cohortGenerationInfoRepository; @Autowired @@ -74,22 +72,20 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData CohortExpression cohortExpression = cohort.getExpression(); String cohortSummaryAsMarkdown = cohortDefinitionService.convertCohortExpressionToMarkdown(cohortExpression); - String cohortSummaryAsHTML = cohortDefinitionService.convertMarkdownToHTML(cohortSummaryAsMarkdown); - - dataConsumers.getAppProperties().accept("cohort_link", String.format("%s/#/cohortdefinition/%s", atlasUrl, generationId)); - dataConsumers.getAppProperties().accept("cohort_name", cohort.getName()); - dataConsumers.getAppProperties().accept("author", getAuthor(cohort)); - dataConsumers.getAppProperties().accept(ASSET_ID_KEY, cohort.getId().toString()); - dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(cohortGenerationInfo)); - dataConsumers.getAppProperties().accept("record_count", getRecordCount(cohortGenerationInfo)); - dataConsumers.getAppProperties().accept("person_count", getPersonCount(cohortGenerationInfo)); - dataConsumers.getAppProperties().accept("author_notes", getDescription(cohort)); - dataConsumers.getAppProperties().accept("referenced_cohorts", cohort.getName()); - dataConsumers.getAppProperties().accept("version_id", getGenerationId(cohortGenerationInfo.getId())); - dataConsumers.getAppProperties().accept("generation_id", getGenerationId(cohortGenerationInfo.getId())); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_COHORT_LINK.getValue(), String.format("%s/#/cohortdefinition/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_COHORT_NAME.getValue(), cohort.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(cohort)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), cohort.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), getRecordCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), getPersonCount(cohortGenerationInfo)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(cohort)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), cohort.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), getGenerationId(cohortGenerationInfo.getId())); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), getGenerationId(cohortGenerationInfo.getId())); dataConsumers.getTextFiles().accept("cohort_summary_markdown.txt", cohortSummaryAsMarkdown); - dataConsumers.getTextFiles().accept("cohort_summary.html", cohortSummaryAsHTML); InclusionRuleReport byEventReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 0); //by event InclusionRuleReport byPersonReport = cohortDefinitionService.getInclusionRuleReport(generationId, sourceKey, 1); //by person @@ -106,35 +102,35 @@ private String getDescription(CohortDefinition cohort) { if (cohort != null && cohort.getDescription() != null) { return cohort.getDescription(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getPersonCount(CohortGenerationInfo cohortGenerationInfo) { if (cohortGenerationInfo != null && cohortGenerationInfo.getPersonCount() != null) { return cohortGenerationInfo.getPersonCount().toString(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getRecordCount(CohortGenerationInfo cohortGenerationInfo) { if (cohortGenerationInfo != null && cohortGenerationInfo.getRecordCount() != null) { return cohortGenerationInfo.getRecordCount().toString(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getGenerationStartTime(CohortGenerationInfo cohortGenerationInfo) { if (cohortGenerationInfo != null && cohortGenerationInfo.getStartTime() != null) { return dateToString(cohortGenerationInfo.getStartTime()); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getAuthor(CohortDefinition cohort) { if (cohort.getCreatedBy() != null) { return cohort.getCreatedBy().getLogin(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } @Override diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 5b70d4bc28..0c5c7c486f 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -68,36 +68,36 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData int totalCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTargetCohortCount).sum(); int personCount = generationResults.getPathwayGroups().stream().mapToInt(TargetCohortPathwaysDTO::getTotalPathwaysCount).sum(); - dataConsumers.getAppProperties().accept("author", getAuthor(pathwayAnalysisDTO)); - dataConsumers.getAppProperties().accept("asset_name", pathwayAnalysisDTO.getName()); - dataConsumers.getAppProperties().accept(ASSET_ID_KEY, pathwayAnalysisDTO.getId().toString()); - dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(generationEntity)); - dataConsumers.getAppProperties().accept("record_count", Integer.toString(totalCount)); - dataConsumers.getAppProperties().accept("person_count", Integer.toString(personCount)); - dataConsumers.getAppProperties().accept("author_notes", getDescription(pathwayAnalysisDTO)); - dataConsumers.getAppProperties().accept("referenced_cohorts", prepareReferencedCohorts(pathwayAnalysisDTO)); - dataConsumers.getAppProperties().accept("version_id", Integer.toString(generationId)); - dataConsumers.getAppProperties().accept("generation_id", Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_NAME.getValue(), pathwayAnalysisDTO.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), pathwayAnalysisDTO.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(generationEntity)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), Integer.toString(totalCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), Integer.toString(personCount)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), prepareReferencedCohorts(pathwayAnalysisDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), Integer.toString(generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), Integer.toString(generationId)); } private String getAuthor(PathwayAnalysisDTO pathwayAnalysisDTO) { if (pathwayAnalysisDTO.getCreatedBy() != null) { return pathwayAnalysisDTO.getCreatedBy().getLogin(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getDescription(PathwayAnalysisDTO pathwayAnalysisDTO) { if (pathwayAnalysisDTO != null && pathwayAnalysisDTO.getDescription() != null) { return pathwayAnalysisDTO.getDescription(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String prepareReferencedCohorts(PathwayAnalysisDTO pathwayAnalysisDTO) { if (pathwayAnalysisDTO == null) { - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } Set referencedCohortNames = new HashSet<>(); for (PathwayCohortDTO eventCohort : pathwayAnalysisDTO.getEventCohorts()) { @@ -113,7 +113,7 @@ private String getGenerationStartTime(PathwayAnalysisGenerationEntity generation if (generationEntity != null) { return dateToString(generationEntity.getStartTime()); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } @Override diff --git a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java index 07a8f489d8..7bf0b04dc4 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CommonShinyPackagingService.java @@ -30,20 +30,15 @@ import java.util.stream.Stream; public abstract class CommonShinyPackagingService { - protected static final String VALUE_NOT_AVAILABLE = "N/A"; - private static final String DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; - private static final Logger LOG = LoggerFactory.getLogger(CommonShinyPackagingService.class); protected final String atlasUrl; protected String repoLink; - protected final FileWriter fileWriter; protected final ManifestUtils manifestUtils; protected final ObjectMapper objectMapper; protected final SourceRepository sourceRepository; protected final CDMResultsService cdmResultsService; protected final DataSourceSummaryConverter dataSourceSummaryConverter; - protected static final String ASSET_ID_KEY = "asset_id"; public CommonShinyPackagingService(String atlasUrl, String repoLink, FileWriter fileWriter, ManifestUtils manifestUtils, ObjectMapper objectMapper, SourceRepository sourceRepository, CDMResultsService cdmResultsService, DataSourceSummaryConverter dataSourceSummaryConverter) { this.atlasUrl = atlasUrl; @@ -146,10 +141,10 @@ public final TemporaryFile packageApp(Integer generationId, String sourceKey, Pa ShinyAppDataConsumers shinyAppDataConsumers = new ShinyAppDataConsumers(); //Default properties common for each shiny app - shinyAppDataConsumers.applicationProperties.put("repo_link", getRepoLink()); - shinyAppDataConsumers.applicationProperties.put("atlas_url", getAtlasUrl()); - shinyAppDataConsumers.applicationProperties.put("datasource", sourceKey); - shinyAppDataConsumers.applicationProperties.put("datasource_name", source.getSourceName()); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_REPO_LINK.getValue(), getRepoLink()); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_ATLAS_URL.getValue(), getAtlasUrl()); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_DATASOURCE_KEY.getValue(), sourceKey); + shinyAppDataConsumers.applicationProperties.put(ShinyConstants.PROPERTY_NAME_DATASOURCE_NAME.getValue(), source.getSourceName()); populateCDMDataSourceSummaryIfPresent(source, shinyAppDataConsumers); @@ -202,7 +197,7 @@ private String convertAppPropertiesToString(Map appProperties) { protected String dateToString(Date date) { if (date == null) return null; - DateFormat df = new SimpleDateFormat(DATE_TIME_FORMAT); + DateFormat df = new SimpleDateFormat(ShinyConstants.DATE_TIME_FORMAT.getValue()); return df.format(date); } } diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index c7310955d8..43bc4f940b 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -89,15 +89,15 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData Integer assetId = analysis.getId(); Integer sourceId = sourceRepository.findBySourceKey(sourceKey).getSourceId(); - dataConsumers.getAppProperties().accept("author", getAuthor(analysis)); - dataConsumers.getAppProperties().accept(ASSET_ID_KEY, analysis.getId().toString()); - dataConsumers.getAppProperties().accept("generated_date", getGenerationStartTime(analysis)); - dataConsumers.getAppProperties().accept("record_count", getRecordCount(analysisInfoDTO)); - dataConsumers.getAppProperties().accept("person_count", getPersonCount(analysisInfoDTO)); - dataConsumers.getAppProperties().accept("author_notes", getDescription(analysis)); - dataConsumers.getAppProperties().accept("referenced_cohorts", prepareReferencedCohorts(expression)); - dataConsumers.getAppProperties().accept("version_id", getGenerationId(assetId, sourceId)); - dataConsumers.getAppProperties().accept("generation_id", getGenerationId(assetId, sourceId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR.getValue(), getAuthor(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ASSET_ID.getValue(), analysis.getId().toString()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATED_DATE.getValue(), getGenerationStartTime(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_RECORD_COUNT.getValue(), getRecordCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_PERSON_COUNT.getValue(), getPersonCount(analysisInfoDTO)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_AUTHOR_NOTES.getValue(), getDescription(analysis)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), prepareReferencedCohorts(expression)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), getGenerationId(assetId, sourceId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), getGenerationId(assetId, sourceId)); String csvWithCohortDetails = prepareCsvWithCohorts(expression); @@ -119,7 +119,7 @@ private String getAuthor(IncidenceRateAnalysis analysis) { if (analysis.getCreatedBy() != null) { return analysis.getCreatedBy().getLogin(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getGenerationStartTime(IncidenceRateAnalysis analysis) { @@ -128,28 +128,28 @@ private String getGenerationStartTime(IncidenceRateAnalysis analysis) { return dateToString(Iterables.getLast(analysis.getExecutionInfoList()).getStartTime()); } } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getDescription(IncidenceRateAnalysis analysis) { if (analysis != null && analysis.getDescription() != null) { return analysis.getDescription(); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getPersonCount(AnalysisInfoDTO analysisInfo) { if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).cases); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getRecordCount(AnalysisInfoDTO analysisInfo) { if (analysisInfo != null && CollectionUtils.isNotEmpty(analysisInfo.getSummaryList())) { return Long.toString(Iterables.getLast(analysisInfo.getSummaryList()).totalPersons); } - return VALUE_NOT_AVAILABLE; + return ShinyConstants.VALUE_NOT_AVAILABLE.getValue(); } private String getGenerationId(Integer assetId, Integer sourceId) { diff --git a/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java b/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java new file mode 100644 index 0000000000..0439ff2056 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/shiny/ShinyConstants.java @@ -0,0 +1,34 @@ +package org.ohdsi.webapi.shiny; + +public enum ShinyConstants { + VALUE_NOT_AVAILABLE("N/A"), + DATE_TIME_FORMAT("yyyy-MM-dd HH:mm:ss"), + PROPERTY_NAME_REPO_LINK("repo_link"), + PROPERTY_NAME_COHORT_LINK("cohort_link"), + PROPERTY_NAME_COHORT_NAME("cohort_name"), + PROPERTY_NAME_ATLAS_URL("atlas_url"), + PROPERTY_NAME_ATLAS_LINK("atlas_link"), + PROPERTY_NAME_DATASOURCE_KEY("datasource"), + PROPERTY_NAME_DATASOURCE_NAME("datasource_name"), + PROPERTY_NAME_ASSET_ID("asset_id"), + PROPERTY_NAME_ASSET_NAME("asset_name"), + PROPERTY_NAME_ANALYSIS_NAME("analysis_name"), + PROPERTY_NAME_AUTHOR("author"), + PROPERTY_NAME_AUTHOR_NOTES("author_notes"), + PROPERTY_NAME_GENERATED_DATE("generated_date"), + PROPERTY_NAME_RECORD_COUNT("record_count"), + PROPERTY_NAME_REFERENCED_COHORTS("referenced_cohorts"), + PROPERTY_NAME_VERSION_ID("version_id"), + PROPERTY_NAME_GENERATION_ID("generation_id"), + PROPERTY_NAME_PERSON_COUNT("person_count"); + + private final String value; + + ShinyConstants(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java index 9f37d65e31..6d4490fd8c 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java +++ b/src/main/java/org/ohdsi/webapi/shiny/summary/DataSourceSummaryConverter.java @@ -7,6 +7,7 @@ import org.ohdsi.webapi.report.CumulativeObservationRecord; import org.ohdsi.webapi.report.MonthObservationRecord; import org.ohdsi.webapi.service.ShinyService; +import org.ohdsi.webapi.shiny.ShinyConstants; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.stereotype.Service; @@ -19,9 +20,7 @@ @ConditionalOnBean(ShinyService.class) public class DataSourceSummaryConverter { - private static final String VALUE_NOT_AVAILABLE = "N/A"; - - private double calculateVariance(List values, double mean) { + private double calculateVariance(List values, double mean) { double variance = 0; for (double value : values) { variance += Math.pow(value - mean, 2); @@ -65,12 +64,14 @@ public DataSourceSummary convert(CDMDashboard cdmDashboard) { } if (cdmDashboard.getAgeAtFirstObservation() != null) { - List percents = cdmDashboard.getAgeAtFirstObservation().stream() - .map(ConceptDistributionRecord::getPercentValue) + List ages = cdmDashboard.getAgeAtFirstObservation().stream() + .map(ConceptDistributionRecord::getIntervalIndex) .collect(Collectors.toList()); - double sum = percents.stream().mapToDouble(Double::doubleValue).sum(); - double mean = sum / percents.size(); - double variance = calculateVariance(percents, mean); + double sum = ages.stream() + .mapToInt(Integer::intValue) + .sum(); + double mean = sum / ages.size(); + double variance = calculateVariance(ages, mean); int minYear = cdmDashboard.getAgeAtFirstObservation().stream() .min(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) @@ -78,17 +79,17 @@ public DataSourceSummary convert(CDMDashboard cdmDashboard) { int maxYear = cdmDashboard.getAgeAtFirstObservation().stream() .max(Comparator.comparingInt(ConceptDistributionRecord::getIntervalIndex)) .orElse(new ConceptDistributionRecord()).getIntervalIndex(); - dataSourceSummary.setAgeAtFirstObservation(String.format("[%d - %d] (M= %.1f; SD= %.1f)", + dataSourceSummary.setAgeAtFirstObservation(String.format("[%d - %d] (M = %.1f; SD = %.1f)", minYear, maxYear, mean, Math.sqrt(variance))); } if (cdmDashboard.getCumulativeObservation() != null) { - List percentPersons = cdmDashboard.getCumulativeObservation().stream() - .map(CumulativeObservationRecord::getyPercentPersons) + List observationLengths = cdmDashboard.getCumulativeObservation().stream() + .map(CumulativeObservationRecord::getxLengthOfObservation) .collect(Collectors.toList()); - double sum = percentPersons.stream().mapToDouble(Double::doubleValue).sum(); - double mean = sum / percentPersons.size(); - double variance = calculateVariance(percentPersons, mean); + double sum = observationLengths.stream().mapToInt(Integer::intValue).sum(); + double mean = sum / observationLengths.size(); + double variance = calculateVariance(observationLengths, mean); int minObs = cdmDashboard.getCumulativeObservation().stream() .min(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) @@ -96,7 +97,7 @@ public DataSourceSummary convert(CDMDashboard cdmDashboard) { int maxObs = cdmDashboard.getCumulativeObservation().stream() .max(Comparator.comparingInt(CumulativeObservationRecord::getxLengthOfObservation)) .orElse(new CumulativeObservationRecord()).getxLengthOfObservation(); - dataSourceSummary.setCumulativeObservation(String.format("[%d - %d] (M= %.1f; SD= %.1f)", + dataSourceSummary.setCumulativeObservation(String.format("[%d - %d] (M = %.1f; SD = %.1f)", minObs, maxObs, mean, Math.sqrt(variance))); } @@ -115,12 +116,12 @@ public DataSourceSummary convert(CDMDashboard cdmDashboard) { public DataSourceSummary emptySummary(String dataSourceName) { DataSourceSummary dataSourceSummary = new DataSourceSummary(); dataSourceSummary.setSourceName(dataSourceName); - dataSourceSummary.setFemale(VALUE_NOT_AVAILABLE); - dataSourceSummary.setMale(VALUE_NOT_AVAILABLE); - dataSourceSummary.setCumulativeObservation(VALUE_NOT_AVAILABLE); - dataSourceSummary.setAgeAtFirstObservation(VALUE_NOT_AVAILABLE); - dataSourceSummary.setContinuousObservationCoverage(VALUE_NOT_AVAILABLE); - dataSourceSummary.setNumberOfPersons(VALUE_NOT_AVAILABLE); + dataSourceSummary.setFemale(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setMale(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setCumulativeObservation(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setAgeAtFirstObservation(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setContinuousObservationCoverage(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); + dataSourceSummary.setNumberOfPersons(ShinyConstants.VALUE_NOT_AVAILABLE.getValue()); return dataSourceSummary; } } From 807603ab98de06d72807d3d35f01026c59a4709f Mon Sep 17 00:00:00 2001 From: oleg-odysseus Date: Tue, 19 Nov 2024 21:33:47 +0100 Subject: [PATCH 31/32] Added atlas_link to Pathways Shiny App Properties --- .../webapi/shiny/CohortPathwaysShinyPackagingService.java | 3 +++ .../webapi/shiny/IncidenceRatesShinyPackagingService.java | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java index 0c5c7c486f..50e458cae0 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/CohortPathwaysShinyPackagingService.java @@ -78,6 +78,9 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_REFERENCED_COHORTS.getValue(), prepareReferencedCohorts(pathwayAnalysisDTO)); dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_VERSION_ID.getValue(), Integer.toString(generationId)); dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_GENERATION_ID.getValue(), Integer.toString(generationId)); + + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/pathways/%s", atlasUrl, pathwayAnalysisDTO.getId())); + } private String getAuthor(PathwayAnalysisDTO pathwayAnalysisDTO) { diff --git a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java index 43bc4f940b..ada600e188 100644 --- a/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java +++ b/src/main/java/org/ohdsi/webapi/shiny/IncidenceRatesShinyPackagingService.java @@ -80,8 +80,8 @@ public void populateAppData(Integer generationId, String sourceKey, ShinyAppData IncidenceRateAnalysis analysis = incidenceRateAnalysisRepository.findOne(generationId); ExceptionUtils.throwNotFoundExceptionIfNull(analysis, String.format("There is no incidence rate analysis with id = %d.", generationId)); try { - dataConsumers.getAppProperties().accept("atlas_link", String.format("%s/#/iranalysis/%s", atlasUrl, generationId)); - dataConsumers.getAppProperties().accept("analysis_name", analysis.getName()); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ATLAS_LINK.getValue(), String.format("%s/#/iranalysis/%s", atlasUrl, generationId)); + dataConsumers.getAppProperties().accept(ShinyConstants.PROPERTY_NAME_ANALYSIS_NAME.getValue(), analysis.getName()); IncidenceRateAnalysisExportExpression expression = objectMapper.readValue(analysis.getDetails().getExpression(), IncidenceRateAnalysisExportExpression.class); AnalysisInfoDTO analysisInfoDTO = irAnalysisResource.getAnalysisInfo(analysis.getId(), sourceKey); From 75cdbd654a91419bead77e6cf544f6610f969b49 Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Tue, 10 Dec 2024 19:44:58 +0100 Subject: [PATCH 32/32] Combining migration scripts --- ....0.20240227181535__add_shiny_publication.sql | 15 --------------- ...4.0.20231220173146__add_shiny_permission.sql | 16 ---------------- ...07__add_shiny_published_and_permissions.sql} | 17 ++++++++++++++++- .../V20231222125707__add_shiny_published.sql | 14 -------------- 4 files changed, 16 insertions(+), 46 deletions(-) delete mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql delete mode 100644 src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql rename src/main/resources/{db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql => shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql} (54%) delete mode 100644 src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql deleted file mode 100644 index 999a5e659e..0000000000 --- a/src/main/resources/db/migration/postgresql/V2.14.0.20240227181535__add_shiny_publication.sql +++ /dev/null @@ -1,15 +0,0 @@ -CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; - -CREATE TABLE ${ohdsiSchema}.shiny_published -( - id BIGINT NOT NULL PRIMARY KEY DEFAULT nextval('${ohdsiSchema}.shiny_published_sequence'), - type VARCHAR, - analysis_id BIGINT, - source_key VARCHAR, - execution_id BIGINT, - content_id UUID, - created_by_id BIGINT, - modified_by_id BIGINT, - created_date TIMESTAMP, - modified_date TIMESTAMP -); \ No newline at end of file diff --git a/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql b/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql deleted file mode 100644 index 3d0c691c39..0000000000 --- a/src/main/resources/shiny/migration/V2.14.0.20231220173146__add_shiny_permission.sql +++ /dev/null @@ -1,16 +0,0 @@ -INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) -SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), - 'shiny:download:*:*:*:get', - 'Download Shiny Application'; -INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) -SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), - 'shiny:publish:*:*:*:get', - 'Download Shiny Application'; - -INSERT INTO ${ohdsiSchema}.sec_role_permission (role_id, permission_id) -SELECT sr.id, sp.id -FROM ${ohdsiSchema}.sec_permission sp, - ${ohdsiSchema}.sec_role sr -WHERE sp."value" in - ('shiny:download:*:*:*:get', 'shiny:publish:*:*:*:get') - AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql b/src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql similarity index 54% rename from src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql rename to src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql index 10f3bec3fb..977bcbab39 100644 --- a/src/main/resources/db/migration/postgresql/V2.14.0.20240227162911__add_shiny_permissions.sql +++ b/src/main/resources/shiny/migration/V2.15.0.20231222125707__add_shiny_published_and_permissions.sql @@ -1,3 +1,18 @@ +CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; + +CREATE TABLE ${ohdsiSchema}.shiny_published( + id BIGINT PRIMARY KEY default nextval('${ohdsiSchema}.shiny_published_sequence'), + type VARCHAR NOT NULL, + analysis_id BIGINT NOT NULL, + execution_id BIGINT, + source_key VARCHAR, + content_id UUID, + created_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + modified_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), + created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), + modified_date TIMESTAMP WITH TIME ZONE +); + INSERT INTO ${ohdsiSchema}.sec_permission (id, value, description) SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), 'shiny:download:*:*:*:get', @@ -17,4 +32,4 @@ WHERE sp."value" in 'shiny:download:*:*:*:get', 'shiny:publish:*:*:*:get' ) - AND sr.name IN ('Atlas users'); \ No newline at end of file + AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql b/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql deleted file mode 100644 index 4845d3e55e..0000000000 --- a/src/main/resources/shiny/migration/V20231222125707__add_shiny_published.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE SEQUENCE ${ohdsiSchema}.shiny_published_sequence START WITH 1; - -CREATE TABLE ${ohdsiSchema}.shiny_published( - id BIGINT PRIMARY KEY default nextval('${ohdsiSchema}.shiny_published_sequence'), - type VARCHAR NOT NULL, - analysis_id BIGINT NOT NULL, - execution_id BIGINT, - source_key VARCHAR, - content_id UUID, - created_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), - modified_by_id BIGINT REFERENCES ${ohdsiSchema}.sec_user(id), - created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), - modified_date TIMESTAMP WITH TIME ZONE -);