From 39813418040a4071b88374b6dbb788f6868d4191 Mon Sep 17 00:00:00 2001 From: sergey-suvorov Date: Fri, 6 Oct 2023 12:25:41 +0300 Subject: [PATCH 01/14] add execution and trends statistics --- pom.xml | 3 +- .../controller/StatisticController.java | 177 ++++++++++++++ .../webapi/statistic/dto/AccessTrendDto.java | 29 +++ .../webapi/statistic/dto/AccessTrendsDto.java | 16 ++ .../statistic/dto/SourceExecutionDto.java | 40 +++ .../statistic/dto/SourceExecutionsDto.java | 16 ++ .../statistic/service/StatisticService.java | 230 ++++++++++++++++++ src/main/resources/application.properties | 1 + ...02112232__source_execution_satatistics.sql | 15 ++ src/main/resources/log4j2.xml | 5 +- 10 files changed, 529 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java create mode 100644 src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java create mode 100644 src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendsDto.java create mode 100644 src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java create mode 100644 src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionsDto.java create mode 100644 src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java create mode 100644 src/main/resources/db/migration/postgresql/V2.13.1.20231002112232__source_execution_satatistics.sql diff --git a/pom.xml b/pom.xml index 4eefb79fb5..989808fcc7 100644 --- a/pom.xml +++ b/pom.xml @@ -299,8 +299,9 @@ 10 - false + true /tmp/atlas/audit/audit.log + /tmp/atlas/audit/audit-%d{yyyy-MM-dd}-%i.log /tmp/atlas/audit/audit-extra.log diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java new file mode 100644 index 0000000000..637abf36ac --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -0,0 +1,177 @@ +package org.ohdsi.webapi.statistic.controller; + +import com.opencsv.CSVWriter; +import org.apache.commons.lang3.tuple.Pair; +import org.ohdsi.webapi.statistic.dto.AccessTrendDto; +import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; +import org.ohdsi.webapi.statistic.dto.SourceExecutionDto; +import org.ohdsi.webapi.statistic.dto.SourceExecutionsDto; +import org.ohdsi.webapi.statistic.service.StatisticService; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Controller; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.ByteArrayOutputStream; +import java.io.StringWriter; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Controller +@Path("/statistic/") +@ConditionalOnProperty(value = "audit.trail.enabled", havingValue = "true") +public class StatisticController { + private StatisticService service; + + private static final List EXECUTION_STASTICS_HEADER = new ArrayList() {{ + add(new String[]{"Date", "Source", "Execution Type"}); + }}; + + public StatisticController(StatisticService service) { + this.service = service; + } + + /** + * Returns execution statistics + * @param executionStatisticsRequest - filter settings for statistics + */ + @GET + @Path("/executions") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + public Response executionStatistics(ExecutionStatisticsRequest executionStatisticsRequest) { + SourceExecutionsDto sourceExecutions = service.getSourceExecutions(executionStatisticsRequest.getStartDate(), + executionStatisticsRequest.getEndDate(), executionStatisticsRequest.getSourceKey()); + + return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); + } + + /** + * Returns access trends statistics + * @param accessTrendsStatisticsRequest - filter settings for statistics + */ + @POST + @Path("/accesstrends") + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStatisticsRequest) { + AccessTrendsDto trends = service.getAccessTrends(accessTrendsStatisticsRequest.getStartDate(), + accessTrendsStatisticsRequest.getEndDate(), accessTrendsStatisticsRequest.getEndpoints()); + + return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); + } + + private Response prepareExecutionResultResponse(List executions, String filename) { + List data = executions.stream() + .flatMap(execution -> + new ArrayList() {{ + add(new String[]{execution.getExecutionDate().toString(), execution.getSourceName(), execution.getExecutionName()}); + }}.stream() + ) + .collect(Collectors.toList()); + return prepareResponse(data, filename); + } + + private Response prepareAccessTrendsResponse(List trends, String filename) { + List data = trends.stream() + .flatMap(trend -> + new ArrayList() {{ + add(new String[]{trend.getExecutionDate().toString(), trend.getEndpointName()}); + }}.stream() + ) + .collect(Collectors.toList()); + return prepareResponse(data, filename); + } + + private Response prepareResponse(List data, String filename) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + StringWriter sw = new StringWriter(); + CSVWriter csvWriter = new CSVWriter(sw, ',', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER); + csvWriter.writeAll(EXECUTION_STASTICS_HEADER); + csvWriter.writeAll(data); + csvWriter.flush(); + baos.write(sw.getBuffer().toString().getBytes()); + + return Response + .ok(baos) + .type(MediaType.APPLICATION_OCTET_STREAM) + .header("Content-Disposition", String.format("attachment; filename=\"%s\"", filename)) + .build(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private static final class ExecutionStatisticsRequest { + // Format - yyyy-MM-dd + LocalDate startDate; + // Format - yyyy-MM-dd + LocalDate endDate; + String sourceKey; + + public LocalDate getStartDate() { + return startDate; + } + + public void setStartDate(LocalDate startDate) { + this.startDate = startDate; + } + + public LocalDate getEndDate() { + return endDate; + } + + public void setEndDate(LocalDate endDate) { + this.endDate = endDate; + } + + public String getSourceKey() { + return sourceKey; + } + + public void setSourceKey(String sourceKey) { + this.sourceKey = sourceKey; + } + } + + private static final class AccessTrendsStatisticsRequest { + // Format - yyyy-MM-dd + LocalDate startDate; + // Format - yyyy-MM-dd + LocalDate endDate; + // Key - method (POST, GET) + // Value - endpoint ("{}" can be used as a placeholder, will be converted to ".*" in regular expression) + List> endpoints; + + public LocalDate getStartDate() { + return startDate; + } + + public void setStartDate(LocalDate startDate) { + this.startDate = startDate; + } + + public LocalDate getEndDate() { + return endDate; + } + + public void setEndDate(LocalDate endDate) { + this.endDate = endDate; + } + + public List> getEndpoints() { + return endpoints; + } + + public void setEndpoints(List> endpoints) { + this.endpoints = endpoints; + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java new file mode 100644 index 0000000000..d00ecc7bf6 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java @@ -0,0 +1,29 @@ +package org.ohdsi.webapi.statistic.dto; + +import java.time.LocalDate; + +public class AccessTrendDto { + private String endpointName; + private LocalDate executionDate; + + public AccessTrendDto(String endpointName, LocalDate executionDate) { + this.endpointName = endpointName; + this.executionDate = executionDate; + } + + public String getEndpointName() { + return endpointName; + } + + public void setEndpointName(String endpointName) { + this.endpointName = endpointName; + } + + public LocalDate getExecutionDate() { + return executionDate; + } + + public void setExecutionDate(LocalDate executionDate) { + this.executionDate = executionDate; + } +} diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendsDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendsDto.java new file mode 100644 index 0000000000..7c80e8c80d --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendsDto.java @@ -0,0 +1,16 @@ +package org.ohdsi.webapi.statistic.dto; + +import java.util.ArrayList; +import java.util.List; + +public class AccessTrendsDto { + private List trends = new ArrayList<>(); + + public AccessTrendsDto(List trends) { + this.trends = trends; + } + + public List getTrends() { + return trends; + } +} diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java new file mode 100644 index 0000000000..cfe4cd0dd7 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java @@ -0,0 +1,40 @@ +package org.ohdsi.webapi.statistic.dto; + +import java.time.Instant; +import java.time.LocalDate; + +public class SourceExecutionDto { + private String sourceName; + private String executionName; + private LocalDate executionDate; + + public SourceExecutionDto(String sourceName, String executionName, LocalDate executionDate) { + this.sourceName = sourceName; + this.executionName = executionName; + this.executionDate = executionDate; + } + + public String getSourceName() { + return sourceName; + } + + public void setSourceName(String sourceName) { + this.sourceName = sourceName; + } + + public String getExecutionName() { + return executionName; + } + + public void setExecutionName(String executionName) { + this.executionName = executionName; + } + + public LocalDate getExecutionDate() { + return executionDate; + } + + public void setExecutionDate(LocalDate executionDate) { + this.executionDate = executionDate; + } +} diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionsDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionsDto.java new file mode 100644 index 0000000000..e9d48b15b7 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionsDto.java @@ -0,0 +1,16 @@ +package org.ohdsi.webapi.statistic.dto; + +import java.util.ArrayList; +import java.util.List; + +public class SourceExecutionsDto { + private List executions = new ArrayList<>(); + + public SourceExecutionsDto(List executions) { + this.executions = executions; + } + + public List getExecutions() { + return executions; + } +} diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java new file mode 100644 index 0000000000..aa65a8dba0 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -0,0 +1,230 @@ +package org.ohdsi.webapi.statistic.service; + +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; +import org.ohdsi.webapi.statistic.dto.AccessTrendDto; +import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; +import org.ohdsi.webapi.statistic.dto.SourceExecutionDto; +import org.ohdsi.webapi.statistic.dto.SourceExecutionsDto; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Service +@ConditionalOnProperty(value = "audit.trail.enabled", havingValue = "true") +public class StatisticService { + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + + @Value("${audit.trail.log.file}") + // TODO remove value + private String absoluteLogFileName = "/tmp/atlas/audit/audit.log"; + + private String logFileName; + + @Value("${audit.trail.log.file.pattern}") + // TODO remove value + private String absoluteLogFileNamePattern = "/tmp/atlas/audit/audit-%d{yyyy-MM-dd}-%i.log"; + + private String logFileNamePattern; + + private SimpleDateFormat logFileDateFormat; + + private int logFileDateStart; + + private int logFileDateEnd; + + // Some execution can have duplicate logs with different parameters + // Duplicate log entries can exist because sometimes ccontroller methods are called from other controller methods + // These regular expressions let us to choose only needed log entries + private static final Pattern COHORT_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*GET\\s/WebAPI/cohortdefinition/\\d+/generate/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final Pattern CHARACTERIZATION_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*POST\\s/WebAPI/cohort-characterization/\\d+/generation/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final Pattern PATHWAY_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*POST\\s/WebAPI/pathway-analysis/\\d+/generation/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final Pattern IR_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*GET\\s/WebAPI/ir/\\d+/execute/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final Pattern PLE_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*POST\\s/WebAPI/estimation/\\d+/generation/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final Pattern PLP_GENERATION_REGEXP = + Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*POST\\s/WebAPI/prediction/\\d+/generation/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); + + private static final String ENDPOINT_REGEXP = + "^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*-\\s({METHOD_PLACEHOLDER}\\s.*{ENDPOINT_PLACEHOLDER})\\s-.*$"; + + private static final String COHORT_GENERATION_NAME = "Cohort Generation"; + + private static final String CHARACTERIZATION_GENERATION_NAME = "Characterization Generation"; + + private static final String PATHWAY_GENERATION_NAME = "Pathway Generation"; + + private static final String IR_GENERATION_NAME = "Incidence Rates Generation"; + + private static final String PLE_GENERATION_NAME = "Estimation Generation"; + + private static final String PLP_GENERATION_NAME = "Prediction Generation"; + + private static final Map patternMap = new HashMap<>(); + + static { + patternMap.put(COHORT_GENERATION_NAME, COHORT_GENERATION_REGEXP); + patternMap.put(CHARACTERIZATION_GENERATION_NAME, CHARACTERIZATION_GENERATION_REGEXP); + patternMap.put(PATHWAY_GENERATION_NAME, PATHWAY_GENERATION_REGEXP); + patternMap.put(IR_GENERATION_NAME, IR_GENERATION_REGEXP); + patternMap.put(PLE_GENERATION_NAME, PLE_GENERATION_REGEXP); + patternMap.put(PLP_GENERATION_NAME, PLP_GENERATION_REGEXP); + } + + public StatisticService() { + logFileName = new File(absoluteLogFileName).getName(); + logFileNamePattern = new File(absoluteLogFileNamePattern).getName(); + + // Pattern contains "%d{yyyy-MM-dd}". "%d" will not be contained in real log file name + int placeHolderPrefixLength = 3; + logFileDateStart = logFileNamePattern.indexOf("{") - placeHolderPrefixLength + 1; + logFileDateEnd = logFileNamePattern.indexOf("}") - placeHolderPrefixLength; + String dateString = logFileNamePattern.substring(logFileDateStart + placeHolderPrefixLength, + logFileDateEnd + placeHolderPrefixLength); + logFileDateFormat = new SimpleDateFormat(dateString); + } + + public SourceExecutionsDto getSourceExecutions(LocalDate startDate, LocalDate endDate, String sourceKey) { + Set paths = getLogPaths(startDate, endDate); + List executions = paths.stream() + .flatMap(path -> extractSourceExecutions(path, sourceKey).stream()) + .collect(Collectors.toList()); + return new SourceExecutionsDto(executions); + } + + public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List> endpoints) { + Set paths = getLogPaths(startDate, endDate); + List trends = paths.stream() + .flatMap(path -> extractAccessTrends(path, endpoints).stream()) + .collect(Collectors.toList()); + return new AccessTrendsDto(trends); + } + + private List extractSourceExecutions(Path path, String sourceKey) { + try (Stream stream = Files.lines(path)) { + return stream + .map(str -> getMatchedExecution(str, sourceKey)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } catch (IOException e) { + LOG.error("Error parsing log file {}. {}", path.getFileName(), e); + throw new RuntimeException(e); + } + } + + private List extractAccessTrends(Path path, List> endpoints) { + List patterns = endpoints.stream() + .map(endpointPair -> { + String method = endpointPair.getKey(); + String endpoint = endpointPair.getValue().replaceAll("\\{\\}", ".*"); + String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); + regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); + + return Pattern.compile(regexpStr); + }) + .collect(Collectors.toList()); + try (Stream stream = Files.lines(path)) { + return stream + .map(str -> + patterns.stream() + .map(pattern -> pattern.matcher(str)) + .filter(matcher -> matcher.matches()) + .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)))) + .findFirst() + ) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } catch (IOException e) { + LOG.error("Error parsing log file {}. {}", path.getFileName(), e); + throw new RuntimeException(e); + } + } + + private Optional getMatchedExecution(String str, String sourceKey) { + return patternMap.entrySet().stream() + .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().matcher(str))) + .filter(pair -> pair.getValue().matches()) + .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(2)))) + .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)))) + .findFirst(); + } + + private Set getLogPaths(LocalDate startDate, LocalDate endDate) { + String folderPath = new File(absoluteLogFileName).getParentFile().getAbsolutePath(); + try (Stream stream = Files.list(Paths.get(folderPath))) { + return stream + .filter(file -> !Files.isDirectory(file)) + .filter(this::isValidLogFile) + .filter(file -> isLogInDateRange(file, startDate, endDate)) + .map(Path::toAbsolutePath) + .collect(Collectors.toSet()); + } catch (IOException e) { + LOG.error("Error getting list of log files", e); + throw new RuntimeException(e); + } + } + + private boolean isValidLogFile(Path path) { + return path.getFileName().toString().endsWith(".log"); + } + + private boolean isLogInDateRange(Path path, LocalDate startDate, LocalDate endDate) { + if (startDate == null && endDate == null) { + return true; + } + LocalDate logDate = getFileDate(path.getFileName()); + if ((startDate != null && logDate.isBefore(startDate)) + || (endDate != null && logDate.isAfter(endDate))) { + return false; + } + return true; + } + + private LocalDate getFileDate(Path path) { + String fileName = path.toString(); + if (logFileName.equals(fileName)) { + return LocalDate.now(); + } + try { + String dateStr = fileName.substring(logFileDateStart, logFileDateEnd); + return logFileDateFormat.parse(dateStr).toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } catch (ParseException | IndexOutOfBoundsException e) { + // If we cannot check the date of a file, then assume that it is a file for the current date + return LocalDate.now(); + } + } +} diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index beaafd1e08..257262eab3 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -281,4 +281,5 @@ versioning.maxAttempt=${versioning.maxAttempt} #Audit trail audit.trail.enabled=${audit.trail.enabled} audit.trail.log.file=${audit.trail.log.file} +audit.trail.log.file.pattern=${audit.trail.log.file.pattern} audit.trail.log.extraFile=${audit.trail.log.extraFile} diff --git a/src/main/resources/db/migration/postgresql/V2.13.1.20231002112232__source_execution_satatistics.sql b/src/main/resources/db/migration/postgresql/V2.13.1.20231002112232__source_execution_satatistics.sql new file mode 100644 index 0000000000..6f26e78221 --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.13.1.20231002112232__source_execution_satatistics.sql @@ -0,0 +1,15 @@ +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES + (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'statistic:executions:get', 'Source execution statistics permission'); + +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES + (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'statistic:accesstrends:post', 'Access trends statistics permission'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) +SELECT nextval('${ohdsiSchema}.sec_role_permission_sequence'), sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp.value IN ('statistic:executions:get') AND sr.name IN ('admin'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) +SELECT nextval('${ohdsiSchema}.sec_role_permission_sequence'), sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp.value IN ('statistic:accesstrends:post') AND sr.name IN ('admin'); \ No newline at end of file diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 0114052983..6e5b5674e2 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -2,7 +2,8 @@ ${bundle:application:audit.trail.log.extraFile} - ${bundle:application:audit.trail.log.file} + ${bundle:application:audit.trail.log.file} + ${bundle:application:audit.trail.log.file.pattern} ${bundle:application:logging.level.org.apache.shiro} ${bundle:application:logging.level.org.ohdsi} ${bundle:application:logging.level.org.pac4j} @@ -12,7 +13,7 @@ + filePattern="${audit.trail.log.file.pattern}"> %m%n From 05e64a5bc2c4616a8db38c3086f6398a9b156cac Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Fri, 10 Nov 2023 21:03:10 +0100 Subject: [PATCH 02/14] Feature readiness --- pom.xml | 2 +- .../controller/StatisticController.java | 68 +++++++++++-------- .../webapi/statistic/dto/EndpointDto.java | 23 +++++++ .../statistic/service/StatisticService.java | 9 +-- 4 files changed, 69 insertions(+), 33 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/statistic/dto/EndpointDto.java diff --git a/pom.xml b/pom.xml index 989808fcc7..c53c8fbd32 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ 10 - true + false /tmp/atlas/audit/audit.log /tmp/atlas/audit/audit-%d{yyyy-MM-dd}-%i.log /tmp/atlas/audit/audit-extra.log diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index 637abf36ac..2564c0c2b9 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -1,9 +1,11 @@ package org.ohdsi.webapi.statistic.controller; import com.opencsv.CSVWriter; + import org.apache.commons.lang3.tuple.Pair; import org.ohdsi.webapi.statistic.dto.AccessTrendDto; import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; +import org.ohdsi.webapi.statistic.dto.EndpointDto; import org.ohdsi.webapi.statistic.dto.SourceExecutionDto; import org.ohdsi.webapi.statistic.dto.SourceExecutionsDto; import org.ohdsi.webapi.statistic.service.StatisticService; @@ -17,9 +19,11 @@ import javax.ws.rs.Produces; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; + import java.io.ByteArrayOutputStream; import java.io.StringWriter; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -30,9 +34,13 @@ public class StatisticController { private StatisticService service; - private static final List EXECUTION_STASTICS_HEADER = new ArrayList() {{ + private static final List EXECUTION_STATISTICS_CSV_RESULT_HEADER = new ArrayList() {{ add(new String[]{"Date", "Source", "Execution Type"}); }}; + + private static final List ACCESS_TRENDS_CSV_RESULT_HEADER = new ArrayList() {{ + add(new String[]{"Date", "Endpoint"}); + }}; public StatisticController(StatisticService service) { this.service = service; @@ -42,13 +50,15 @@ public StatisticController(StatisticService service) { * Returns execution statistics * @param executionStatisticsRequest - filter settings for statistics */ - @GET + @POST @Path("/executions") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public Response executionStatistics(ExecutionStatisticsRequest executionStatisticsRequest) { - SourceExecutionsDto sourceExecutions = service.getSourceExecutions(executionStatisticsRequest.getStartDate(), - executionStatisticsRequest.getEndDate(), executionStatisticsRequest.getSourceKey()); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + SourceExecutionsDto sourceExecutions = service.getSourceExecutions(LocalDate.parse(executionStatisticsRequest.getStartDate(), formatter), + LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey()); return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); } @@ -62,8 +72,10 @@ public Response executionStatistics(ExecutionStatisticsRequest executionStatisti @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStatisticsRequest) { - AccessTrendsDto trends = service.getAccessTrends(accessTrendsStatisticsRequest.getStartDate(), - accessTrendsStatisticsRequest.getEndDate(), accessTrendsStatisticsRequest.getEndpoints()); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + + AccessTrendsDto trends = service.getAccessTrends(LocalDate.parse(accessTrendsStatisticsRequest.getStartDate(), formatter), + LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints()); return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); } @@ -76,7 +88,7 @@ private Response prepareExecutionResultResponse(List executi }}.stream() ) .collect(Collectors.toList()); - return prepareResponse(data, filename); + return prepareResponse(data, filename, EXECUTION_STATISTICS_CSV_RESULT_HEADER); } private Response prepareAccessTrendsResponse(List trends, String filename) { @@ -87,14 +99,14 @@ private Response prepareAccessTrendsResponse(List trends, String }}.stream() ) .collect(Collectors.toList()); - return prepareResponse(data, filename); + return prepareResponse(data, filename, ACCESS_TRENDS_CSV_RESULT_HEADER); } - private Response prepareResponse(List data, String filename) { + private Response prepareResponse(List data, String filename, List header) { try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { StringWriter sw = new StringWriter(); CSVWriter csvWriter = new CSVWriter(sw, ',', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER); - csvWriter.writeAll(EXECUTION_STASTICS_HEADER); + csvWriter.writeAll(header); csvWriter.writeAll(data); csvWriter.flush(); baos.write(sw.getBuffer().toString().getBytes()); @@ -109,26 +121,26 @@ private Response prepareResponse(List data, String filename) { } } - private static final class ExecutionStatisticsRequest { + public static final class ExecutionStatisticsRequest { // Format - yyyy-MM-dd - LocalDate startDate; + String startDate; // Format - yyyy-MM-dd - LocalDate endDate; + String endDate; String sourceKey; - public LocalDate getStartDate() { + public String getStartDate() { return startDate; } - public void setStartDate(LocalDate startDate) { + public void setStartDate(String startDate) { this.startDate = startDate; } - public LocalDate getEndDate() { + public String getEndDate() { return endDate; } - public void setEndDate(LocalDate endDate) { + public void setEndDate(String endDate) { this.endDate = endDate; } @@ -141,36 +153,36 @@ public void setSourceKey(String sourceKey) { } } - private static final class AccessTrendsStatisticsRequest { + public static final class AccessTrendsStatisticsRequest { // Format - yyyy-MM-dd - LocalDate startDate; + String startDate; // Format - yyyy-MM-dd - LocalDate endDate; + String endDate; // Key - method (POST, GET) // Value - endpoint ("{}" can be used as a placeholder, will be converted to ".*" in regular expression) - List> endpoints; - - public LocalDate getStartDate() { + List endpoints; + + public String getStartDate() { return startDate; } - public void setStartDate(LocalDate startDate) { + public void setStartDate(String startDate) { this.startDate = startDate; } - public LocalDate getEndDate() { + public String getEndDate() { return endDate; } - public void setEndDate(LocalDate endDate) { + public void setEndDate(String endDate) { this.endDate = endDate; } - public List> getEndpoints() { + public List getEndpoints() { return endpoints; } - public void setEndpoints(List> endpoints) { + public void setEndpoints(List endpoints) { this.endpoints = endpoints; } } diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/EndpointDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/EndpointDto.java new file mode 100644 index 0000000000..d6082203b5 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/EndpointDto.java @@ -0,0 +1,23 @@ +package org.ohdsi.webapi.statistic.dto; + +public class EndpointDto { + String method; + String urlPattern; + + public String getMethod() { + return method; + } + + public void setMethod(String method) { + this.method = method; + } + + public String getUrlPattern() { + return urlPattern; + } + + public void setUrlPattern(String urlPattern) { + this.urlPattern = urlPattern; + } +} + diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index aa65a8dba0..86ce1815c9 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -4,6 +4,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.ohdsi.webapi.statistic.dto.AccessTrendDto; import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; +import org.ohdsi.webapi.statistic.dto.EndpointDto; import org.ohdsi.webapi.statistic.dto.SourceExecutionDto; import org.ohdsi.webapi.statistic.dto.SourceExecutionsDto; import org.slf4j.Logger; @@ -124,7 +125,7 @@ public SourceExecutionsDto getSourceExecutions(LocalDate startDate, LocalDate en return new SourceExecutionsDto(executions); } - public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List> endpoints) { + public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List endpoints) { Set paths = getLogPaths(startDate, endDate); List trends = paths.stream() .flatMap(path -> extractAccessTrends(path, endpoints).stream()) @@ -145,11 +146,11 @@ private List extractSourceExecutions(Path path, String sourc } } - private List extractAccessTrends(Path path, List> endpoints) { + private List extractAccessTrends(Path path, List endpoints) { List patterns = endpoints.stream() .map(endpointPair -> { - String method = endpointPair.getKey(); - String endpoint = endpointPair.getValue().replaceAll("\\{\\}", ".*"); + String method = endpointPair.getMethod(); + String endpoint = endpointPair.getUrlPattern().replaceAll("\\{\\}", ".*"); String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); From 53e27a6791182e1a98ff22596f3464aa90c7a4d6 Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Wed, 6 Dec 2023 19:12:22 +0100 Subject: [PATCH 03/14] Fixing statistics endpoint permission method GET -> POST --- ...31206190600__fixing_statistics_enpoint_permission_method.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20231206190600__fixing_statistics_enpoint_permission_method.sql diff --git a/src/main/resources/db/migration/postgresql/V2.14.0.20231206190600__fixing_statistics_enpoint_permission_method.sql b/src/main/resources/db/migration/postgresql/V2.14.0.20231206190600__fixing_statistics_enpoint_permission_method.sql new file mode 100644 index 0000000000..df5fe8b3cb --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.14.0.20231206190600__fixing_statistics_enpoint_permission_method.sql @@ -0,0 +1,2 @@ +UPDATE ${ohdsiSchema}.sec_permission SET value='statistic:executions:post' +WHERE value='statistic:executions:get'; \ No newline at end of file From 8e28b98f6900289f539d046e6ab98892eeed812d Mon Sep 17 00:00:00 2001 From: Hernaldo Urbina Date: Tue, 30 Jan 2024 23:18:45 -0600 Subject: [PATCH 04/14] ATL-8: Atlas Usage Statistics --- .../controller/StatisticController.java | 76 +++++++++++++++++-- .../webapi/statistic/dto/AccessTrendDto.java | 12 ++- .../statistic/dto/SourceExecutionDto.java | 12 ++- .../statistic/service/StatisticService.java | 21 ++--- 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index 2564c0c2b9..64548dffc8 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -2,7 +2,7 @@ import com.opencsv.CSVWriter; -import org.apache.commons.lang3.tuple.Pair; +import org.ohdsi.webapi.shiro.TokenManager; import org.ohdsi.webapi.statistic.dto.AccessTrendDto; import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; import org.ohdsi.webapi.statistic.dto.EndpointDto; @@ -11,9 +11,10 @@ import org.ohdsi.webapi.statistic.service.StatisticService; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Controller; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; import javax.ws.rs.Consumes; -import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -26,6 +27,8 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; @Controller @@ -34,6 +37,10 @@ public class StatisticController { private StatisticService service; + public enum ResponseFormat { + CSV, JSON + } + private static final List EXECUTION_STATISTICS_CSV_RESULT_HEADER = new ArrayList() {{ add(new String[]{"Date", "Source", "Execution Type"}); }}; @@ -56,11 +63,16 @@ public StatisticController(StatisticService service) { @Consumes(MediaType.APPLICATION_JSON) public Response executionStatistics(ExecutionStatisticsRequest executionStatisticsRequest) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + String userID = executionStatisticsRequest.isShowUserInformation() ? extractUserID() : null; SourceExecutionsDto sourceExecutions = service.getSourceExecutions(LocalDate.parse(executionStatisticsRequest.getStartDate(), formatter), - LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey()); + LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey(), userID); - return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); + if (ResponseFormat.CSV.equals(executionStatisticsRequest.getResponseFormat())) { + return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); + } else { + return Response.ok(sourceExecutions).build(); + } } /** @@ -73,11 +85,16 @@ public Response executionStatistics(ExecutionStatisticsRequest executionStatisti @Consumes(MediaType.APPLICATION_JSON) public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStatisticsRequest) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - + String userID = accessTrendsStatisticsRequest.isShowUserInformation() ? extractUserID() : null; + AccessTrendsDto trends = service.getAccessTrends(LocalDate.parse(accessTrendsStatisticsRequest.getStartDate(), formatter), - LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints()); + LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints(), userID); - return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); + if (ResponseFormat.CSV.equals(accessTrendsStatisticsRequest.getResponseFormat())) { + return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); + } else { + return Response.ok(trends).build(); + } } private Response prepareExecutionResultResponse(List executions, String filename) { @@ -121,12 +138,23 @@ private Response prepareResponse(List data, String filename, List endpoints; + ResponseFormat responseFormat; + boolean showUserInformation; public String getStartDate() { return startDate; @@ -185,5 +231,21 @@ public List getEndpoints() { public void setEndpoints(List endpoints) { this.endpoints = endpoints; } + + public ResponseFormat getResponseFormat() { + return responseFormat; + } + + public void setResponseFormat(ResponseFormat responseFormat) { + this.responseFormat = responseFormat; + } + + public boolean isShowUserInformation() { + return showUserInformation; + } + + public void setShowUserInformation(boolean showUserInformation) { + this.showUserInformation = showUserInformation; + } } } diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java index d00ecc7bf6..9b92208d01 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java @@ -5,10 +5,12 @@ public class AccessTrendDto { private String endpointName; private LocalDate executionDate; + private String userID; - public AccessTrendDto(String endpointName, LocalDate executionDate) { + public AccessTrendDto(String endpointName, LocalDate executionDate, String userID) { this.endpointName = endpointName; this.executionDate = executionDate; + this.userID = userID; } public String getEndpointName() { @@ -26,4 +28,12 @@ public LocalDate getExecutionDate() { public void setExecutionDate(LocalDate executionDate) { this.executionDate = executionDate; } + + public String getUserID() { + return userID; + } + + public void setUserID(String userID) { + this.userID = userID; + } } diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java index cfe4cd0dd7..fc141e9d0a 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/SourceExecutionDto.java @@ -7,11 +7,13 @@ public class SourceExecutionDto { private String sourceName; private String executionName; private LocalDate executionDate; + private String userID; - public SourceExecutionDto(String sourceName, String executionName, LocalDate executionDate) { + public SourceExecutionDto(String sourceName, String executionName, LocalDate executionDate, String userID) { this.sourceName = sourceName; this.executionName = executionName; this.executionDate = executionDate; + this.userID = userID; } public String getSourceName() { @@ -37,4 +39,12 @@ public LocalDate getExecutionDate() { public void setExecutionDate(LocalDate executionDate) { this.executionDate = executionDate; } + + public String getUserID() { + return userID; + } + + public void setUserID(String userID) { + this.userID = userID; + } } diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index 86ce1815c9..89936fff22 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -117,26 +117,26 @@ public StatisticService() { logFileDateFormat = new SimpleDateFormat(dateString); } - public SourceExecutionsDto getSourceExecutions(LocalDate startDate, LocalDate endDate, String sourceKey) { + public SourceExecutionsDto getSourceExecutions(LocalDate startDate, LocalDate endDate, String sourceKey, String userID) { Set paths = getLogPaths(startDate, endDate); List executions = paths.stream() - .flatMap(path -> extractSourceExecutions(path, sourceKey).stream()) + .flatMap(path -> extractSourceExecutions(path, sourceKey, userID).stream()) .collect(Collectors.toList()); return new SourceExecutionsDto(executions); } - public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List endpoints) { + public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List endpoints, String userID) { Set paths = getLogPaths(startDate, endDate); List trends = paths.stream() - .flatMap(path -> extractAccessTrends(path, endpoints).stream()) + .flatMap(path -> extractAccessTrends(path, endpoints, userID).stream()) .collect(Collectors.toList()); return new AccessTrendsDto(trends); } - private List extractSourceExecutions(Path path, String sourceKey) { + private List extractSourceExecutions(Path path, String sourceKey, String userID) { try (Stream stream = Files.lines(path)) { return stream - .map(str -> getMatchedExecution(str, sourceKey)) + .map(str -> getMatchedExecution(str, sourceKey, userID)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); @@ -146,13 +146,14 @@ private List extractSourceExecutions(Path path, String sourc } } - private List extractAccessTrends(Path path, List endpoints) { + private List extractAccessTrends(Path path, List endpoints, String userID) { List patterns = endpoints.stream() .map(endpointPair -> { String method = endpointPair.getMethod(); String endpoint = endpointPair.getUrlPattern().replaceAll("\\{\\}", ".*"); String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); + regexpStr = regexpStr.replace("{USERID_PLACEHOLDER}", userID); return Pattern.compile(regexpStr); }) @@ -163,7 +164,7 @@ private List extractAccessTrends(Path path, List en patterns.stream() .map(pattern -> pattern.matcher(str)) .filter(matcher -> matcher.matches()) - .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)))) + .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)), matcher.group(3))) .findFirst() ) .filter(Optional::isPresent) @@ -175,12 +176,12 @@ private List extractAccessTrends(Path path, List en } } - private Optional getMatchedExecution(String str, String sourceKey) { + private Optional getMatchedExecution(String str, String sourceKey, String userID) { return patternMap.entrySet().stream() .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().matcher(str))) .filter(pair -> pair.getValue().matches()) .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(2)))) - .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)))) + .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), userID)) .findFirst(); } From 1fe7db0281eed2f759d738dd539478a2649405cb Mon Sep 17 00:00:00 2001 From: Hernaldo Urbina Date: Mon, 12 Feb 2024 22:21:42 -0600 Subject: [PATCH 05/14] ATL-8: Addressing userId info --- .../controller/StatisticController.java | 22 +++------------- .../webapi/statistic/dto/EndpointDto.java | 11 +++++++- .../statistic/service/StatisticService.java | 25 ++++++++----------- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index 64548dffc8..c075f0befb 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -2,7 +2,6 @@ import com.opencsv.CSVWriter; -import org.ohdsi.webapi.shiro.TokenManager; import org.ohdsi.webapi.statistic.dto.AccessTrendDto; import org.ohdsi.webapi.statistic.dto.AccessTrendsDto; import org.ohdsi.webapi.statistic.dto.EndpointDto; @@ -11,8 +10,6 @@ import org.ohdsi.webapi.statistic.service.StatisticService; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Controller; -import org.springframework.web.context.request.RequestContextHolder; -import org.springframework.web.context.request.ServletRequestAttributes; import javax.ws.rs.Consumes; import javax.ws.rs.POST; @@ -27,8 +24,6 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; -import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; @Controller @@ -63,10 +58,10 @@ public StatisticController(StatisticService service) { @Consumes(MediaType.APPLICATION_JSON) public Response executionStatistics(ExecutionStatisticsRequest executionStatisticsRequest) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - String userID = executionStatisticsRequest.isShowUserInformation() ? extractUserID() : null; + boolean showUserInformation = executionStatisticsRequest.isShowUserInformation(); SourceExecutionsDto sourceExecutions = service.getSourceExecutions(LocalDate.parse(executionStatisticsRequest.getStartDate(), formatter), - LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey(), userID); + LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey(), showUserInformation); if (ResponseFormat.CSV.equals(executionStatisticsRequest.getResponseFormat())) { return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); @@ -85,10 +80,10 @@ public Response executionStatistics(ExecutionStatisticsRequest executionStatisti @Consumes(MediaType.APPLICATION_JSON) public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStatisticsRequest) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); - String userID = accessTrendsStatisticsRequest.isShowUserInformation() ? extractUserID() : null; + boolean showUserInformation = accessTrendsStatisticsRequest.isShowUserInformation(); AccessTrendsDto trends = service.getAccessTrends(LocalDate.parse(accessTrendsStatisticsRequest.getStartDate(), formatter), - LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints(), userID); + LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints(), showUserInformation); if (ResponseFormat.CSV.equals(accessTrendsStatisticsRequest.getResponseFormat())) { return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); @@ -138,15 +133,6 @@ private Response prepareResponse(List data, String filename, List paths = getLogPaths(startDate, endDate); List executions = paths.stream() - .flatMap(path -> extractSourceExecutions(path, sourceKey, userID).stream()) + .flatMap(path -> extractSourceExecutions(path, sourceKey, showUserInformation).stream()) .collect(Collectors.toList()); return new SourceExecutionsDto(executions); } - public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List endpoints, String userID) { + public AccessTrendsDto getAccessTrends(LocalDate startDate, LocalDate endDate, List endpoints, boolean showUserInformation) { Set paths = getLogPaths(startDate, endDate); List trends = paths.stream() - .flatMap(path -> extractAccessTrends(path, endpoints, userID).stream()) + .flatMap(path -> extractAccessTrends(path, endpoints, showUserInformation).stream()) .collect(Collectors.toList()); return new AccessTrendsDto(trends); } - private List extractSourceExecutions(Path path, String sourceKey, String userID) { + private List extractSourceExecutions(Path path, String sourceKey, boolean showUserInformation) { try (Stream stream = Files.lines(path)) { return stream - .map(str -> getMatchedExecution(str, sourceKey, userID)) + .map(str -> getMatchedExecution(str, sourceKey, showUserInformation)) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); @@ -146,14 +142,15 @@ private List extractSourceExecutions(Path path, String sourc } } - private List extractAccessTrends(Path path, List endpoints, String userID) { + private List extractAccessTrends(Path path, List endpoints, boolean showUserInformation) { List patterns = endpoints.stream() .map(endpointPair -> { String method = endpointPair.getMethod(); String endpoint = endpointPair.getUrlPattern().replaceAll("\\{\\}", ".*"); + String userId = endpointPair.getUserId(); String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); - regexpStr = regexpStr.replace("{USERID_PLACEHOLDER}", userID); + regexpStr = regexpStr.replace("{USERID_PLACEHOLDER}", userId); return Pattern.compile(regexpStr); }) @@ -176,12 +173,12 @@ private List extractAccessTrends(Path path, List en } } - private Optional getMatchedExecution(String str, String sourceKey, String userID) { + private Optional getMatchedExecution(String str, String sourceKey, boolean showUserInformation) { return patternMap.entrySet().stream() .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().matcher(str))) .filter(pair -> pair.getValue().matches()) .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(2)))) - .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), userID)) + .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), pair.getValue().group(3))) .findFirst(); } From 462d81dc866aa33c21f7658a1445ff23aeff149c Mon Sep 17 00:00:00 2001 From: Hernaldo Urbina Date: Mon, 4 Mar 2024 09:25:22 -0600 Subject: [PATCH 06/14] ATL-8: Updating username logic --- .../statistic/service/StatisticService.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index b92abc7e0e..32b50311c7 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -21,11 +21,8 @@ import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.ZoneId; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; +import java.util.*; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -77,6 +74,7 @@ public class StatisticService { private static final String ENDPOINT_REGEXP = "^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*-\\s({METHOD_PLACEHOLDER}\\s.*{ENDPOINT_PLACEHOLDER})\\s-.*$"; + private static final Pattern PLP_USER_REGEXP = Pattern.compile("- ([a-zA-Z0-9_]+) \\d+:\\d+:\\d+:\\d+:\\d+:\\d+:\\d+:\\d+"); private static final String COHORT_GENERATION_NAME = "Cohort Generation"; private static final String CHARACTERIZATION_GENERATION_NAME = "Characterization Generation"; @@ -89,6 +87,8 @@ public class StatisticService { private static final String PLP_GENERATION_NAME = "Prediction Generation"; + private static final String PLP_USERNAME = "Username"; + private static final Map patternMap = new HashMap<>(); static { @@ -98,6 +98,7 @@ public class StatisticService { patternMap.put(IR_GENERATION_NAME, IR_GENERATION_REGEXP); patternMap.put(PLE_GENERATION_NAME, PLE_GENERATION_REGEXP); patternMap.put(PLP_GENERATION_NAME, PLP_GENERATION_REGEXP); + patternMap.put(PLP_USERNAME, PLP_USER_REGEXP); } public StatisticService() { @@ -147,23 +148,22 @@ private List extractAccessTrends(Path path, List en .map(endpointPair -> { String method = endpointPair.getMethod(); String endpoint = endpointPair.getUrlPattern().replaceAll("\\{\\}", ".*"); - String userId = endpointPair.getUserId(); String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); - regexpStr = regexpStr.replace("{USERID_PLACEHOLDER}", userId); return Pattern.compile(regexpStr); }) .collect(Collectors.toList()); try (Stream stream = Files.lines(path)) { return stream - .map(str -> - patterns.stream() + .map(str -> { + Matcher userMatcher = PLP_USER_REGEXP.matcher(str); + return patterns.stream() .map(pattern -> pattern.matcher(str)) - .filter(matcher -> matcher.matches()) - .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)), matcher.group(3))) - .findFirst() - ) + .filter(Matcher::matches) + .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)), showUserInformation && userMatcher.find() ? userMatcher.group(1) : null)) + .findFirst(); + }) .filter(Optional::isPresent) .map(Optional::get) .collect(Collectors.toList()); @@ -178,7 +178,7 @@ private Optional getMatchedExecution(String str, String sour .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().matcher(str))) .filter(pair -> pair.getValue().matches()) .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(2)))) - .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), pair.getValue().group(3))) + .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), showUserInformation ? pair.getValue().group(1) : null)) .findFirst(); } From 01c0d514438f53b5173b84b61930b7e8470f4ff4 Mon Sep 17 00:00:00 2001 From: Hernaldo Urbina Date: Thu, 7 Mar 2024 07:24:14 -0600 Subject: [PATCH 07/14] ATL-8: Adding accesstrends regexp update --- .../controller/StatisticController.java | 4 ++-- .../statistic/service/StatisticService.java | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index c075f0befb..f807fbeca8 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -41,7 +41,7 @@ public enum ResponseFormat { }}; private static final List ACCESS_TRENDS_CSV_RESULT_HEADER = new ArrayList() {{ - add(new String[]{"Date", "Endpoint"}); + add(new String[]{"Date", "Endpoint", "UserID"}); }}; public StatisticController(StatisticService service) { @@ -107,7 +107,7 @@ private Response prepareAccessTrendsResponse(List trends, String List data = trends.stream() .flatMap(trend -> new ArrayList() {{ - add(new String[]{trend.getExecutionDate().toString(), trend.getEndpointName()}); + add(new String[]{trend.getExecutionDate().toString(), trend.getEndpointName(), trend.getUserID()}); }}.stream() ) .collect(Collectors.toList()); diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index 32b50311c7..99cd703dac 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -21,7 +21,11 @@ import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.ZoneId; -import java.util.*; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -72,9 +76,8 @@ public class StatisticService { Pattern.compile("^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*POST\\s/WebAPI/prediction/\\d+/generation/(.+)\\s-\\s.*status::String,startDate::Date,endDate::Date.*$"); private static final String ENDPOINT_REGEXP = - "^.*(\\d{4}-\\d{2}-\\d{2})T\\d{2}:\\d{2}:\\d{2}.*-\\s({METHOD_PLACEHOLDER}\\s.*{ENDPOINT_PLACEHOLDER})\\s-.*$"; + "^.*(\\d{4}-\\d{2}-\\d{2})T(\\d{2}:\\d{2}:\\d{2}).*-\\s([\\w.-]+)\\s.*-\\s({METHOD_PLACEHOLDER}\\s.*{ENDPOINT_PLACEHOLDER})\\s-.*$"; - private static final Pattern PLP_USER_REGEXP = Pattern.compile("- ([a-zA-Z0-9_]+) \\d+:\\d+:\\d+:\\d+:\\d+:\\d+:\\d+:\\d+"); private static final String COHORT_GENERATION_NAME = "Cohort Generation"; private static final String CHARACTERIZATION_GENERATION_NAME = "Characterization Generation"; @@ -87,8 +90,6 @@ public class StatisticService { private static final String PLP_GENERATION_NAME = "Prediction Generation"; - private static final String PLP_USERNAME = "Username"; - private static final Map patternMap = new HashMap<>(); static { @@ -98,7 +99,6 @@ public class StatisticService { patternMap.put(IR_GENERATION_NAME, IR_GENERATION_REGEXP); patternMap.put(PLE_GENERATION_NAME, PLE_GENERATION_REGEXP); patternMap.put(PLP_GENERATION_NAME, PLP_GENERATION_REGEXP); - patternMap.put(PLP_USERNAME, PLP_USER_REGEXP); } public StatisticService() { @@ -147,21 +147,22 @@ private List extractAccessTrends(Path path, List en List patterns = endpoints.stream() .map(endpointPair -> { String method = endpointPair.getMethod(); + String endpoint = endpointPair.getUrlPattern().replaceAll("\\{\\}", ".*"); String regexpStr = ENDPOINT_REGEXP.replace("{METHOD_PLACEHOLDER}", method); regexpStr = regexpStr.replace("{ENDPOINT_PLACEHOLDER}", endpoint); return Pattern.compile(regexpStr); }) + .collect(Collectors.toList()); try (Stream stream = Files.lines(path)) { return stream .map(str -> { - Matcher userMatcher = PLP_USER_REGEXP.matcher(str); return patterns.stream() .map(pattern -> pattern.matcher(str)) .filter(Matcher::matches) - .map(matcher -> new AccessTrendDto(matcher.group(2), LocalDate.parse(matcher.group(1)), showUserInformation && userMatcher.find() ? userMatcher.group(1) : null)) + .map(matcher -> new AccessTrendDto(matcher.group(4), LocalDate.parse(matcher.group(1)), showUserInformation ? matcher.group(3) : null)) .findFirst(); }) .filter(Optional::isPresent) From af3f6821ad578520cbef89a7e7de6738537d5876 Mon Sep 17 00:00:00 2001 From: Hernaldo Urbina Date: Thu, 7 Mar 2024 20:18:46 -0600 Subject: [PATCH 08/14] ATL-8: Adding userInformation to the response --- .../controller/StatisticController.java | 44 +++++++++++++------ .../webapi/statistic/dto/AccessTrendDto.java | 8 ++-- .../statistic/dto/SourceExecutionDto.java | 8 ++-- .../statistic/service/StatisticService.java | 20 ++++----- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index f807fbeca8..b2fe8552e8 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -42,7 +42,7 @@ public enum ResponseFormat { private static final List ACCESS_TRENDS_CSV_RESULT_HEADER = new ArrayList() {{ add(new String[]{"Date", "Endpoint", "UserID"}); - }}; + }}; public StatisticController(StatisticService service) { this.service = service; @@ -64,7 +64,7 @@ public Response executionStatistics(ExecutionStatisticsRequest executionStatisti LocalDate.parse(executionStatisticsRequest.getEndDate(), formatter), executionStatisticsRequest.getSourceKey(), showUserInformation); if (ResponseFormat.CSV.equals(executionStatisticsRequest.getResponseFormat())) { - return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip"); + return prepareExecutionResultResponse(sourceExecutions.getExecutions(), "execution_statistics.zip", showUserInformation); } else { return Response.ok(sourceExecutions).build(); } @@ -86,29 +86,29 @@ public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStati LocalDate.parse(accessTrendsStatisticsRequest.getEndDate(), formatter), accessTrendsStatisticsRequest.getEndpoints(), showUserInformation); if (ResponseFormat.CSV.equals(accessTrendsStatisticsRequest.getResponseFormat())) { - return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip"); + return prepareAccessTrendsResponse(trends.getTrends(), "execution_trends.zip", showUserInformation); } else { return Response.ok(trends).build(); } } - private Response prepareExecutionResultResponse(List executions, String filename) { + private Response prepareExecutionResultResponse(List executions, String filename, boolean showUserInformation) { + updateExecutionStatisticsHeader(showUserInformation); List data = executions.stream() - .flatMap(execution -> - new ArrayList() {{ - add(new String[]{execution.getExecutionDate().toString(), execution.getSourceName(), execution.getExecutionName()}); - }}.stream() + .map(execution -> showUserInformation + ? new String[]{execution.getExecutionDate(), execution.getSourceName(), execution.getExecutionName(), execution.getUserID()} + : new String[]{execution.getExecutionDate(), execution.getSourceName(), execution.getExecutionName()} ) .collect(Collectors.toList()); return prepareResponse(data, filename, EXECUTION_STATISTICS_CSV_RESULT_HEADER); } - private Response prepareAccessTrendsResponse(List trends, String filename) { + private Response prepareAccessTrendsResponse(List trends, String filename, boolean showUserInformation) { + updateAccessTrendsHeader(showUserInformation); List data = trends.stream() - .flatMap(trend -> - new ArrayList() {{ - add(new String[]{trend.getExecutionDate().toString(), trend.getEndpointName(), trend.getUserID()}); - }}.stream() + .map(trend -> showUserInformation + ? new String[]{trend.getExecutionDate().toString(), trend.getEndpointName(), trend.getUserID()} + : new String[]{trend.getExecutionDate().toString(), trend.getEndpointName()} ) .collect(Collectors.toList()); return prepareResponse(data, filename, ACCESS_TRENDS_CSV_RESULT_HEADER); @@ -133,6 +133,24 @@ private Response prepareResponse(List data, String filename, List extractAccessTrends(Path path, List en return patterns.stream() .map(pattern -> pattern.matcher(str)) .filter(Matcher::matches) - .map(matcher -> new AccessTrendDto(matcher.group(4), LocalDate.parse(matcher.group(1)), showUserInformation ? matcher.group(3) : null)) + .map(matcher -> new AccessTrendDto(matcher.group(4), matcher.group(1), showUserInformation ? matcher.group(3) : null)) .findFirst(); }) .filter(Optional::isPresent) @@ -178,8 +178,8 @@ private Optional getMatchedExecution(String str, String sour return patternMap.entrySet().stream() .map(entry -> new ImmutablePair<>(entry.getKey(), entry.getValue().matcher(str))) .filter(pair -> pair.getValue().matches()) - .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(2)))) - .map(pair -> new SourceExecutionDto(pair.getValue().group(2), pair.getKey(), LocalDate.parse(pair.getValue().group(1)), showUserInformation ? pair.getValue().group(1) : null)) + .filter(pair -> sourceKey == null || (sourceKey != null && sourceKey.equals(pair.getValue().group(3)))) + .map(pair -> new SourceExecutionDto(pair.getValue().group(3), pair.getKey(), pair.getValue().group(1), showUserInformation ? pair.getValue().group(2) : null)) .findFirst(); } From 4fe2f04233811cbe76870124581e4bf1dfe615a2 Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Tue, 12 Mar 2024 15:10:22 +0100 Subject: [PATCH 09/14] Minor formatting --- .../java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java index 1b48b8a117..4a35113739 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java +++ b/src/main/java/org/ohdsi/webapi/statistic/dto/AccessTrendDto.java @@ -1,10 +1,8 @@ package org.ohdsi.webapi.statistic.dto; -import java.time.LocalDate; - public class AccessTrendDto { private String endpointName; - private String executionDate; + private String executionDate; private String userID; public AccessTrendDto(String endpointName, String executionDate, String userID) { From 3233ba3620727964d1679ee150e4c6977e417d3f Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Wed, 8 Jan 2025 13:56:45 +0100 Subject: [PATCH 10/14] Log entry added, non-closed resources are correctly handled, file header collections are stateless, hard-coded constants deleted relying on the application configuration file Migration scripts combined --- .../controller/StatisticController.java | 47 +++++++++---------- .../statistic/dto/SourceExecutionDto.java | 17 +++---- .../statistic/service/StatisticService.java | 10 ++-- ...g_statistics_enpoint_permission_method.sql | 2 - ..._access_trends_statistics_permissions.sql} | 6 +-- 5 files changed, 37 insertions(+), 45 deletions(-) delete mode 100644 src/main/resources/db/migration/postgresql/V2.14.0.20231206190600__fixing_statistics_enpoint_permission_method.sql rename src/main/resources/db/migration/postgresql/{V2.13.1.20231002112232__source_execution_satatistics.sql => V2.15.0.20231002112232__source_execution_and_access_trends_statistics_permissions.sql} (84%) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index b2fe8552e8..555af355c2 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -8,6 +8,8 @@ import org.ohdsi.webapi.statistic.dto.SourceExecutionDto; import org.ohdsi.webapi.statistic.dto.SourceExecutionsDto; import org.ohdsi.webapi.statistic.service.StatisticService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Controller; @@ -30,6 +32,9 @@ @Path("/statistic/") @ConditionalOnProperty(value = "audit.trail.enabled", havingValue = "true") public class StatisticController { + + private static final Logger log = LoggerFactory.getLogger(StatisticController.class); + private StatisticService service; public enum ResponseFormat { @@ -40,8 +45,16 @@ public enum ResponseFormat { add(new String[]{"Date", "Source", "Execution Type"}); }}; + private static final List EXECUTION_STATISTICS_CSV_RESULT_HEADER_WITH_USER_ID = new ArrayList() {{ + add(new String[]{"Date", "Source", "Execution Type", "User ID"}); + }}; + private static final List ACCESS_TRENDS_CSV_RESULT_HEADER = new ArrayList() {{ - add(new String[]{"Date", "Endpoint", "UserID"}); + add(new String[]{"Date", "Endpoint"}); + }}; + + private static final List ACCESS_TRENDS_CSV_RESULT_HEADER_WITH_USER_ID = new ArrayList() {{ + add(new String[]{"Date", "Endpoint", "User ID"}); }}; public StatisticController(StatisticService service) { @@ -93,31 +106,30 @@ public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStati } private Response prepareExecutionResultResponse(List executions, String filename, boolean showUserInformation) { - updateExecutionStatisticsHeader(showUserInformation); List data = executions.stream() .map(execution -> showUserInformation - ? new String[]{execution.getExecutionDate(), execution.getSourceName(), execution.getExecutionName(), execution.getUserID()} + ? new String[]{execution.getExecutionDate(), execution.getSourceName(), execution.getExecutionName(), execution.getUserId()} : new String[]{execution.getExecutionDate(), execution.getSourceName(), execution.getExecutionName()} ) .collect(Collectors.toList()); - return prepareResponse(data, filename, EXECUTION_STATISTICS_CSV_RESULT_HEADER); + return prepareResponse(data, filename, showUserInformation ? EXECUTION_STATISTICS_CSV_RESULT_HEADER_WITH_USER_ID : EXECUTION_STATISTICS_CSV_RESULT_HEADER); } private Response prepareAccessTrendsResponse(List trends, String filename, boolean showUserInformation) { - updateAccessTrendsHeader(showUserInformation); List data = trends.stream() .map(trend -> showUserInformation ? new String[]{trend.getExecutionDate().toString(), trend.getEndpointName(), trend.getUserID()} : new String[]{trend.getExecutionDate().toString(), trend.getEndpointName()} ) .collect(Collectors.toList()); - return prepareResponse(data, filename, ACCESS_TRENDS_CSV_RESULT_HEADER); + return prepareResponse(data, filename, showUserInformation ? ACCESS_TRENDS_CSV_RESULT_HEADER_WITH_USER_ID : ACCESS_TRENDS_CSV_RESULT_HEADER); } private Response prepareResponse(List data, String filename, List header) { - try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); StringWriter sw = new StringWriter(); - CSVWriter csvWriter = new CSVWriter(sw, ',', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER); + CSVWriter csvWriter = new CSVWriter(sw, ',', CSVWriter.DEFAULT_QUOTE_CHARACTER, CSVWriter.DEFAULT_ESCAPE_CHARACTER)) { + csvWriter.writeAll(header); csvWriter.writeAll(data); csvWriter.flush(); @@ -129,28 +141,11 @@ private Response prepareResponse(List data, String filename, List Date: Thu, 27 Feb 2025 19:51:51 +0100 Subject: [PATCH 11/14] Fixed issue with a method reference in a stream during the docker build --- .../org/ohdsi/webapi/statistic/service/StatisticService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index ff60e27098..c4057dd6da 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -163,7 +163,7 @@ private List extractAccessTrends(Path path, List en .map(str -> { return patterns.stream() .map(pattern -> pattern.matcher(str)) - .filter(Matcher::matches) + .filter(matcher -> matcher.matches()) .map(matcher -> new AccessTrendDto(matcher.group(4), matcher.group(1), showUserInformation ? matcher.group(3) : null)) .findFirst(); }) From 038a60baf7167eaad1db3e1ab73f0257bb827e0c Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Tue, 11 Mar 2025 11:24:30 +0100 Subject: [PATCH 12/14] Having the controller instantiated without conditions and responding with a 404 Not Found instead when the Audit Trail property is not switched on --- .../statistic/controller/StatisticController.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index 555af355c2..036b8a1afe 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -10,10 +10,11 @@ import org.ohdsi.webapi.statistic.service.StatisticService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Controller; import javax.ws.rs.Consumes; +import javax.ws.rs.NotFoundException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; @@ -30,12 +31,14 @@ @Controller @Path("/statistic/") -@ConditionalOnProperty(value = "audit.trail.enabled", havingValue = "true") public class StatisticController { private static final Logger log = LoggerFactory.getLogger(StatisticController.class); private StatisticService service; + + @Value("${audit.trail.enabled}") + private boolean auditTrailEnabled; public enum ResponseFormat { CSV, JSON @@ -70,6 +73,9 @@ public StatisticController(StatisticService service) { @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public Response executionStatistics(ExecutionStatisticsRequest executionStatisticsRequest) { + if (!auditTrailEnabled) { + throw new NotFoundException("Audit Trail functionality should be enabled (audit.trail.enabled) to serve this endpoint"); + } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); boolean showUserInformation = executionStatisticsRequest.isShowUserInformation(); @@ -92,6 +98,9 @@ public Response executionStatistics(ExecutionStatisticsRequest executionStatisti @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public Response accessStatistics(AccessTrendsStatisticsRequest accessTrendsStatisticsRequest) { + if (!auditTrailEnabled) { + throw new NotFoundException("Audit Trail functionality should be enabled (audit.trail.enabled) to serve this endpoint"); + } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); boolean showUserInformation = accessTrendsStatisticsRequest.isShowUserInformation(); From 32c98a741a3264284bed572b713977ae3b371c81 Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Tue, 11 Mar 2025 11:50:10 +0100 Subject: [PATCH 13/14] Removing a condition from the service as well --- .../ohdsi/webapi/statistic/controller/StatisticController.java | 2 ++ .../org/ohdsi/webapi/statistic/service/StatisticService.java | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java index 036b8a1afe..cd78782acf 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java +++ b/src/main/java/org/ohdsi/webapi/statistic/controller/StatisticController.java @@ -10,6 +10,7 @@ import org.ohdsi.webapi.statistic.service.StatisticService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Controller; @@ -60,6 +61,7 @@ public enum ResponseFormat { add(new String[]{"Date", "Endpoint", "User ID"}); }}; + @Autowired public StatisticController(StatisticService service) { this.service = service; } diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index c4057dd6da..0d21252a8e 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -9,7 +9,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import java.io.File; @@ -32,7 +31,6 @@ import java.util.stream.Stream; @Service -@ConditionalOnProperty(value = "audit.trail.enabled", havingValue = "true") public class StatisticService { protected final Logger LOG = LoggerFactory.getLogger(getClass()); From 24ebe0c9b6aae7a9dc12c0172ae045ae3481e4da Mon Sep 17 00:00:00 2001 From: alex-odysseus Date: Tue, 11 Mar 2025 12:08:51 +0100 Subject: [PATCH 14/14] Correcting StatisticService instantiation --- .../ohdsi/webapi/statistic/service/StatisticService.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java index 0d21252a8e..f368db8a59 100644 --- a/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java +++ b/src/main/java/org/ohdsi/webapi/statistic/service/StatisticService.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -35,12 +34,12 @@ public class StatisticService { protected final Logger LOG = LoggerFactory.getLogger(getClass()); @Value("${audit.trail.log.file}") - private String absoluteLogFileName; + private String absoluteLogFileName = "/tmp/atlas/audit/audit.log"; private String logFileName; @Value("${audit.trail.log.file.pattern}") - private String absoluteLogFileNamePattern; + private String absoluteLogFileNamePattern = "/tmp/atlas/audit/audit-%d{yyyy-MM-dd}-%i.log"; private String logFileNamePattern; @@ -98,9 +97,6 @@ public class StatisticService { } public StatisticService() { - if (absoluteLogFileName == null || absoluteLogFileNamePattern == null) { - throw new RuntimeException("Application statistics can't operate because of missing configuration values for the audit trail log file or its pattern"); - } logFileName = new File(absoluteLogFileName).getName(); logFileNamePattern = new File(absoluteLogFileNamePattern).getName();