diff --git a/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql b/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql index 63da322bbad..2ff0e758ea0 100644 --- a/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql +++ b/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql @@ -6,15 +6,10 @@ ALTER TABLE filemetadata ADD COLUMN prov_freeform text; -- ALTER TABLE datafile ADD COLUMN prov_cplid int; ALTER TABLE datafile ADD COLUMN prov_entityname text; - ---MAD: Creates with wrong owner, ---MAD: Do I need to grant on creation? CREATE TABLE METRIC ( ID SERIAL NOT NULL, metricName character varying(255) NOT NULL, --- metricMonth integer NOT NULL, --- metricYear integer NOT NULL, - metricValue text NOT NULL, --MAD: THIS ACTUALLY NEEDS TO BE A STRING AND A LONG ONE + metricValue text NOT NULL, lastCalledDate timestamp without time zone NOT NULL, CONSTRAINT metric_pkey PRIMARY KEY (id), CONSTRAINT metric_name_key UNIQUE (metricName) diff --git a/src/main/java/edu/harvard/iq/dataverse/Metric.java b/src/main/java/edu/harvard/iq/dataverse/Metric.java index 52ac932d686..cc1b308bf83 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Metric.java +++ b/src/main/java/edu/harvard/iq/dataverse/Metric.java @@ -24,7 +24,7 @@ */ @Entity @Table(indexes = { - @Index(columnList = "id") //MAD: UNSURE ABOUT USE OF THIS ANNOTATION AND MY CUSTOMIZATION + @Index(columnList = "id") }) public class Metric implements Serializable { @@ -51,14 +51,14 @@ public Metric() { public Metric(String metricTitle, String yyyymm, String metricValue) { this.metricName = generateMetricName(metricTitle, yyyymm); this.metricValue = metricValue; - this.lastCalledDate = new Timestamp(new Date().getTime()); //MAD: SHOULD I BE GENERATING THIS IN CODE? + this.lastCalledDate = new Timestamp(new Date().getTime()); } //For all-time metrics public Metric(String metricName, String metricValue) { this.metricName = metricName; this.metricValue = metricValue; - this.lastCalledDate = new Timestamp(new Date().getTime()); //MAD: SHOULD I BE GENERATING THIS IN CODE? + this.lastCalledDate = new Timestamp(new Date().getTime()); } /** @@ -115,7 +115,6 @@ public void setLastCalledDate(Date calledDate) { this.lastCalledDate = calledDate; } - //MAD: Should this live in a util? public static String generateMetricName(String title, String dateString) { if(title.contains(seperator) || dateString.contains(seperator)) { throw new IllegalArgumentException("Metric title or date contains character reserved for seperator"); diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java index 390dff42700..2a43d9f0814 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java @@ -41,14 +41,15 @@ public Response getDataversesByMonthCurrent() { @Path("dataverses/byMonth/{yyyymm}") public Response getDataversesByMonth(@PathParam("yyyymm") String yyyymm) { try { + String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); String metricName = "dataversesByMonth"; - String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm); + String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm); if(null == jsonString) { //run query and save - Long count = metricsSvc.dataversesByMonth(yyyymm); + Long count = metricsSvc.dataversesByMonth(sanitizedyyyymm); JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count); jsonString = jsonObjBuilder.build().toString(); - metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new + metricsSvc.save(new Metric(metricName,sanitizedyyyymm, jsonString), true); //if not using cache save new } return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString))); @@ -68,14 +69,15 @@ public Response getDatasetsByMonthCurrent() { @Path("datasets/byMonth/{yyyymm}") public Response getDatasetsByMonth(@PathParam("yyyymm") String yyyymm) { try { + String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); String metricName = "datasetsByMonth"; - String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm); + String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm); if(null == jsonString) { //run query and save - Long count = metricsSvc.datasetsByMonth(yyyymm); + Long count = metricsSvc.datasetsByMonth(sanitizedyyyymm); JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count); jsonString = jsonObjBuilder.build().toString(); - metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new + metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new } return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString))); @@ -95,14 +97,15 @@ public Response getFilesByMonthCurrent() { @Path("files/byMonth/{yyyymm}") public Response getFilesByMonth(@PathParam("yyyymm") String yyyymm) { try { + String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); String metricName = "filesByMonth"; - String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm); + String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm); if(null == jsonString) { //run query and save - Long count = metricsSvc.filesByMonth(yyyymm); + Long count = metricsSvc.filesByMonth(sanitizedyyyymm); JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count); jsonString = jsonObjBuilder.build().toString(); - metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new + metricsSvc.save(new Metric(metricName,sanitizedyyyymm, jsonString), true); //if not using cache save new } return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString))); @@ -121,14 +124,15 @@ public Response getDownloadsByMonthCurrent() { @Path("downloads/byMonth/{yyyymm}") public Response getDownloadsByMonth(@PathParam("yyyymm") String yyyymm) { try { + String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); String metricName = "downloadsByMonth"; - String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm); + String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm); if(null == jsonString) { //run query and save - Long count = metricsSvc.downloadsByMonth(yyyymm); + Long count = metricsSvc.downloadsByMonth(sanitizedyyyymm); JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count); jsonString = jsonObjBuilder.build().toString(); - metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new + metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new } return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString))); diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java index 19298d9a413..1aea7b1d5a6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java @@ -4,17 +4,13 @@ import edu.harvard.iq.dataverse.util.SystemConfig; import java.io.Serializable; import java.text.SimpleDateFormat; -import java.time.Instant; -import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneId; -import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.List; import java.util.logging.Logger; import javax.ejb.EJB; import javax.ejb.Stateless; -import javax.json.JsonObjectBuilder; import javax.persistence.EntityManager; import javax.persistence.NonUniqueResultException; import javax.persistence.PersistenceContext; @@ -32,13 +28,9 @@ public class MetricsServiceBean implements Serializable { @EJB SystemConfig systemConfig; - - - - /** * @param yyyymm Month in YYYY-MM format. - */ + */ public long dataversesByMonth(String yyyymm) throws Exception{ Query query = em.createNativeQuery("" + "select count(dvobject.id)\n" @@ -74,17 +66,85 @@ public long datasetsByMonth(String yyyymm) throws Exception{ return (long) query.getSingleResult(); } + + /** + * @param yyyymm Month in YYYY-MM format. + */ + public long filesByMonth(String yyyymm) throws Exception{ + Query query = em.createNativeQuery("" + + "select count(*)\n" + + "from filemetadata\n" + + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n" + + "where concat(datasetversion.dataset_id,':', datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) in \n" + + "(\n" + + "select concat(datasetversion.dataset_id,':', max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))) as max \n" + + "from datasetversion\n" + + "join dataset on dataset.id = datasetversion.dataset_id\n" + + "where versionstate='RELEASED'\n" + // + "and date_trunc('month', releasetime) <= to_date('2018-03','YYYY-MM')\n" + // FIXME: Remove SQL injection vector: https://software-security.sans.org/developer-how-to/fix-sql-injection-in-java-persistence-api-jpa + + "and date_trunc('month', releasetime) <= to_date('" + yyyymm + "','YYYY-MM')\n" + + "and dataset.harvestingclient_id is null\n" + + "group by dataset_id \n" + + ");" + ); + logger.fine("query: " + query); + return (long) query.getSingleResult(); + } + + /** + * @param yyyymm Month in YYYY-MM format. + */ + public long downloadsByMonth(String yyyymm) throws Exception{ + Query query = em.createNativeQuery("" + + "select count(id)\n" + + "from guestbookresponse\n" + + "where date_trunc('month', responsetime) <= to_date('" + yyyymm + "','YYYY-MM');" + ); + logger.fine("query: " + query); + return (long) query.getSingleResult(); + } - - - //---MAD: REORG MY NEW CLASSES - - /** Helper functions for metric caching **/ - + public List dataversesByCategory() throws Exception { + + Query query = em.createNativeQuery("" + + "select dataversetype, count(dataversetype) from dataverse\n" + + "join dvobject on dvobject.id = dataverse.id\n" + + "where dvobject.publicationdate is not null\n" + + "group by dataversetype\n" + + "order by count desc;" + ); + + logger.fine("query: " + query); + return query.getResultList(); + } + + public List datasetsBySubject() { + Query query = em.createNativeQuery("" + + "SELECT strvalue, count(dataset.id)\n" + + "FROM datasetfield_controlledvocabularyvalue \n" + + "JOIN controlledvocabularyvalue ON controlledvocabularyvalue.id = datasetfield_controlledvocabularyvalue.controlledvocabularyvalues_id\n" + + "JOIN datasetfield ON datasetfield.id = datasetfield_controlledvocabularyvalue.datasetfield_id\n" + + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n" + + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n" + + "JOIN dvobject ON dvobject.id = datasetversion.dataset_id\n" + + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n" + + "WHERE\n" + + "datasetfieldtype.name = 'subject'\n" + + "AND dvobject.publicationdate is NOT NULL\n" + + "AND dataset.harvestingclient_id IS NULL\n" + + "GROUP BY strvalue\n" + + "ORDER BY count(dataset.id) desc;" + ); + logger.info("query: " + query); + + return query.getResultList(); + } + + /* Helper functions for metric caching */ public String returnUnexpiredCacheMonthly(String metricName, String yyyymm) throws Exception { - String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); //remove other sanatize? - Metric queriedMetric = getMetric(metricName,sanitizedyyyymm); + Metric queriedMetric = getMetric(metricName,yyyymm); if(!doWeQueryAgainMonthly(queriedMetric)) { return queriedMetric.getMetricValue(); @@ -131,12 +191,12 @@ public boolean doWeQueryAgainMonthly(Metric queriedMetric) { } } - //MAD: MAYBE THIS LOGIC CAN BE USED ABOVE TO STOP REPEATED CODE? //This is for deciding whether to used a cached value over all time public boolean doWeQueryAgainAllTime(Metric queriedMetric) { if(null == queriedMetric) { //never queried before return true; } + int minutesUntilNextQuery = systemConfig.getMetricsCacheTimeoutMinutes(); Date lastCalled = queriedMetric.getLastCalledDate(); LocalDateTime ldt = LocalDateTime.ofInstant((new Date()).toInstant(), ZoneId.systemDefault()); @@ -148,7 +208,6 @@ public boolean doWeQueryAgainAllTime(Metric queriedMetric) { return (todayMinus.after(lastCalled)); } - //MAD: I DON'T LIKE HOW SAVE HAS A MONTHLY BOOLEAN BUT FOR OTHERS I'VE DUPLICATED CODE public Metric save(Metric newMetric, boolean monthly) throws Exception { Metric oldMetric; if(monthly) { @@ -162,14 +221,12 @@ public Metric save(Metric newMetric, boolean monthly) throws Exception { } em.persist(newMetric); return em.merge(newMetric); - } public Metric getMetric(String metricTitle, String yymmmm) throws Exception { String searchMetricName = Metric.generateMetricName(metricTitle, yymmmm); return getMetric(searchMetricName); - } public Metric getMetric(String searchMetricName) throws Exception{ @@ -185,100 +242,5 @@ public Metric getMetric(String searchMetricName) throws Exception{ } return metric; } - - - - - //MAD: datasets by month... - - - - /** - * @param yyyymm Month in YYYY-MM format. - */ - public long filesByMonth(String yyyymm) throws Exception{ - String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); - Query query = em.createNativeQuery("" - + "select count(*)\n" - + "from filemetadata\n" - + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n" - + "where concat(datasetversion.dataset_id,':', datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) in \n" - + "(\n" - + "select concat(datasetversion.dataset_id,':', max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))) as max \n" - + "from datasetversion\n" - + "join dataset on dataset.id = datasetversion.dataset_id\n" - + "where versionstate='RELEASED'\n" - // + "and date_trunc('month', releasetime) <= to_date('2018-03','YYYY-MM')\n" - // FIXME: Remove SQL injection vector: https://software-security.sans.org/developer-how-to/fix-sql-injection-in-java-persistence-api-jpa - + "and date_trunc('month', releasetime) <= to_date('" + sanitizedyyyymm + "','YYYY-MM')\n" - + "and dataset.harvestingclient_id is null\n" - + "group by dataset_id \n" - + ");" - ); - logger.fine("query: " + query); - return (long) query.getSingleResult(); - } - - /** - * @param yyyymm Month in YYYY-MM format. - */ - public long downloadsByMonth(String yyyymm) throws Exception{ - String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); - Query query = em.createNativeQuery("" - + "select count(id)\n" - + "from guestbookresponse\n" - + "where date_trunc('month', responsetime) <= to_date('" + sanitizedyyyymm + "','YYYY-MM');" - ); - logger.fine("query: " + query); - return (long) query.getSingleResult(); - } - - public List dataversesByCategory() throws Exception { - String metricName = "dataversesByCategory"; - Metric queriedMetric = getMetric(metricName); - - if(!doWeQueryAgainAllTime(queriedMetric)) { - String cachedMetricString = queriedMetric.getMetricValue(); - - return null; //queriedMetric.getMetricValue(); //MAD: FIX ME! - } - - Query query = em.createNativeQuery("" - + "select dataversetype, count(dataversetype) from dataverse\n" - + "join dvobject on dvobject.id = dataverse.id\n" - + "where dvobject.publicationdate is not null\n" - + "group by dataversetype\n" - + "order by count desc;" - ); - - //MAD: collapse these after debugging - List returnList = query.getResultList(); - return returnList; - } - - public List datasetsBySubject() { - Query query = em.createNativeQuery("" - + "SELECT strvalue, count(dataset.id)\n" - + "FROM datasetfield_controlledvocabularyvalue \n" - + "JOIN controlledvocabularyvalue ON controlledvocabularyvalue.id = datasetfield_controlledvocabularyvalue.controlledvocabularyvalues_id\n" - + "JOIN datasetfield ON datasetfield.id = datasetfield_controlledvocabularyvalue.datasetfield_id\n" - + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n" - + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n" - + "JOIN dvobject ON dvobject.id = datasetversion.dataset_id\n" - + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n" - + "WHERE\n" - + "datasetfieldtype.name = 'subject'\n" - + "AND dvobject.publicationdate is NOT NULL\n" - + "AND dataset.harvestingclient_id IS NULL\n" - + "GROUP BY strvalue\n" - + "ORDER BY count(dataset.id) desc;" - ); - logger.info("query: " + query); - - //MAD: collapse these after debugging - List returnList = query.getResultList(); - return returnList; - - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java index 8287467d621..229a066cdfc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsUtil.java @@ -92,8 +92,6 @@ public static String sanitizeYearMonthUserInput(String userInput) throws Excepti } LocalDate currentDate = (new Date()).toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); - - //MAD: THESE ERRORS AND OTHERS NEED TO BE TURNED INTO BUNDLE PROPERTIES if(inputLocalDate.isAfter(currentDate)) { throw new Exception("The inputted date is set past the current month."); diff --git a/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java index a68583c6c65..0b0f81a6ce8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/metrics/MetricsUtilTest.java @@ -1,14 +1,12 @@ package edu.harvard.iq.dataverse.metrics; import edu.harvard.iq.dataverse.util.json.JsonUtil; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.List; import javax.json.Json; import javax.json.JsonArray; import javax.json.JsonArrayBuilder; import javax.json.JsonObject; -import javax.json.JsonObjectBuilder; import static org.junit.Assert.assertEquals; import org.junit.Test;