Metric code clean, mainly service bean #4257

IQSS · May 10, 2018 · 5a4bb01 · 5a4bb01
1 parent 25615ea
commit 5a4bb01
Show file tree

Hide file tree

Showing 6 changed files with 98 additions and 142 deletions.
diff --git a/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql b/scripts/database/upgrades/upgrade_v4.8.6_to_v4.9.0.sql
@@ -6,15 +6,10 @@ ALTER TABLE filemetadata ADD COLUMN prov_freeform text;
 -- ALTER TABLE datafile ADD COLUMN prov_cplid int;
 ALTER TABLE datafile ADD COLUMN prov_entityname text;
 
-
---MAD: Creates with wrong owner,
---MAD: Do I need to grant on creation?
 CREATE TABLE METRIC (
     ID  SERIAL NOT NULL,
     metricName character varying(255) NOT NULL,
---    metricMonth integer NOT NULL,
---    metricYear integer NOT NULL,
-    metricValue text NOT NULL, --MAD: THIS ACTUALLY NEEDS TO BE A STRING AND A LONG ONE
+    metricValue text NOT NULL,
     lastCalledDate timestamp without time zone NOT NULL,
     CONSTRAINT metric_pkey PRIMARY KEY (id),
     CONSTRAINT metric_name_key UNIQUE (metricName)

diff --git a/src/main/java/edu/harvard/iq/dataverse/Metric.java b/src/main/java/edu/harvard/iq/dataverse/Metric.java
@@ -24,7 +24,7 @@
  */
 @Entity
 @Table(indexes = {
-        @Index(columnList = "id") //MAD: UNSURE ABOUT USE OF THIS ANNOTATION AND MY CUSTOMIZATION
+        @Index(columnList = "id")
 }) 
 public class Metric implements Serializable {
 
@@ -51,14 +51,14 @@ public Metric() {
     public Metric(String metricTitle, String yyyymm, String metricValue) {  
         this.metricName = generateMetricName(metricTitle, yyyymm); 
         this.metricValue = metricValue;
-        this.lastCalledDate = new Timestamp(new Date().getTime()); //MAD: SHOULD I BE GENERATING THIS IN CODE?
+        this.lastCalledDate = new Timestamp(new Date().getTime());
     }
 
     //For all-time metrics
     public Metric(String metricName, String metricValue) {
         this.metricName = metricName;
         this.metricValue = metricValue;
-        this.lastCalledDate = new Timestamp(new Date().getTime()); //MAD: SHOULD I BE GENERATING THIS IN CODE?
+        this.lastCalledDate = new Timestamp(new Date().getTime());
     }
 
     /**
@@ -115,7 +115,6 @@ public void setLastCalledDate(Date calledDate) {
         this.lastCalledDate = calledDate;
     }
 
-    //MAD: Should this live in a util?
     public static String generateMetricName(String title, String dateString) {
         if(title.contains(seperator) || dateString.contains(seperator)) {
             throw new IllegalArgumentException("Metric title or date contains character reserved for seperator");

diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java b/src/main/java/edu/harvard/iq/dataverse/api/Metrics.java
@@ -41,14 +41,15 @@ public Response getDataversesByMonthCurrent() {
     @Path("dataverses/byMonth/{yyyymm}")
     public Response getDataversesByMonth(@PathParam("yyyymm") String yyyymm) {
         try {
+            String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
             String metricName = "dataversesByMonth";
-            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm);
+            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
 
             if(null == jsonString) { //run query and save
-                Long count = metricsSvc.dataversesByMonth(yyyymm);
+                Long count = metricsSvc.dataversesByMonth(sanitizedyyyymm);
                 JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
                 jsonString = jsonObjBuilder.build().toString();
-                metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new
+                metricsSvc.save(new Metric(metricName,sanitizedyyyymm, jsonString), true); //if not using cache save new
             }
 
             return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -68,14 +69,15 @@ public Response getDatasetsByMonthCurrent() {
     @Path("datasets/byMonth/{yyyymm}")
     public Response getDatasetsByMonth(@PathParam("yyyymm") String yyyymm) {
         try {
+            String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
             String metricName = "datasetsByMonth";
-            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm);
+            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
 
             if(null == jsonString) { //run query and save
-                Long count = metricsSvc.datasetsByMonth(yyyymm);
+                Long count = metricsSvc.datasetsByMonth(sanitizedyyyymm);
                 JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
                 jsonString = jsonObjBuilder.build().toString();
-                metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new
+                metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
             }
 
             return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -95,14 +97,15 @@ public Response getFilesByMonthCurrent() {
     @Path("files/byMonth/{yyyymm}")
     public Response getFilesByMonth(@PathParam("yyyymm") String yyyymm) {
         try {
+            String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
             String metricName = "filesByMonth";
-            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm);
+            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
 
             if(null == jsonString) { //run query and save
-                Long count = metricsSvc.filesByMonth(yyyymm);
+                Long count = metricsSvc.filesByMonth(sanitizedyyyymm);
                 JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
                 jsonString = jsonObjBuilder.build().toString();
-                metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new
+                metricsSvc.save(new Metric(metricName,sanitizedyyyymm, jsonString), true); //if not using cache save new
             }
 
             return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));
@@ -121,14 +124,15 @@ public Response getDownloadsByMonthCurrent() {
     @Path("downloads/byMonth/{yyyymm}")
     public Response getDownloadsByMonth(@PathParam("yyyymm") String yyyymm) {
         try {
+            String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
             String metricName = "downloadsByMonth";
-            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, yyyymm);
+            String jsonString = metricsSvc.returnUnexpiredCacheMonthly(metricName, sanitizedyyyymm);
 
             if(null == jsonString) { //run query and save
-                Long count = metricsSvc.downloadsByMonth(yyyymm);
+                Long count = metricsSvc.downloadsByMonth(sanitizedyyyymm);
                 JsonObjectBuilder jsonObjBuilder = MetricsUtil.countToJson(count);
                 jsonString = jsonObjBuilder.build().toString();
-                metricsSvc.save(new Metric(metricName,yyyymm, jsonString), true); //if not using cache save new
+                metricsSvc.save(new Metric(metricName, sanitizedyyyymm, jsonString), true); //if not using cache save new
             }
 
             return allowCors(ok(MetricsUtil.stringToJsonObjectBuilder(jsonString)));

diff --git a/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/metrics/MetricsServiceBean.java
@@ -4,17 +4,13 @@
 import edu.harvard.iq.dataverse.util.SystemConfig;
 import java.io.Serializable;
 import java.text.SimpleDateFormat;
-import java.time.Instant;
-import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
 import java.util.Date;
 import java.util.List;
 import java.util.logging.Logger;
 import javax.ejb.EJB;
 import javax.ejb.Stateless;
-import javax.json.JsonObjectBuilder;
 import javax.persistence.EntityManager;
 import javax.persistence.NonUniqueResultException;
 import javax.persistence.PersistenceContext;
@@ -32,13 +28,9 @@ public class MetricsServiceBean implements Serializable {
     @EJB
     SystemConfig systemConfig;    
 
-
-
-
-
     /**
      * @param yyyymm Month in YYYY-MM format.
-     */
+     */   
     public long dataversesByMonth(String yyyymm) throws Exception{
         Query query = em.createNativeQuery(""
             + "select count(dvobject.id)\n"
@@ -74,17 +66,85 @@ public long datasetsByMonth(String yyyymm) throws Exception{
 
         return (long) query.getSingleResult();
     }
+
+    /**
+     * @param yyyymm Month in YYYY-MM format.
+     */
+    public long filesByMonth(String yyyymm) throws Exception{
+        Query query = em.createNativeQuery(""
+                + "select count(*)\n"
+                + "from filemetadata\n"
+                + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n"
+                + "where concat(datasetversion.dataset_id,':', datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) in \n"
+                + "(\n"
+                + "select concat(datasetversion.dataset_id,':', max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))) as max \n"
+                + "from datasetversion\n"
+                + "join dataset on dataset.id = datasetversion.dataset_id\n"
+                + "where versionstate='RELEASED'\n"
+                //                + "and date_trunc('month', releasetime) <=  to_date('2018-03','YYYY-MM')\n"
+                // FIXME: Remove SQL injection vector: https://software-security.sans.org/developer-how-to/fix-sql-injection-in-java-persistence-api-jpa
+                + "and date_trunc('month', releasetime) <=  to_date('" + yyyymm + "','YYYY-MM')\n"
+                + "and dataset.harvestingclient_id is null\n"
+                + "group by dataset_id \n"
+                + ");"
+        );
+        logger.fine("query: " + query);
+        return (long) query.getSingleResult();
+    }
+
+    /**
+     * @param yyyymm Month in YYYY-MM format.
+     */
+    public long downloadsByMonth(String yyyymm) throws Exception{
+        Query query = em.createNativeQuery(""
+                + "select count(id)\n"
+                + "from guestbookresponse\n"
+                + "where date_trunc('month', responsetime) <=  to_date('" + yyyymm + "','YYYY-MM');"
+        );
+        logger.fine("query: " + query);
+        return (long) query.getSingleResult();
+    }
 
-
-
-    //---MAD: REORG MY NEW CLASSES
-
-    /** Helper functions for metric caching **/ 
-
+    public List<Object[]> dataversesByCategory() throws Exception {
+
+        Query query = em.createNativeQuery(""
+                + "select dataversetype, count(dataversetype) from dataverse\n"
+                + "join dvobject on dvobject.id = dataverse.id\n"
+                + "where dvobject.publicationdate is not null\n"
+                + "group by dataversetype\n"
+                + "order by count desc;"
+        );
+
+        logger.fine("query: " + query);
+        return query.getResultList(); 
+    }
+
+    public List<Object[]> datasetsBySubject() {
+        Query query = em.createNativeQuery(""
+                + "SELECT strvalue, count(dataset.id)\n"
+                + "FROM datasetfield_controlledvocabularyvalue \n"
+                + "JOIN controlledvocabularyvalue ON controlledvocabularyvalue.id = datasetfield_controlledvocabularyvalue.controlledvocabularyvalues_id\n"
+                + "JOIN datasetfield ON datasetfield.id = datasetfield_controlledvocabularyvalue.datasetfield_id\n"
+                + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n"
+                + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n"
+                + "JOIN dvobject ON dvobject.id = datasetversion.dataset_id\n"
+                + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n"
+                + "WHERE\n"
+                + "datasetfieldtype.name = 'subject'\n"
+                + "AND dvobject.publicationdate is NOT NULL\n"
+                + "AND dataset.harvestingclient_id IS NULL\n"
+                + "GROUP BY strvalue\n"
+                + "ORDER BY count(dataset.id) desc;"
+        );
+        logger.info("query: " + query);
+
+        return query.getResultList(); 
+    }
+
+    /* Helper functions for metric caching */ 
 
     public String returnUnexpiredCacheMonthly(String metricName, String yyyymm) throws Exception {
-        String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm); //remove other sanatize?
-        Metric queriedMetric = getMetric(metricName,sanitizedyyyymm);
+        Metric queriedMetric = getMetric(metricName,yyyymm);
 
         if(!doWeQueryAgainMonthly(queriedMetric)) {
             return queriedMetric.getMetricValue();
@@ -131,12 +191,12 @@ public boolean doWeQueryAgainMonthly(Metric queriedMetric) {
         }
     }
 
-    //MAD: MAYBE THIS LOGIC CAN BE USED ABOVE TO STOP REPEATED CODE?
     //This is for deciding whether to used a cached value over all time
     public boolean doWeQueryAgainAllTime(Metric queriedMetric) {
         if(null == queriedMetric) { //never queried before
             return true;
         }
+
         int minutesUntilNextQuery = systemConfig.getMetricsCacheTimeoutMinutes();
         Date lastCalled = queriedMetric.getLastCalledDate();
         LocalDateTime ldt = LocalDateTime.ofInstant((new Date()).toInstant(), ZoneId.systemDefault());
@@ -148,7 +208,6 @@ public boolean doWeQueryAgainAllTime(Metric queriedMetric) {
         return (todayMinus.after(lastCalled)); 
     }
 
-    //MAD: I DON'T LIKE HOW SAVE HAS A MONTHLY BOOLEAN BUT FOR OTHERS I'VE DUPLICATED CODE
     public Metric save(Metric newMetric, boolean monthly) throws Exception {
         Metric oldMetric;
         if(monthly) {
@@ -162,14 +221,12 @@ public Metric save(Metric newMetric, boolean monthly) throws Exception {
         }
         em.persist(newMetric);
         return em.merge(newMetric);
-
     }
 
     public Metric getMetric(String metricTitle, String yymmmm) throws Exception {
         String searchMetricName = Metric.generateMetricName(metricTitle, yymmmm);
 
         return getMetric(searchMetricName);
-
     }
 
     public Metric getMetric(String searchMetricName) throws Exception{
@@ -185,100 +242,5 @@ public Metric getMetric(String searchMetricName) throws Exception{
         }
         return metric;
     }
-
-
-
-
-    //MAD: datasets by month...
-
-
-
-    /**
-     * @param yyyymm Month in YYYY-MM format.
-     */
-    public long filesByMonth(String yyyymm) throws Exception{
-        String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
-        Query query = em.createNativeQuery(""
-                + "select count(*)\n"
-                + "from filemetadata\n"
-                + "join datasetversion on datasetversion.id = filemetadata.datasetversion_id\n"
-                + "where concat(datasetversion.dataset_id,':', datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber)) in \n"
-                + "(\n"
-                + "select concat(datasetversion.dataset_id,':', max(datasetversion.versionnumber + (.1 * datasetversion.minorversionnumber))) as max \n"
-                + "from datasetversion\n"
-                + "join dataset on dataset.id = datasetversion.dataset_id\n"
-                + "where versionstate='RELEASED'\n"
-                //                + "and date_trunc('month', releasetime) <=  to_date('2018-03','YYYY-MM')\n"
-                // FIXME: Remove SQL injection vector: https://software-security.sans.org/developer-how-to/fix-sql-injection-in-java-persistence-api-jpa
-                + "and date_trunc('month', releasetime) <=  to_date('" + sanitizedyyyymm + "','YYYY-MM')\n"
-                + "and dataset.harvestingclient_id is null\n"
-                + "group by dataset_id \n"
-                + ");"
-        );
-        logger.fine("query: " + query);
-        return (long) query.getSingleResult();
-    }
-
-    /**
-     * @param yyyymm Month in YYYY-MM format.
-     */
-    public long downloadsByMonth(String yyyymm) throws Exception{
-        String sanitizedyyyymm = MetricsUtil.sanitizeYearMonthUserInput(yyyymm);
-        Query query = em.createNativeQuery(""
-                + "select count(id)\n"
-                + "from guestbookresponse\n"
-                + "where date_trunc('month', responsetime) <=  to_date('" + sanitizedyyyymm + "','YYYY-MM');"
-        );
-        logger.fine("query: " + query);
-        return (long) query.getSingleResult();
-    }
-
-    public List<Object[]> dataversesByCategory() throws Exception {
-        String metricName = "dataversesByCategory";
-        Metric queriedMetric = getMetric(metricName);
-
-        if(!doWeQueryAgainAllTime(queriedMetric)) {
-            String cachedMetricString = queriedMetric.getMetricValue();
-
-            return null; //queriedMetric.getMetricValue(); //MAD: FIX ME!
-        }
-
-        Query query = em.createNativeQuery(""
-                + "select dataversetype, count(dataversetype) from dataverse\n"
-                + "join dvobject on dvobject.id = dataverse.id\n"
-                + "where dvobject.publicationdate is not null\n"
-                + "group by dataversetype\n"
-                + "order by count desc;"
-        );
-
-        //MAD: collapse these after debugging
-        List<Object[]> returnList = query.getResultList(); 
-        return returnList;
-    }
-
-    public List<Object[]> datasetsBySubject() {
-        Query query = em.createNativeQuery(""
-                + "SELECT strvalue, count(dataset.id)\n"
-                + "FROM datasetfield_controlledvocabularyvalue \n"
-                + "JOIN controlledvocabularyvalue ON controlledvocabularyvalue.id = datasetfield_controlledvocabularyvalue.controlledvocabularyvalues_id\n"
-                + "JOIN datasetfield ON datasetfield.id = datasetfield_controlledvocabularyvalue.datasetfield_id\n"
-                + "JOIN datasetfieldtype ON datasetfieldtype.id = controlledvocabularyvalue.datasetfieldtype_id\n"
-                + "JOIN datasetversion ON datasetversion.id = datasetfield.datasetversion_id\n"
-                + "JOIN dvobject ON dvobject.id = datasetversion.dataset_id\n"
-                + "JOIN dataset ON dataset.id = datasetversion.dataset_id\n"
-                + "WHERE\n"
-                + "datasetfieldtype.name = 'subject'\n"
-                + "AND dvobject.publicationdate is NOT NULL\n"
-                + "AND dataset.harvestingclient_id IS NULL\n"
-                + "GROUP BY strvalue\n"
-                + "ORDER BY count(dataset.id) desc;"
-        );
-        logger.info("query: " + query);
-
-        //MAD: collapse these after debugging
-        List<Object[]> returnList = query.getResultList(); 
-        return returnList;
-
-    }
 
 }