Merge pull request #121 from KPMP/develop

Merge for DLU v2.3 release

.travis.yml

@@ -1,3 +1,4 @@
+dist: trusty
 language: java
 
 jdk:

README.md

@@ -1,7 +1,39 @@
 # orion-data
 
 [![Build Status](https://travis-ci.org/KPMP/orion-data.svg?branch=develop)](https://travis-ci.org/KPMP/orion-data)
+**Description**: Repo for the KPMP upload tool back-end
 
-Repo for the KPMP upload tool back-end
+## GenerateUploadReport
+To generate an upload report (before production release):
+1. Shut down any running docker containers with mongo
+2. Until we have better state tracking, we need to update the 'inError' flag on packages that were unable to generate a zip file.
+- Connect to production mongo
+- Navigate to http://upload.kpmp.org
+- Make sure any packages that are missing a download button have the 'inError' attribute in mongo
+3. Check out the latest from orion-data
+4. Connect to prod mongo
+ `ssh uploader-prod -L 27017:localhost:27017`
+5. Change src/main/resources/application.properties spring.data.mongodb.uri to `spring.data.mongodb.uri=mongodb://localhost:27017/dataLake`
+6. Rebuild the orion-data jar
+`./gradlew build`
+7. Run the report generator
+`java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.GenerateUploadReport org.springframework.boot.loader.PropertiesLauncher`
 
+This will generate a report.csv file in your current directory
+
+**Remember to revert your application.properties file before checking it in!!**
+Make sure you do not check in the report.csv
+
+## RegenerateZipFiles
+To regenerate zip files:
+1. Connect to mongo
+2. Set `regenerateZip` to true for any packages you want to regenerate zip files for
+3. Connect to server you need to regenerate zips for
+4. Navigate to heavens-docker/orion
+5. Bash into the spring container
+`docker exec -it spring bash`
+6. Rebuild the orion-data jar
+`./gradlew build`
+7. Run the zip generator
+`java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.RegenerateZipFiles org.springframework.boot.loader.PropertiesLauncher`
 

build.gradle

@@ -15,7 +15,7 @@ apply plugin: 'io.spring.dependency-management'
 
 jar {
     baseName='orion-data'
-    version= '2.2'
+    version= '2.3'
 }
 
 repositories {

src/main/java/org/kpmp/GenerateUploadReport.java

@@ -0,0 +1,132 @@
+package org.kpmp;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.kpmp.packages.CustomPackageRepository;
+import org.kpmp.packages.PackageKeys;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.CommandLineRunner;
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.WebApplicationType;
+import org.springframework.context.annotation.ComponentScan;
+
+@ComponentScan(basePackages = { "org.kpmp" })
+public class GenerateUploadReport implements CommandLineRunner {
+
+	private CustomPackageRepository packageRepository;
+
+	@Autowired
+	public GenerateUploadReport(CustomPackageRepository packageRepository) {
+		this.packageRepository = packageRepository;
+	}
+
+	public static void main(String[] args) {
+		SpringApplication app = new SpringApplication(GenerateUploadReport.class);
+		app.setWebApplicationType(WebApplicationType.NONE);
+		app.run(args);
+	}
+
+	@SuppressWarnings({ "rawtypes" })
+	@Override
+	public void run(String... args) throws Exception {
+		List<JSONObject> jsons = packageRepository.findAll();
+		List<Map> packageDatas = new ArrayList<Map>();
+		for (JSONObject packageInfo : jsons) {
+			Map<String, String> packageData = new LinkedHashMap<>();
+			JSONObject submitter = packageInfo.getJSONObject(PackageKeys.SUBMITTER.getKey());
+			String submitterName = submitter.getString(PackageKeys.FIRST_NAME.getKey()) + " "
+					+ submitter.getString(PackageKeys.LAST_NAME.getKey());
+			packageData.put("Package ID", packageInfo.getString(PackageKeys.ID.getKey()));
+			packageData.put("In ERROR?", packageInfo.getString("inError"));
+			packageData.put("Submitter", submitterName);
+			packageData.put("TIS Name", packageInfo.getString(PackageKeys.TIS_NAME.getKey()));
+			packageData.put("Specimen ID", packageInfo.getString(PackageKeys.SUBJECT_ID.getKey()));
+			if (packageInfo.has(PackageKeys.TIS_INTERNAL_EXPERIMENT_ID.getKey())) {
+				packageData.put("TIS Internal Experiment ID",
+						packageInfo.getString(PackageKeys.TIS_INTERNAL_EXPERIMENT_ID.getKey()));
+			} else {
+				packageData.put("TIS Internal Experiment ID", "N/A");
+			}
+			if (packageInfo.has(PackageKeys.DATA_GENERATORS.getKey())) {
+				packageData.put("Data Generator(s)", packageInfo.getString(PackageKeys.DATA_GENERATORS.getKey()));
+			} else {
+				packageData.put("Data Generator(s)", "N/A");
+			}
+			packageData.put("Package Type", packageInfo.getString(PackageKeys.PACKAGE_TYPE.getKey()));
+			packageData.put("Protocol", packageInfo.getString(PackageKeys.PROTOCOL.getKey()));
+			String description = packageInfo.getString(PackageKeys.DESCRIPTION.getKey());
+			description = description.replace("\n", " ");
+			description = description.replace("\r", " ");
+			description = description.replace("\r\n", " ");
+			packageData.put("Dataset Description", description);
+			packageData.put("Created At", packageInfo.getString(PackageKeys.CREATED_AT.getKey()));
+			JSONArray files = packageInfo.getJSONArray(PackageKeys.FILES.getKey());
+			StringBuilder fileNames = new StringBuilder();
+			for (int i = 0; i < files.length(); i++) {
+				JSONObject file = files.getJSONObject(i);
+				if (fileNames.length() != 0) {
+					fileNames.append(", ");
+				} else {
+					fileNames.append("\"");
+				}
+				fileNames.append(file.get(PackageKeys.FILE_NAME.getKey()));
+			}
+			fileNames.append("\"");
+			packageData.put("Files", fileNames.toString());
+			packageDatas.add(packageData);
+		}
+
+		writeToCSV(packageDatas);
+	}
+
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	private void writeToCSV(List<Map> packageDatas) throws FileNotFoundException, IOException {
+		FileOutputStream report = new FileOutputStream(new File("report.tsv"));
+		if (packageDatas.size() == 0) {
+			System.err.println("****  Retrieved 0 packages. NO REPORT GENERATED!  ****");
+			System.exit(0);
+		}
+
+		Map<String, Object> firstPackage = packageDatas.get(0);
+		Set<String> headers = firstPackage.keySet();
+
+		int count = 0;
+		for (String header : headers) {
+			if (count > 0) {
+				report.write("\t".getBytes());
+			}
+			report.write(header.getBytes());
+			count++;
+		}
+		report.write(System.lineSeparator().getBytes());
+
+		count = 0;
+		for (Map packageData : packageDatas) {
+			for (String key : headers) {
+				if (count > 0) {
+					report.write("\t".getBytes());
+				}
+				if (packageData.get(key) == null) {
+					report.write("N/A".getBytes());
+				} else {
+					report.write(packageData.get(key).toString().getBytes());
+				}
+				count++;
+			}
+			report.write(System.lineSeparator().getBytes());
+			count = 0;
+		}
+		report.close();
+	}
+
+}

src/main/java/org/kpmp/WebConfig.java

@@ -4,7 +4,9 @@
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
 import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
+import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.web.client.RestTemplate;
 import org.springframework.web.servlet.config.annotation.CorsRegistration;
 import org.springframework.web.servlet.config.annotation.CorsRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
@@ -21,4 +23,9 @@ public void addCorsMappings(CorsRegistry registry) {
 		corsRegistration.allowedMethods("GET", "POST");
 	}
 
+	@Bean
+	public RestTemplate restTemplate() {
+		return new RestTemplate();
+	}
+
 }

src/main/java/org/kpmp/error/ErrorController.java

@@ -1,7 +1,9 @@
 package org.kpmp.error;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import javax.servlet.http.HttpServletRequest;
+
+import org.kpmp.logging.LoggingService;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.stereotype.Controller;
@@ -13,12 +15,20 @@
 @Controller
 public class ErrorController {
 
-	private final Logger log = LoggerFactory.getLogger(this.getClass());
+	private LoggingService logger;
+
+	@Autowired
+	public ErrorController(LoggingService logger) {
+		this.logger = logger;
+	}
 
 	@RequestMapping(value = "/v1/error", method = RequestMethod.POST, consumes = "application/json")
-	public @ResponseBody ResponseEntity<Boolean> logError(@RequestBody FrontEndError errorMessage) {
+	public @ResponseBody ResponseEntity<Boolean> logError(@RequestBody FrontEndError errorMessage,
+			HttpServletRequest request) {
+
+		logger.logErrorMessage(this.getClass(), null,
+				errorMessage.getError() + " with stacktrace: " + errorMessage.getStackTrace(), request);
 
-		log.error(errorMessage.getError() + " with stacktrace: " + errorMessage.getStackTrace());
 		return new ResponseEntity<>(true, HttpStatus.OK);
 	}
 

src/main/java/org/kpmp/filters/AuthorizationFilter.java

@@ -0,0 +1,57 @@
+package org.kpmp.filters;
+
+import java.io.IOException;
+
+import javax.servlet.Filter;
+import javax.servlet.FilterChain;
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+
+import org.kpmp.logging.LoggingService;
+import org.kpmp.shibboleth.ShibbolethUserService;
+import org.kpmp.users.User;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class AuthorizationFilter implements Filter {
+
+	private LoggingService logger;
+	private ShibbolethUserService shibUserService;
+
+	@Autowired
+	public AuthorizationFilter(LoggingService logger, ShibbolethUserService shibUserService) {
+		this.logger = logger;
+		this.shibUserService = shibUserService;
+	}
+
+	@Override
+	public void init(FilterConfig filterConfig) throws ServletException {
+		logger.logInfoMessage(this.getClass(), null, null, this.getClass().getSimpleName() + ".init",
+				"Initializing filter: " + this.getClass().getSimpleName());
+	}
+
+	@Override
+	public void doFilter(ServletRequest incomingRequest, ServletResponse incomingResponse, FilterChain chain)
+			throws IOException, ServletException {
+
+		HttpServletRequest request = (HttpServletRequest) incomingRequest;
+		User user = shibUserService.getUser(request);
+		// This is where we will implement the logic to talk to the user portal and do
+		// authorization.
+
+		logger.logInfoMessage(this.getClass(), user, null, this.getClass().getSimpleName() + ".doFilter",
+				"Passing through authentication filter");
+		chain.doFilter(incomingRequest, incomingResponse);
+	}
+
+	@Override
+	public void destroy() {
+		logger.logInfoMessage(this.getClass(), null, null, this.getClass().getSimpleName() + ".destroy",
+				"Destroying filter: AuthorizationFilter");
+	}
+
+}

src/main/java/org/kpmp/forms/FormController.java

@@ -1,9 +1,8 @@
 package org.kpmp.forms;
 
-import java.text.MessageFormat;
+import javax.servlet.http.HttpServletRequest;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.kpmp.logging.LoggingService;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Controller;
 import org.springframework.web.bind.annotation.PathVariable;
@@ -14,25 +13,24 @@
 @Controller
 public class FormController {
 
-	private final Logger log = LoggerFactory.getLogger(this.getClass());
-	private static final MessageFormat formRequest = new MessageFormat("Request|{0}");
-
 	private FormRepository repository;
+	private LoggingService logger;
 
 	@Autowired
-	public FormController(FormRepository repository) {
+	public FormController(FormRepository repository, LoggingService logger) {
 		this.repository = repository;
+		this.logger = logger;
 	}
 
 	@RequestMapping(value = "/v1/form", method = RequestMethod.GET)
-	public @ResponseBody Form getFormDTD() {
-		log.info(formRequest.format(new Object[] { "getFormDTD" }));
+	public @ResponseBody Form getFormDTD(HttpServletRequest request) {
+		logger.logInfoMessage(this.getClass(), null, "Request for all forms", request);
 		return this.repository.findTopByOrderByVersionDesc();
 	}
 
 	@RequestMapping(value = "/v1/form/version/{version}", method = RequestMethod.GET)
-	public @ResponseBody Form getFormDTD(@PathVariable Double version) {
-		log.info(formRequest.format(new Object[] { "getFormDTD" }));
+	public @ResponseBody Form getFormDTD(@PathVariable Double version, HttpServletRequest request) {
+		logger.logInfoMessage(this.getClass(), null, "Request for form with version: " + version, request);
 		return this.repository.findByVersion(version).get(0);
 	}