Merge pull request #154 from KPMP/develop

v2.6 release merge

.gitignore

@@ -17,3 +17,4 @@ out
 **/credentials.json
 tokens
 **/node_modules
+globus_tokens

README.md

@@ -1,53 +1,40 @@
-# orion-data
-
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/6af5499a0365459e8f755ec19589534a)](https://www.codacy.com/manual/rlreamy/orion-data?utm_source=github.com&utm_medium=referral&utm_content=KPMP/orion-data&utm_campaign=Badge_Grade)
 [![Build Status](https://travis-ci.org/KPMP/orion-data.svg?branch=develop)](https://travis-ci.org/KPMP/orion-data)
-**Description**: Repo for the KPMP upload tool back-end
-
-## GenerateUploadReport
-To generate an upload report (before production release):
-1. Shut down any running docker containers with mongo
-2. Until we have better state tracking, we need to update the 'inError' flag on packages that were unable to generate a zip file.
-- Connect to production mongo
-- Navigate to http://upload.kpmp.org
-- Make sure any packages that are missing a download button have the 'inError' attribute in mongo
-3. Check out the latest from orion-data
-4. Connect to prod mongo
- `ssh uploader-prod -L 27017:localhost:27017`
-5. Change src/main/resources/application.properties spring.data.mongodb.uri to `spring.data.mongodb.uri=mongodb://localhost:27017/dataLake`
-6. Rebuild the orion-data jar
-`./gradlew build`
-7. Run the report generator
-`java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.GenerateUploadReport org.springframework.boot.loader.PropertiesLauncher`
 
-This will generate a report.csv file in your current directory
-
-**Remember to revert your application.properties file before checking it in!!**
-Make sure you do not check in the report.csv
+# orion-data
+  **Description**: Repo for the KPMP upload tool back-end
 
 ## RegenerateZipFiles
 To regenerate zip files:
-1. Connect to mongo
-2. Set `regenerateZip` to true for any packages you want to regenerate zip files for
-3. Connect to server you need to regenerate zips for
-4. Navigate to heavens-docker/orion
-5. Bash into the spring container
-`docker exec -it spring bash`
-6. Rebuild the orion-data jar
-`./gradlew build -x test`
-7. Run the zip generator
-`java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.RegenerateZipFiles org.springframework.boot.loader.PropertiesLauncher`
+ 1. Connect to mongo
+ 2. Set `regenerateZip` to true for any packages you want to regenerate zip files for
+ 3. Connect to server you need to regenerate zips for
+ 4. Navigate to heavens-docker/orion
+ 5. Bash into the spring container
+ `docker exec -it spring bash`
+ 6. Rebuild the orion-data jar
+ `./gradlew build -x test`
+ 7. Run the zip generator
+ `java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.RegenerateZipFiles org.springframework.boot.loader.PropertiesLauncher`
 
 ## Getting and storing the credentials for Google Drive
-1. Get the credentials.json file from kpmp-secure/orion-data and put it in the `src/main/resources` directory.
-2. Get the StoredCredential file from kpmp-secure/orion-data and put it in the `tokens` directory (create if it doesn't exist).
-3. Restart the spring container.
+ 1. Get the credentials.json file from kpmp-secure/orion-data and put it in the `src/main/resources` directory.
+ 2. Get the StoredCredential file from kpmp-secure/orion-data and put it in the `tokens` directory (create if it doesn't  exist).
+ 3. Restart the spring container.
 
-## Creating new credentials files
+## Creating new credentials files for Google Drive
 NOTE: You need to create a new credentials file if the permissions change
-1. Delete (if necessary) the old `tokens` directory.
-2. Bring the application down and then back up.
-3. Spring will generate a URL and print it to stdout, grab it and open it in a browser.
-4. Authenticate as [email protected] and grant the requested access.
-5. The redirect to localhost will probably fail, since you're not running the app locally. Copy this URL.
-6. Go into the spring container and do a wget on the pasted URL. This will create the credentials file and the app will start running.
-
+ 1. Delete (if necessary) the old `tokens` directory.
+ 2. Bring the application down and then back up.
+ 3. Spring will generate a URL and print it to stdout, grab it and open it in a browser.
+ 4. Authenticate as [email protected] and grant the requested access.
+ 5. The redirect to localhost will probably fail, since you're not running the app locally. Copy this URL.
+ 6. Go into the spring container and do a wget on the pasted URL. This will create the credentials file and the app will start running.
+
+## Creating new credentials files for Globus
+ 1. Delete (if necessary) the `StoredCredential` file in the `globus_tokens` directory.
+ 2. Bring the application down and then back up.
+ 3. Spring will generate a URL and print it to stdout (you may have to look in Kibana), grab it and open it in a browser.
+ 4. Authenticate.
+ 5. The redirect to localhost will probably fail, since you're not running the app locally. Copy this URL.
+ 6. Go into the spring container and do a wget on the pasted URL. This will create the credentials file and the app will start running.

build.gradle

@@ -15,7 +15,7 @@ apply plugin: 'io.spring.dependency-management'
 
 jar {
     baseName='orion-data'
-    version= '2.5'
+    version= '2.6'
 }
 
 repositories {
@@ -39,6 +39,7 @@ dependencies {
 	compile 'com.google.apis:google-api-services-drive:v3-rev110-1.23.0'
 	compile 'org.springframework.boot:spring-boot-starter-test'
 	testCompile 'org.mockito:mockito-core'
+    testCompile 'org.springframework.boot:spring-boot-starter-test:2.2.0.RELEASE'
 }
 
 springBoot {

src/main/java/org/kpmp/WebConfig.java

@@ -1,5 +1,7 @@
 package org.kpmp;
 
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.HttpTransport;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
 import org.springframework.boot.autoconfigure.jdbc.DataSourceTransactionManagerAutoConfiguration;
@@ -11,6 +13,9 @@
 import org.springframework.web.servlet.config.annotation.CorsRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
 
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+
 @Configuration
 @EnableAutoConfiguration(exclude = { DataSourceAutoConfiguration.class,
 		DataSourceTransactionManagerAutoConfiguration.class, HibernateJpaAutoConfiguration.class })
@@ -28,4 +33,7 @@ public RestTemplate restTemplate() {
 		return new RestTemplate();
 	}
 
+	@Bean
+	public HttpTransport httpTransport() throws GeneralSecurityException, IOException { return GoogleNetHttpTransport.newTrustedTransport(); };
+
 }

src/main/java/org/kpmp/globus/GlobusAuthService.java

@@ -0,0 +1,53 @@
+package org.kpmp.globus;
+
+import com.google.api.client.auth.oauth2.AuthorizationCodeFlow;
+import com.google.api.client.auth.oauth2.BearerToken;
+import com.google.api.client.auth.oauth2.ClientParametersAuthentication;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.store.FileDataStoreFactory;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.util.Arrays;
+
+@Service
+public class GlobusAuthService {
+
+    private final File DATA_STORE_DIR =
+            new File("globus_tokens");
+
+    private final String apiKey;
+    private final String apiSecret;
+    private JsonFactory JSON_FACTORY = new JacksonFactory();
+
+    public GlobusAuthService(Environment env) {
+        this.apiKey = env.getProperty("GLOBUS_API_KEY");
+        this.apiSecret = env.getProperty("GLOBUS_API_SECRET");
+    }
+
+    public Credential authorize(HttpTransport httpTransport) throws Exception {
+        String tokenServerUrl = "https://auth.globus.org/v2/oauth2/token";
+        String authServerUrl = "https://auth.globus.org/v2/oauth2/authorize";
+        AuthorizationCodeFlow flow = new AuthorizationCodeFlow.Builder(BearerToken
+                .authorizationHeaderAccessMethod(),
+                httpTransport,
+                JSON_FACTORY,
+                new GenericUrl(tokenServerUrl),
+                new ClientParametersAuthentication(
+                        apiKey, apiSecret),
+                apiKey,
+                authServerUrl).setScopes(Arrays.asList("openid", "profile", "email", "urn:globus:auth:scope:transfer.api.globus.org:all"))
+                .setDataStoreFactory(new FileDataStoreFactory(DATA_STORE_DIR)).build();
+
+        LocalServerReceiver receiver = new LocalServerReceiver.Builder().setHost(
+                "localhost").setPort(8888).build();
+        return new GlobusAuthorizationCodeInstalledApp(flow, receiver).globusAuthorize("user");
+    }
+
+}

src/main/java/org/kpmp/globus/GlobusAuthorizationCodeInstalledApp.java

@@ -0,0 +1,54 @@
+package org.kpmp.globus;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+
+import com.google.api.client.auth.oauth2.AuthorizationCodeFlow;
+import com.google.api.client.auth.oauth2.AuthorizationCodeRequestUrl;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.auth.oauth2.TokenResponse;
+import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
+import com.google.api.client.extensions.java6.auth.oauth2.VerificationCodeReceiver;
+import com.google.api.client.util.ArrayMap;
+
+public class GlobusAuthorizationCodeInstalledApp extends AuthorizationCodeInstalledApp {
+	private final AuthorizationCodeFlow flow;
+	private final VerificationCodeReceiver receiver;
+
+	public GlobusAuthorizationCodeInstalledApp(AuthorizationCodeFlow flow, VerificationCodeReceiver receiver) {
+		super(flow, receiver);
+		this.flow = flow;
+		this.receiver = receiver;
+	}
+
+	@SuppressWarnings({ "rawtypes", "unchecked" })
+	public Credential globusAuthorize(String userId) throws IOException {
+		try {
+			Credential credential = flow.loadCredential(userId);
+			if (credential != null && (credential.getRefreshToken() != null || credential.getExpiresInSeconds() == null
+					|| credential.getExpiresInSeconds() > 60)) {
+				return credential;
+			}
+			String redirectUri = receiver.getRedirectUri();
+			AuthorizationCodeRequestUrl authorizationUrl = flow.newAuthorizationUrl().setRedirectUri(redirectUri);
+			authorizationUrl.set("access_type", "offline");
+			onAuthorization(authorizationUrl);
+			String code = receiver.waitForCode();
+			TokenResponse response = flow.newTokenRequest(code).setRedirectUri(redirectUri).execute();
+			ArrayList<ArrayMap> otherTokens = (ArrayList<ArrayMap>) response.get("other_tokens");
+			ArrayMap transferToken = otherTokens.get(0);
+			TokenResponse transferTokenResponse = new TokenResponse();
+			transferTokenResponse.setAccessToken((String) transferToken.get("access_token"));
+			transferTokenResponse.setRefreshToken((String) transferToken.get("refresh_token"));
+			transferTokenResponse.setTokenType((String) transferToken.get("token_type"));
+			BigDecimal expires = (BigDecimal) transferToken.get("expires_in");
+			transferTokenResponse.setExpiresInSeconds(expires.longValue());
+			transferTokenResponse.setScope((String) transferToken.get("scope"));
+			return flow.createAndStoreCredential(transferTokenResponse, userId);
+
+		} finally {
+			receiver.stop();
+		}
+	}
+}

src/main/java/org/kpmp/globus/GlobusService.java

@@ -0,0 +1,77 @@
+package org.kpmp.globus;
+
+import java.io.IOException;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.http.ByteArrayContent;
+import com.google.api.client.http.GenericUrl;
+import com.google.api.client.http.HttpRequest;
+import com.google.api.client.http.HttpRequestFactory;
+import com.google.api.client.http.HttpTransport;
+
+@Service
+public class GlobusService {
+
+    private HttpRequestFactory requestFactory;
+
+    @Value("${globus.endpoint.ID}")
+    private String endpointID;
+
+    @Value("${globus.file.manager.url}")
+    private String fileManagerUrl;
+
+    private Environment env;
+
+    private class GlobusTransferRequest {
+        private String path;
+        private String dataType;
+
+        public String getPath() {
+            return path;
+        }
+
+        public void setPath(String path) {
+            this.path = path;
+        }
+
+        @JsonProperty("DATA_TYPE")
+        public String getDataType() {
+            return dataType;
+        }
+
+        @JsonProperty("DATA_TYPE")
+        public void setDataType(String dataType) {
+            this.dataType = dataType;
+        }
+    }
+
+    public GlobusService(HttpTransport httpTransport, GlobusAuthService globusAuthService, Environment env) throws Exception {
+        Credential credential = globusAuthService.authorize(httpTransport);
+        requestFactory = httpTransport.createRequestFactory(credential);
+        this.env = env;
+    }
+
+    public String createDirectory(String dirName) throws IOException {
+        String topDirectory = env.getProperty("GLOBUS_DIR");
+        ObjectMapper mapper = new ObjectMapper();
+        String apiUrl = "https://transfer.api.globusonline.org/v0.10";
+        GenericUrl url = new GenericUrl(apiUrl + "/operation/endpoint/" + endpointID + "/mkdir");
+        String fullDirName = topDirectory + "/" + dirName;
+        GlobusTransferRequest globusTransferRequest = new GlobusTransferRequest();
+        globusTransferRequest.setPath(fullDirName);
+        globusTransferRequest.setDataType("mkdir");
+        HttpRequest request = requestFactory.buildPostRequest(url, ByteArrayContent.fromString("application/json", mapper.writeValueAsString(globusTransferRequest)));
+        request.execute();
+        return getFileManagerUrl(fullDirName);
+    }
+
+    public String getFileManagerUrl(String fullDirName) {
+        return fileManagerUrl + "?origin_id=" + endpointID + "&origin_path=" + fullDirName;
+    }
+}