Merge pull request #144 from KPMP/develop

Merge for v2.5 release

.gitignore

@@ -14,3 +14,6 @@ out
 .idea
 .settings
 .classpath
+**/credentials.json
+tokens
+**/node_modules

README.md

@@ -33,7 +33,7 @@ To regenerate zip files:
 5. Bash into the spring container
 `docker exec -it spring bash`
 6. Rebuild the orion-data jar
-`./gradlew build`
+`./gradlew build -x test`
 7. Run the zip generator
 `java -cp build/libs/orion-data.jar -Dloader.main=org.kpmp.RegenerateZipFiles org.springframework.boot.loader.PropertiesLauncher`
 

build.gradle

@@ -15,82 +15,32 @@ apply plugin: 'io.spring.dependency-management'
 
 jar {
     baseName='orion-data'
-    version= '2.4'
+    version= '2.5'
 }
 
 repositories {
     mavenCentral()
 }
 
-sourceSets {
-	integrationTest {
-		java {
-			compileClasspath += main.output + test.output
-			runtimeClasspath += main.output + test.output
-			srcDir file('src/integration-test/java')
-		}
-		resources.srcDir file('src/integration-test/resources')
-	}
-}
-
-configurations {
-	integrationTestCompile.extendsFrom testCompile
-	integrationTestRuntime.extendsFrom testRuntime
-}
-
-processResources {
-    filesMatching('application.properties') {
-        expand(project.properties)
-    }
-}
-
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
     compile 'org.springframework.boot:spring-boot-starter-web' 
 	compile 'commons-io:commons-io:2.6' 
-	compile 'org.mockito:mockito-core'
-    compile 'org.junit.jupiter:junit-jupiter-engine:5.2.0' 
     compile 'mysql:mysql-connector-java:6.0.5'
-	compile 'org.springframework.boot:spring-boot-starter-test'
 	compile 'org.springframework:spring-test:5.0.5.RELEASE'
 	compile 'org.springframework.data:spring-data-mongodb:2.0.8.RELEASE'
 	compile 'org.springframework.boot:spring-boot-starter-data-mongodb'
-	testCompile 'de.flapdoodle.embed:de.flapdoodle.embed.mongo:2.1.1'
-	testCompile 'cz.jirutka.spring:embedmongo-spring:1.3.1'
 	compile 'org.apache.commons:commons-compress:1.17'
 	compile 'org.apache.commons:commons-text:1.7'
 	compile 'com.google.api-client:google-api-client:1.23.0'
 	compile 'com.google.oauth-client:google-oauth-client-jetty:1.23.0'
 	compile 'com.google.apis:google-api-services-drive:v3-rev110-1.23.0'
+	compile 'org.springframework.boot:spring-boot-starter-test'
+	testCompile 'org.mockito:mockito-core'
 }
 
-ext {
-	snippetsDir = file('build/generated-snippets')
-}
-
-test {
-	outputs.dir snippetsDir
-}
-
-tasks.withType(Test) {
-	testLogging {
-	  exceptionFormat "full"
-	  events "skipped", "passed", "failed"
-	  showStandardStreams false
-	}
-  }
-
-task integrationTest(type: Test) {
-	testClassesDirs = sourceSets.integrationTest.output.classesDirs
-	classpath = sourceSets.integrationTest.runtimeClasspath
-	outputs.upToDateWhen { false }
-}
-
-check.dependsOn integrationTest
-integrationTest.mustRunAfter test
-
 springBoot {
     mainClassName = "org.kpmp.Application"
 }

scripts/2.5/initializeStates.js

@@ -0,0 +1,79 @@
+const MongoClient = require('mongodb').MongoClient;
+const assert = require('assert');
+
+const url = 'mongodb://localhost:27017';
+const dbName = 'dataLake';
+var stateMap = new Map();
+var missingStates = new Array();
+
+const createStateMap = function(db, callback) {
+	var stateCollection = db.collection("state");
+
+	stateCollection.find({}).toArray(function(err, docs) {
+		assert.equal(null, err);
+		docs.forEach(function(doc) {
+			var packageId = doc.packageId;
+			var docList = stateMap.get(packageId);
+			if (docList === undefined) {
+				docList= [];
+			}
+			docList.push(doc);
+			stateMap.set(packageId, docList);
+		});
+		callback(stateMap);
+	});
+}
+
+const determineMissingStates= function(stateMap, db, callback) {
+	var packageCollection = db.collection("packages");
+
+
+	packageCollection.find({}).toArray(function(err, docs) {
+		assert.equal(null, err);
+		docs.forEach(function(doc) {
+			let packageId = doc._id;
+			if (doc.largeFilesChecked === true) {
+				if (!stateMap.has(packageId)) {
+					console.log("********************");
+					console.log("Large file upload is mising the corresponding METADATA_RECEIVED state: " + doc._id);
+					console.log("Please address manually");
+					console.log("********************");
+				}
+			} else if (doc.inError === true) {
+				if (!stateMap.has(packageId)) {
+					var state = {  'packageId' : packageId, 'state': 'UPLOAD_FAILED', codicil: 'backfilling errors',  stateChangeDate: doc.createdAt};
+					missingStates.push(state);
+				}
+			} else if (!stateMap.has(packageId)) {
+				var state = { 'packageId' : packageId, 'state': 'UPLOAD_SUCCEEDED', codicil: '',  stateChangeDate: doc.createdAt};
+				missingStates.push(state);
+			}
+		});
+		callback();
+	});
+
+} 
+
+const insertStates = function(db, callback) {
+	var stateCollection = db.collection("state");
+	stateCollection.insertMany(missingStates);
+	callback();
+}
+
+MongoClient.connect(url, {'forceServerObjectId' : true}, function(err, client) {
+
+	assert.equal(null, err);
+	console.log("successfully connected to server");
+
+	const db = client.db(dbName);
+
+	createStateMap(db, function() {
+		determineMissingStates(stateMap, db, function() {
+			insertStates(db, function() {
+				client.close();
+			});
+		});
+	});
+
+});
+