Merge branch 'develop'

.gitignore

@@ -1,3 +1,4 @@
+.env
 *.iml
 Callback*
 .*_history

build.gradle

@@ -15,7 +15,7 @@ apply plugin: 'io.spring.dependency-management'
 
 jar {
     baseName='orion-data'
-    version= '2.8.1'
+    version= '2.9.1'
 }
 
 repositories {

data/stateDisplay.json

@@ -7,6 +7,9 @@
     "dlu": {
       "text": "Finishing upload",
       "alertType": "success"
+    },
+    "dmd": {
+      "text": "Finishing Upload"
     }
   }
 },{
@@ -18,6 +21,9 @@
     "dlu": {
       "text": "Waiting for files...",
       "alertType": "primary"
+    },
+    "dmd": {
+      "text": "Waiting for files"
     }
   }
 },{
@@ -29,6 +35,9 @@
     "dlu": {
       "text": "Upload failed",
       "alertType": "danger"
+    },
+    "dmd": {
+      "text": "Upload failed"
     }
   }
 },{
@@ -50,80 +59,52 @@
   "apps": {
     "dlu": {
       "text": "DVC QC Pending",
-      "alertType": "warning",
-      "showDownload": true
+      "alertType": "warning"
+    },
+    "dmd": {
+      "text": "Ready for review"
     }
   }
 },{
   "_id": {
     "$oid": "5ecd777c2e72d5a73904dce4"
   },
-  "state": "PACKAGE_LEVEL_REVIEW_STARTED",
+  "state": "PACKAGE_REVIEW_STARTED",
   "apps": {
     "dlu": {
       "text": "DVC QC Pending",
-      "alertType": "warning",
-      "showDownload": true
+      "alertType": "warning"
+    },
+    "dmd": {
+      "text": "Package Review Started"
     }
   }
 },{
   "_id": {
     "$oid": "5ecd778f2e72d5a73904dce5"
   },
-  "state": "PACKAGE_LEVEL_REVIEW_SUCCEEDED",
+  "state": "PACKAGE_REVIEW_SUCCEEDED",
   "apps": {
     "dlu": {
       "text": "DVC QC Pending",
-      "alertType": "warning",
-      "showDownload": true
+      "alertType": "warning"
+    },
+    "dmd": {
+      "text": "Package Review Succeeded"
     }
   }
 },{
   "_id": {
     "$oid": "5ecd77a42e72d5a73904dce6"
   },
-  "state": "PACKAGE_LEVEL_REVIEW_FAILED",
+  "state": "PACKAGE_REVIEW_FAILED",
   "apps": {
     "dlu": {
       "text": "DVC QC Rejected",
-      "alertType": "danger",
-      "showDownload": true
-    }
-  }
-},{
-  "_id": {
-    "$oid": "5ecd78882e72d5a73904dce7"
-  },
-  "state": "PROPERTY_LEVEL_REVIEW_STARTED",
-  "apps": {
-    "dlu": {
-      "text": "DVC QC Pending",
-      "alertType": "warning",
-      "showDownload": true
-    }
-  }
-},{
-  "_id": {
-    "$oid": "5ecd78ab2e72d5a73904dce8"
-  },
-  "state": "PROPERTY_LEVEL_REVIEW_FAILED",
-  "apps": {
-    "dlu": {
-      "text": "DVC QC Rejected",
-      "alertType": "danger",
-      "showDownload": true
-    }
-  }
-},{
-  "_id": {
-    "$oid": "5ecd78c32e72d5a73904dce9"
-  },
-  "state": "PROPERTY_LEVEL_REVIEW_SUCCEEDED",
-  "apps": {
-    "dlu": {
-      "text": "DVC QC Approved",
-      "alertType": "success",
-      "showDownload": true
+      "alertType": "danger"
+    },
+    "dmd": {
+      "text": "Package Review Failed"
     }
   }
 }]

data/tokens-template.json

@@ -0,0 +1,7 @@
+{
+  "_id":"5f3ce8525471f40049fd339f",
+  "tokenString":"bdVFkm4OBPnv6g7fRElFWexOSAZWBnwp4LPLOFkQJMt2",
+  "shibId":"[email protected]",
+  "expiration":"2021-08-19T08:52:34.539Z",
+  "active":true
+}

scripts/dataPromotion/.env.example

@@ -1,6 +1,7 @@
 minio_access_key =
 minio_secret_key =
 destination_bucket = "kpmp-knowledge-environment"
+source_bucket = "orion-data-uploads"
 datalake_dir =
 minio_host = "localhost:9000"
 mysql_user =

scripts/dataPromotion/README.md

@@ -3,28 +3,14 @@ pip install -r requirements.txt
 
 # Moving files from DL to S3
 1. Add packageIDs,filenames to files_to_s3.txt one per line
-2. Execute 'python datalakeToS3.py'
+2. Execute 'python filesToS3.py'
 
-# Utility scripts for moving data into Knowledge Environment Database
-1. Uncomment what you need in indexToKE.py and execute. 
-
-# Creating / Updating index records for Atlas portal
-## Option 1: 
-1. Execute 'python datalakeToAtlasIndex.py'
-2. Follow prompts
-
-## Option 2:
-1. Create a comma-delimited file with the same headers as package_to_atlas_index.csv
-2. Execute 'python datalakeToAtlasIndex.py -f my_package_to_atlas_index.csv'
-3. Redirect output to file or copy/paste into your favorite POST client. 
-
-## Option 3:
-### Generate using the Knowledge Environment database. 
-Requirements: Make sure the knowledge_environment MYSQL database is available on 3306.
-### All records
-1. Execute python keDatabaseToAtlasIndex.py' without arguments 
-### By release version
-1. Execute python keDatabaseToAtlasIndex.py -v <release_ver>'
-### Per file
-1. Execute python keDatabaseToAtlasIndex.py -f <file_>'
+# Move the files from “file_pending” table to the “file” table in the Staging DB
+1. Requires a connection to the DLU Mongo and the Staging DB MySQL (e.g. through tunnels)
+2. Expression Matrix files get a filesize of 0 and should be updated when they are created.
+3. Execute 'python filesToKE.py'
 
+# Adds clinical data and participants from a CSV file to the Staging Database
+1. Requires a connection to the Staging DB MySQL (e.g. through a tunnel)
+2. Edit script to point to clinical .csv file
+3. Execute 'python clinicalToKE.py'