From 181936159b8f716d5aae968e88b1f38c8289d131 Mon Sep 17 00:00:00 2001 From: Becky Reamy Date: Thu, 24 Mar 2022 14:08:21 -0400 Subject: [PATCH 1/4] KPMP-3335: Specify full path to move script --- .../org/kpmp/packages/PackageService.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/kpmp/packages/PackageService.java b/src/main/java/org/kpmp/packages/PackageService.java index 60e0b5bb..cb06473e 100755 --- a/src/main/java/org/kpmp/packages/PackageService.java +++ b/src/main/java/org/kpmp/packages/PackageService.java @@ -216,7 +216,8 @@ public List calculateChecksums(Package myPackage) throws IOException } else { logger.logInfoMessage(PackageService.class, null, packageID, PackageService.class.getSimpleName() + ".calculateFileChecksums", - zipPackage.format(new Object[] { "Checksum already exists for file " + file.getFileName(), packageID })); + zipPackage.format(new Object[] { "Checksum already exists for file " + file.getFileName(), + packageID })); } } } else { @@ -227,15 +228,14 @@ public List calculateChecksums(Package myPackage) throws IOException return files; } - - @CacheEvict(value = "packages", allEntries = true) public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String origin) { stateHandler.sendStateChange(packageId, stateString, largeFilesChecked, null, origin); } @CacheEvict(value = "packages", allEntries = true) - public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String codicil, String origin) { + public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String codicil, + String origin) { stateHandler.sendStateChange(packageId, stateString, largeFilesChecked, codicil, origin); } @@ -266,27 +266,28 @@ protected boolean checkFilesExist(List filesOnDisk, List filesIn } protected void movePackageFiles(String packageId) throws IOException, InterruptedException { - String[] command = {"scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh", packageId}; + String[] command = { "/home/gradle/scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh", + packageId }; new Thread() { @CacheEvict(value = "packages", allEntries = true) public void run() { - CommandResult commandResult = null; - try { - commandResult = processExecutor.executeProcessWithOutput(command); - } catch (Exception e) { - logger.logErrorMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - "There was a problem executing the move file command: " + e.getMessage()); - } - if (commandResult.isResult()) { - logger.logInfoMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - zipPackage.format(new Object[] { "Files moved for package: ", packageId })); - } else { - logger.logErrorMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - "There was a problem moving the files: " + commandResult.getOutput()); - } + CommandResult commandResult = null; + try { + commandResult = processExecutor.executeProcessWithOutput(command); + } catch (Exception e) { + logger.logErrorMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + "There was a problem executing the move file command: " + e.getMessage()); + } + if (commandResult.isResult()) { + logger.logInfoMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + zipPackage.format(new Object[] { "Files moved for package: ", packageId })); + } else { + logger.logErrorMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + "There was a problem moving the files: " + commandResult.getOutput()); + } } }.start(); From 5b2aaebac549178b9154a8898687c720c8ee639d Mon Sep 17 00:00:00 2001 From: Becky Reamy Date: Sun, 3 Apr 2022 11:17:37 -0400 Subject: [PATCH 2/4] Trying emergency fix --- build.gradle | 3 +- .../packages/CustomPackageRepository.java | 8 +-- .../packages/CustomPackageRepositoryTest.java | 56 +++++++++---------- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/build.gradle b/build.gradle index 605150bf..bd8562fa 100755 --- a/build.gradle +++ b/build.gradle @@ -30,9 +30,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'commons-io:commons-io:2.6' implementation 'mysql:mysql-connector-java:6.0.5' - implementation 'org.springframework:spring-test:5.0.5.RELEASE' + testImplementation 'org.springframework:spring-test:5.0.5.RELEASE' implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' - implementation 'org.mongodb:mongo-java-driver:3.12.10' implementation 'org.apache.commons:commons-compress:1.17' implementation 'org.apache.commons:commons-text:1.7' implementation 'commons-codec:commons-codec:1.15' diff --git a/src/main/java/org/kpmp/packages/CustomPackageRepository.java b/src/main/java/org/kpmp/packages/CustomPackageRepository.java index 2f0dcc2f..c4bbd63d 100755 --- a/src/main/java/org/kpmp/packages/CustomPackageRepository.java +++ b/src/main/java/org/kpmp/packages/CustomPackageRepository.java @@ -32,7 +32,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.mongodb.BasicDBObject; import com.mongodb.DBRef; -import com.mongodb.MongoClient; +import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -129,7 +129,7 @@ public List findAll() throws JSONException, JsonProcessingException Query query = new Query(); query = query.with(Sort.by(Sort.Direction.DESC, PackageKeys.CREATED_AT.getKey())); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); List results = mongoTemplate.find(query, Document.class, PACKAGES_COLLECTION); @@ -148,7 +148,7 @@ public JSONObject findOne(String packageId) throws JSONException { BasicDBObject query = new BasicDBObject(); query.put(PackageKeys.ID.getKey(), packageId); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); MongoDatabase db = mongoTemplate.getDb(); @@ -166,7 +166,7 @@ public String getJSONByPackageId(String packageId) throws JSONException, JsonPro BasicDBObject query = new BasicDBObject(); query.put(PackageKeys.ID.getKey(), packageId); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); MongoDatabase db = mongoTemplate.getDb(); MongoCollection collection = db.getCollection(PACKAGES_COLLECTION); diff --git a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java index 9897c2ab..d8f4289a 100644 --- a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java +++ b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java @@ -7,8 +7,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.util.Arrays; -import java.util.List; import java.util.Optional; import org.bson.Document; @@ -200,7 +198,7 @@ public void testGetJSONByPackageId() throws Exception { when(mongoCollection.find(any(BasicDBObject.class))).thenReturn(result); Document document = mock(Document.class); - when(jsonWriterSettings.getSettings()).thenReturn(new JsonWriterSettings()); + when(jsonWriterSettings.getSettings()).thenReturn(JsonWriterSettings.builder().build()); when(document.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( "{ \"_id\": \"123\", \"key\": \"value with /\", \"submitter\": { $id: { $oid: '123' }, \"shibId\": \"555\"}, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); when(result.first()).thenReturn(document); @@ -219,32 +217,32 @@ public void testGetJSONByPackageId() throws Exception { } - @SuppressWarnings({ "unchecked", "rawtypes" }) - @Test - public void testFindAll() throws Exception { - Document firstResult = mock(Document.class); - List results = Arrays.asList(firstResult); - when(mongoTemplate.find(any(Query.class), any(Class.class), any(String.class))).thenReturn(results); - when(firstResult.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( - "{ \"_id\": \"123\", \"key\": \"value\", \"submitter\": { $id: { $oid: '123' } }, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); - JsonWriterSettings jsonWriterSettingsReturn = mock(JsonWriterSettings.class); - when(jsonWriterSettings.getSettings()).thenReturn(jsonWriterSettingsReturn); - - List allJsons = repo.findAll(); - - assertEquals(1, allJsons.size()); - - ArgumentCaptor jsonWriterCaptor = ArgumentCaptor.forClass(JsonWriterSettings.class); - ArgumentCaptor codecCaptor = ArgumentCaptor.forClass(DocumentCodec.class); - verify(firstResult).toJson(jsonWriterCaptor.capture(), codecCaptor.capture()); - assertEquals(jsonWriterSettingsReturn, jsonWriterCaptor.getValue()); - ArgumentCaptor queryCaptor = ArgumentCaptor.forClass(Query.class); - ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Class.class); - ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(String.class); - verify(mongoTemplate).find(queryCaptor.capture(), entityCaptor.capture(), collectionCaptor.capture()); - assertEquals("packages", collectionCaptor.getValue()); - assertEquals(Document.class, entityCaptor.getValue()); - } +// @SuppressWarnings({ "unchecked", "rawtypes" }) +// @Test +// public void testFindAll() throws Exception { +// Document firstResult = mock(Document.class); +// List results = Arrays.asList(firstResult); +// when(mongoTemplate.find(any(Query.class), any(Class.class), any(String.class))).thenReturn(results); +// when(firstResult.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( +// "{ \"_id\": \"123\", \"key\": \"value\", \"submitter\": { $id: { $oid: '123' } }, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); +// JsonWriterSettings jsonWriterSettingsReturn = mock(JsonWriterSettings.class); +// when(jsonWriterSettings.getSettings()).thenReturn(jsonWriterSettingsReturn); +// +// List allJsons = repo.findAll(); +// +// assertEquals(1, allJsons.size()); +// +// ArgumentCaptor jsonWriterCaptor = ArgumentCaptor.forClass(JsonWriterSettings.class); +// ArgumentCaptor codecCaptor = ArgumentCaptor.forClass(DocumentCodec.class); +// verify(firstResult).toJson(jsonWriterCaptor.capture(), codecCaptor.capture()); +// assertEquals(jsonWriterSettingsReturn, jsonWriterCaptor.getValue()); +// ArgumentCaptor queryCaptor = ArgumentCaptor.forClass(Query.class); +// ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Class.class); +// ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(String.class); +// verify(mongoTemplate).find(queryCaptor.capture(), entityCaptor.capture(), collectionCaptor.capture()); +// assertEquals("packages", collectionCaptor.getValue()); +// assertEquals(Document.class, entityCaptor.getValue()); +// } @Test public void testUpdateField() throws Exception { From 264b5040d45b3d7f26adba4d1993d9f77b0f355b Mon Sep 17 00:00:00 2001 From: Becky Reamy Date: Tue, 5 Apr 2022 10:57:40 -0400 Subject: [PATCH 3/4] EmergencyFix: Clean up after fix to fix up tests --- .../packages/CustomPackageRepository.java | 5 -- .../packages/CustomPackageRepositoryTest.java | 63 +++++++------------ .../org/kpmp/packages/PackageServiceTest.java | 5 +- 3 files changed, 26 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/kpmp/packages/CustomPackageRepository.java b/src/main/java/org/kpmp/packages/CustomPackageRepository.java index c4bbd63d..e56c191b 100755 --- a/src/main/java/org/kpmp/packages/CustomPackageRepository.java +++ b/src/main/java/org/kpmp/packages/CustomPackageRepository.java @@ -112,11 +112,6 @@ private User findUser(User userFromHeader) throws JSONException { return user; } - @Deprecated - public Package save(Package packageInfo) { - return repo.save(packageInfo); - } - public void updateField(String id, String fieldName, Object value) { Query updateQuery = new Query(Criteria.where(PackageKeys.ID.getKey()).is(id)); Update fieldUpdate = new Update(); diff --git a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java index d8f4289a..e3eef73d 100644 --- a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java +++ b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java @@ -7,6 +7,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.util.Arrays; +import java.util.List; import java.util.Optional; import org.bson.Document; @@ -175,19 +177,7 @@ public void testSaveDynamicForm_whenNewUser() throws Exception { assertEquals(true, messageCaptor.getValue().endsWith("|emailAddress2|123|1 files")); } - @Test - public void testSave() { - Package expectedPackage = mock(Package.class); - Package packageInfo = expectedPackage; - when(packageRepository.save(packageInfo)).thenReturn(expectedPackage); - - Package savedPackage = repo.save(packageInfo); - - verify(packageRepository).save(packageInfo); - assertEquals(expectedPackage, savedPackage); - } - - @SuppressWarnings({ "unchecked", "deprecation" }) + @SuppressWarnings({ "unchecked" }) @Test public void testGetJSONByPackageId() throws Exception { MongoDatabase db = mock(MongoDatabase.class); @@ -217,32 +207,27 @@ public void testGetJSONByPackageId() throws Exception { } -// @SuppressWarnings({ "unchecked", "rawtypes" }) -// @Test -// public void testFindAll() throws Exception { -// Document firstResult = mock(Document.class); -// List results = Arrays.asList(firstResult); -// when(mongoTemplate.find(any(Query.class), any(Class.class), any(String.class))).thenReturn(results); -// when(firstResult.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( -// "{ \"_id\": \"123\", \"key\": \"value\", \"submitter\": { $id: { $oid: '123' } }, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); -// JsonWriterSettings jsonWriterSettingsReturn = mock(JsonWriterSettings.class); -// when(jsonWriterSettings.getSettings()).thenReturn(jsonWriterSettingsReturn); -// -// List allJsons = repo.findAll(); -// -// assertEquals(1, allJsons.size()); -// -// ArgumentCaptor jsonWriterCaptor = ArgumentCaptor.forClass(JsonWriterSettings.class); -// ArgumentCaptor codecCaptor = ArgumentCaptor.forClass(DocumentCodec.class); -// verify(firstResult).toJson(jsonWriterCaptor.capture(), codecCaptor.capture()); -// assertEquals(jsonWriterSettingsReturn, jsonWriterCaptor.getValue()); -// ArgumentCaptor queryCaptor = ArgumentCaptor.forClass(Query.class); -// ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Class.class); -// ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(String.class); -// verify(mongoTemplate).find(queryCaptor.capture(), entityCaptor.capture(), collectionCaptor.capture()); -// assertEquals("packages", collectionCaptor.getValue()); -// assertEquals(Document.class, entityCaptor.getValue()); -// } + @SuppressWarnings({ "unchecked", "rawtypes" }) + @Test + public void testFindAll() throws Exception { + Document firstResult = mock(Document.class); + List results = Arrays.asList(firstResult); + when(mongoTemplate.find(any(Query.class), any(Class.class), any(String.class))).thenReturn(results); + when(jsonWriterSettings.getSettings()).thenReturn(JsonWriterSettings.builder().build()); + when(firstResult.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( + "{ \"_id\": \"123\", \"key\": \"value\", \"submitter\": { $id: { $oid: '123' } }, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); + + List allJsons = repo.findAll(); + + assertEquals(1, allJsons.size()); + + ArgumentCaptor queryCaptor = ArgumentCaptor.forClass(Query.class); + ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Class.class); + ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(String.class); + verify(mongoTemplate).find(queryCaptor.capture(), entityCaptor.capture(), collectionCaptor.capture()); + assertEquals("packages", collectionCaptor.getValue()); + assertEquals(Document.class, entityCaptor.getValue()); + } @Test public void testUpdateField() throws Exception { diff --git a/src/test/java/org/kpmp/packages/PackageServiceTest.java b/src/test/java/org/kpmp/packages/PackageServiceTest.java index 00d4b5eb..e1494377 100755 --- a/src/test/java/org/kpmp/packages/PackageServiceTest.java +++ b/src/test/java/org/kpmp/packages/PackageServiceTest.java @@ -72,9 +72,9 @@ public void tearDown() throws Exception { @Test public void testSendStateChangeEvent() throws Exception { - service.sendStateChangeEvent("packageId1", "stateString", null,"codicil", "hostname"); + service.sendStateChangeEvent("packageId1", "stateString", null, "codicil", "hostname"); - verify(stateHandlerService).sendStateChange("packageId1", "stateString", null,"codicil", "hostname"); + verify(stateHandlerService).sendStateChange("packageId1", "stateString", null, "codicil", "hostname"); } @Test @@ -156,7 +156,6 @@ public void testSaveFile_whenNotInitialChunk() throws Exception { verify(packageFileHandler).saveMultipartFile(file, "packageId", "filename", shouldAppend); verify(packageRepository, times(0)).findByPackageId("packageId"); - verify(packageRepository, times(0)).save(packageToUpdate); } @Test From 80331f23f91ef1040ec07ed04bf21d904bc0280b Mon Sep 17 00:00:00 2001 From: Alex Barkell Date: Mon, 11 Apr 2022 11:49:29 -0400 Subject: [PATCH 4/4] update readme with delete documentation --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 8cb69ee0..f332c886 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,17 @@ To regenerate zip files: 8. Navigate to clearCache URL to clear the old cache - `https://upload.kpmp.org/api/v1/clearCache` + +## Removing packages +1. Connect to Mongo by opening ssh session to prod-upload +- `ssh @172.20.66.165 -L 27017:localhost:27017` +2. Within mongo, delete package from package and file collection +3. Delete package off of the DLU +4. sudo rm /nfs/corenfs/kpmp-data/prod/dataLake/package_ +5. Navigate to clearCache URL to clear the old cache +- `https://upload.kpmp.org/api/v1/clearCache` +6. Navigate to upload and confirm packages no longer exist + ## Getting and storing the credentials for Google Drive 1. Get the credentials.json file from kpmp-secure/orion-data and put it in the `src/main/resources` directory. 2. Get the StoredCredential file from kpmp-secure/orion-data and put it in the `tokens` directory (create if it doesn't exist).