diff --git a/README.md b/README.md index 8cb69ee0..f332c886 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,17 @@ To regenerate zip files: 8. Navigate to clearCache URL to clear the old cache - `https://upload.kpmp.org/api/v1/clearCache` + +## Removing packages +1. Connect to Mongo by opening ssh session to prod-upload +- `ssh @172.20.66.165 -L 27017:localhost:27017` +2. Within mongo, delete package from package and file collection +3. Delete package off of the DLU +4. sudo rm /nfs/corenfs/kpmp-data/prod/dataLake/package_ +5. Navigate to clearCache URL to clear the old cache +- `https://upload.kpmp.org/api/v1/clearCache` +6. Navigate to upload and confirm packages no longer exist + ## Getting and storing the credentials for Google Drive 1. Get the credentials.json file from kpmp-secure/orion-data and put it in the `src/main/resources` directory. 2. Get the StoredCredential file from kpmp-secure/orion-data and put it in the `tokens` directory (create if it doesn't exist). diff --git a/build.gradle b/build.gradle index 605150bf..bd8562fa 100755 --- a/build.gradle +++ b/build.gradle @@ -30,9 +30,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'commons-io:commons-io:2.6' implementation 'mysql:mysql-connector-java:6.0.5' - implementation 'org.springframework:spring-test:5.0.5.RELEASE' + testImplementation 'org.springframework:spring-test:5.0.5.RELEASE' implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' - implementation 'org.mongodb:mongo-java-driver:3.12.10' implementation 'org.apache.commons:commons-compress:1.17' implementation 'org.apache.commons:commons-text:1.7' implementation 'commons-codec:commons-codec:1.15' diff --git a/src/main/java/org/kpmp/packages/CustomPackageRepository.java b/src/main/java/org/kpmp/packages/CustomPackageRepository.java index 2f0dcc2f..e56c191b 100755 --- a/src/main/java/org/kpmp/packages/CustomPackageRepository.java +++ b/src/main/java/org/kpmp/packages/CustomPackageRepository.java @@ -32,7 +32,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.mongodb.BasicDBObject; import com.mongodb.DBRef; -import com.mongodb.MongoClient; +import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; @@ -112,11 +112,6 @@ private User findUser(User userFromHeader) throws JSONException { return user; } - @Deprecated - public Package save(Package packageInfo) { - return repo.save(packageInfo); - } - public void updateField(String id, String fieldName, Object value) { Query updateQuery = new Query(Criteria.where(PackageKeys.ID.getKey()).is(id)); Update fieldUpdate = new Update(); @@ -129,7 +124,7 @@ public List findAll() throws JSONException, JsonProcessingException Query query = new Query(); query = query.with(Sort.by(Sort.Direction.DESC, PackageKeys.CREATED_AT.getKey())); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); List results = mongoTemplate.find(query, Document.class, PACKAGES_COLLECTION); @@ -148,7 +143,7 @@ public JSONObject findOne(String packageId) throws JSONException { BasicDBObject query = new BasicDBObject(); query.put(PackageKeys.ID.getKey(), packageId); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); MongoDatabase db = mongoTemplate.getDb(); @@ -166,7 +161,7 @@ public String getJSONByPackageId(String packageId) throws JSONException, JsonPro BasicDBObject query = new BasicDBObject(); query.put(PackageKeys.ID.getKey(), packageId); - CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClient.getDefaultCodecRegistry()); + CodecRegistry codecRegistry = CodecRegistries.fromRegistries(MongoClientSettings.getDefaultCodecRegistry()); DocumentCodec codec = new DocumentCodec(codecRegistry, new BsonTypeClassMap()); MongoDatabase db = mongoTemplate.getDb(); MongoCollection collection = db.getCollection(PACKAGES_COLLECTION); diff --git a/src/main/java/org/kpmp/packages/PackageService.java b/src/main/java/org/kpmp/packages/PackageService.java index 60e0b5bb..cb06473e 100755 --- a/src/main/java/org/kpmp/packages/PackageService.java +++ b/src/main/java/org/kpmp/packages/PackageService.java @@ -216,7 +216,8 @@ public List calculateChecksums(Package myPackage) throws IOException } else { logger.logInfoMessage(PackageService.class, null, packageID, PackageService.class.getSimpleName() + ".calculateFileChecksums", - zipPackage.format(new Object[] { "Checksum already exists for file " + file.getFileName(), packageID })); + zipPackage.format(new Object[] { "Checksum already exists for file " + file.getFileName(), + packageID })); } } } else { @@ -227,15 +228,14 @@ public List calculateChecksums(Package myPackage) throws IOException return files; } - - @CacheEvict(value = "packages", allEntries = true) public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String origin) { stateHandler.sendStateChange(packageId, stateString, largeFilesChecked, null, origin); } @CacheEvict(value = "packages", allEntries = true) - public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String codicil, String origin) { + public void sendStateChangeEvent(String packageId, String stateString, String largeFilesChecked, String codicil, + String origin) { stateHandler.sendStateChange(packageId, stateString, largeFilesChecked, codicil, origin); } @@ -266,27 +266,28 @@ protected boolean checkFilesExist(List filesOnDisk, List filesIn } protected void movePackageFiles(String packageId) throws IOException, InterruptedException { - String[] command = {"scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh", packageId}; + String[] command = { "/home/gradle/scripts/processLargeFileUpload/processLargeFileUploadNoGlobus.sh", + packageId }; new Thread() { @CacheEvict(value = "packages", allEntries = true) public void run() { - CommandResult commandResult = null; - try { - commandResult = processExecutor.executeProcessWithOutput(command); - } catch (Exception e) { - logger.logErrorMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - "There was a problem executing the move file command: " + e.getMessage()); - } - if (commandResult.isResult()) { - logger.logInfoMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - zipPackage.format(new Object[] { "Files moved for package: ", packageId })); - } else { - logger.logErrorMessage(PackageService.class, null, packageId, - PackageService.class.getSimpleName() + ".movePackageFiles", - "There was a problem moving the files: " + commandResult.getOutput()); - } + CommandResult commandResult = null; + try { + commandResult = processExecutor.executeProcessWithOutput(command); + } catch (Exception e) { + logger.logErrorMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + "There was a problem executing the move file command: " + e.getMessage()); + } + if (commandResult.isResult()) { + logger.logInfoMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + zipPackage.format(new Object[] { "Files moved for package: ", packageId })); + } else { + logger.logErrorMessage(PackageService.class, null, packageId, + PackageService.class.getSimpleName() + ".movePackageFiles", + "There was a problem moving the files: " + commandResult.getOutput()); + } } }.start(); diff --git a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java index 9897c2ab..e3eef73d 100644 --- a/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java +++ b/src/test/java/org/kpmp/packages/CustomPackageRepositoryTest.java @@ -177,19 +177,7 @@ public void testSaveDynamicForm_whenNewUser() throws Exception { assertEquals(true, messageCaptor.getValue().endsWith("|emailAddress2|123|1 files")); } - @Test - public void testSave() { - Package expectedPackage = mock(Package.class); - Package packageInfo = expectedPackage; - when(packageRepository.save(packageInfo)).thenReturn(expectedPackage); - - Package savedPackage = repo.save(packageInfo); - - verify(packageRepository).save(packageInfo); - assertEquals(expectedPackage, savedPackage); - } - - @SuppressWarnings({ "unchecked", "deprecation" }) + @SuppressWarnings({ "unchecked" }) @Test public void testGetJSONByPackageId() throws Exception { MongoDatabase db = mock(MongoDatabase.class); @@ -200,7 +188,7 @@ public void testGetJSONByPackageId() throws Exception { when(mongoCollection.find(any(BasicDBObject.class))).thenReturn(result); Document document = mock(Document.class); - when(jsonWriterSettings.getSettings()).thenReturn(new JsonWriterSettings()); + when(jsonWriterSettings.getSettings()).thenReturn(JsonWriterSettings.builder().build()); when(document.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( "{ \"_id\": \"123\", \"key\": \"value with /\", \"submitter\": { $id: { $oid: '123' }, \"shibId\": \"555\"}, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); when(result.first()).thenReturn(document); @@ -225,19 +213,14 @@ public void testFindAll() throws Exception { Document firstResult = mock(Document.class); List results = Arrays.asList(firstResult); when(mongoTemplate.find(any(Query.class), any(Class.class), any(String.class))).thenReturn(results); + when(jsonWriterSettings.getSettings()).thenReturn(JsonWriterSettings.builder().build()); when(firstResult.toJson(any(JsonWriterSettings.class), any(DocumentCodec.class))).thenReturn( "{ \"_id\": \"123\", \"key\": \"value\", \"submitter\": { $id: { $oid: '123' } }, \"regenerateZip\": true, \"createdAt\": { $date: 123567 } }"); - JsonWriterSettings jsonWriterSettingsReturn = mock(JsonWriterSettings.class); - when(jsonWriterSettings.getSettings()).thenReturn(jsonWriterSettingsReturn); List allJsons = repo.findAll(); assertEquals(1, allJsons.size()); - ArgumentCaptor jsonWriterCaptor = ArgumentCaptor.forClass(JsonWriterSettings.class); - ArgumentCaptor codecCaptor = ArgumentCaptor.forClass(DocumentCodec.class); - verify(firstResult).toJson(jsonWriterCaptor.capture(), codecCaptor.capture()); - assertEquals(jsonWriterSettingsReturn, jsonWriterCaptor.getValue()); ArgumentCaptor queryCaptor = ArgumentCaptor.forClass(Query.class); ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Class.class); ArgumentCaptor collectionCaptor = ArgumentCaptor.forClass(String.class); diff --git a/src/test/java/org/kpmp/packages/PackageServiceTest.java b/src/test/java/org/kpmp/packages/PackageServiceTest.java index 00d4b5eb..e1494377 100755 --- a/src/test/java/org/kpmp/packages/PackageServiceTest.java +++ b/src/test/java/org/kpmp/packages/PackageServiceTest.java @@ -72,9 +72,9 @@ public void tearDown() throws Exception { @Test public void testSendStateChangeEvent() throws Exception { - service.sendStateChangeEvent("packageId1", "stateString", null,"codicil", "hostname"); + service.sendStateChangeEvent("packageId1", "stateString", null, "codicil", "hostname"); - verify(stateHandlerService).sendStateChange("packageId1", "stateString", null,"codicil", "hostname"); + verify(stateHandlerService).sendStateChange("packageId1", "stateString", null, "codicil", "hostname"); } @Test @@ -156,7 +156,6 @@ public void testSaveFile_whenNotInitialChunk() throws Exception { verify(packageFileHandler).saveMultipartFile(file, "packageId", "filename", shouldAppend); verify(packageRepository, times(0)).findByPackageId("packageId"); - verify(packageRepository, times(0)).save(packageToUpdate); } @Test