#2502 : mass nifti deletion update

shanoir-ng-datasets/src/main/java/org/shanoir/ng/dataset/service/DatasetServiceImpl.java

@@ -16,7 +16,7 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import jakarta.transaction.Transactional;
+import org.apache.commons.collections4.ListUtils;
 import org.apache.commons.io.FileUtils;
 import org.apache.solr.client.solrj.SolrServerException;
 import org.shanoir.ng.dataset.dto.VolumeByFormatDTO;
@@ -53,10 +53,14 @@
 import org.springframework.amqp.rabbit.core.RabbitTemplate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Lazy;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.http.HttpStatus;
+import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
 import org.springframework.util.CollectionUtils;
 import org.springframework.web.util.UriUtils;
 
@@ -66,6 +70,8 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.Future;
 import java.util.stream.Collectors;
 
 /**
@@ -105,9 +111,11 @@ public class DatasetServiceImpl implements DatasetService {
 	private boolean dicomWeb;
 
 	@Autowired
+	@Lazy
 	private DatasetProcessingService processingService;
 
 	@Autowired
+	@Lazy
 	private ProcessingResourceService processingResourceService;
 
 	@Autowired
@@ -371,19 +379,67 @@ public List<Object[]> queryStatistics(String studyNameInRegExp, String studyName
 	}
 
 	@Override
+	@Async
 	public void deleteNiftis(Long studyId) {
-		List<Dataset> datasets = this.findByStudyId(studyId);
-		for (Dataset dataset : datasets) {
+		List<Long> datasets = repository.findIdsByStudyId(studyId);
+
+		ShanoirEvent event = new ShanoirEvent(ShanoirEventType.DELETE_NIFTI_EVENT, studyId.toString(), KeycloakUtil.getTokenUserId(), "Preparing deletion of niftis", ShanoirEvent.IN_PROGRESS, 0, studyId);
+		shanoirEventService.publishEvent(event);
+		try {
+			int total = datasets.size();
+			updateEvent(0f, event, studyId);
+			for (List<Long> partition : ListUtils.partition(datasets, 1000)){
+				deletePartitionOfNiftis(partition, total, event).get();
+			}
+			updateEvent(1f, event, studyId);
+		} catch (Exception e) {
+			updateEvent(-1f, event, studyId, e);
+
+		}
+	}
+
+	@Transactional
+	protected Future<Void> deletePartitionOfNiftis(List<Long> partition, float total, ShanoirEvent event) {
+
+		float progress = event.getProgress();
+		for (Dataset dataset : repository.findAllById(partition)) {
+			progress += 1f / total;
+			updateEvent(progress, event);
 			deleteNifti(dataset);
 		}
+		return CompletableFuture.completedFuture(null);
+	}
+
+	protected void updateEvent(float progress, ShanoirEvent event) {
+		updateEvent(progress, event, null, null);
+	}
+
+    protected void updateEvent(float progress, ShanoirEvent event, Long id) {
+		updateEvent(progress, event, id, null);
+	}
+
+	@Transactional(propagation = Propagation.REQUIRES_NEW)
+    protected void updateEvent(float progress, ShanoirEvent event, Long id, Exception e) {
+		event.setProgress(progress);
+		if (progress == 1f) {
+			event.setStatus(ShanoirEvent.SUCCESS);
+			event.setMessage("Deleting nifti for study: " + id + ": Success.");
+		} else if (progress == -1f) {
+			LOG.error("Could not properly delete niftis: ", e);
+			event.setStatus(ShanoirEvent.ERROR);
+			event.setMessage("Deleting nifti for study: " + id + ": Error. " + e.getMessage());
+		} else if(Objects.nonNull(id)){
+			event.setMessage("Deleting nifti for study: " + id);
+		}
+		shanoirEventService.publishEvent(event);
 	}
 
 	/**
 	 * Deletes nifti on file server
 	 * @param dataset
 	 */
-	private void deleteNifti(Dataset dataset) {
-		List<DatasetExpression> expressionsToDelete = new ArrayList<>();
+	public void deleteNifti(Dataset dataset) {
+		Set<DatasetExpression> expressionsToDelete = new HashSet<>();
 
 		for (Iterator<DatasetExpression> iterex = dataset.getDatasetExpressions().iterator(); iterex.hasNext(); ) {
 			DatasetExpression expression = iterex.next();

shanoir-ng-ms-common/src/main/java/org/shanoir/ng/shared/event/ShanoirEventType.java

@@ -95,8 +95,9 @@ public class ShanoirEventType {
 
 	public static final String CHECK_QUALITY_EVENT = "checkQuality.event";
 
-	/**
-	 * Index all datasets in solr
-	 */
+	/** Index all datasets in solr */
 	public static final String SOLR_INDEX_ALL_EVENT = "solrIndexAll.event";
+
+	/** TEMPORARY: Delete existing niftis in shanoir. */
+	public static final String DELETE_NIFTI_EVENT = "deleteNiftis.event";
 }

shanoir-ng-users/src/main/java/org/shanoir/ng/events/ShanoirEventsService.java

@@ -1,16 +1,9 @@
 package org.shanoir.ng.events;
 
-import java.io.IOException;
-import java.util.*;
-import java.util.stream.Collectors;
-
-import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.time.DateUtils;
 import org.shanoir.ng.shared.event.ShanoirEventType;
 import org.shanoir.ng.tasks.AsyncTaskApiController;
 import org.shanoir.ng.tasks.UserSseEmitter;
-import org.shanoir.ng.user.model.User;
-import org.shanoir.ng.user.model.dto.UserDTO;
 import org.shanoir.ng.user.repository.UserRepository;
 import org.shanoir.ng.utils.KeycloakUtil;
 import org.shanoir.ng.utils.Utils;
@@ -19,16 +12,16 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
-import org.springframework.data.util.Pair;
-import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
-import org.springframework.http.ResponseEntity;
 import org.springframework.scheduling.annotation.Scheduled;
-import org.springframework.security.access.prepost.PostAuthorize;
-import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.stereotype.Service;
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+
 /**
  * Service managing ShanoirEvents
  * @author fli
@@ -59,7 +52,8 @@ public void addEvent(ShanoirEvent event) {
 				|| ShanoirEventType.COPY_DATASET_EVENT.equals(event.getEventType())
 				|| ShanoirEventType.CHECK_QUALITY_EVENT.equals(event.getEventType())
 				|| ShanoirEventType.DOWNLOAD_STATISTICS_EVENT.equals(event.getEventType())
-				|| ShanoirEventType.DELETE_EXAMINATION_EVENT.equals(event.getEventType())) {
+				|| ShanoirEventType.DELETE_EXAMINATION_EVENT.equals(event.getEventType())
+				|| ShanoirEventType.DELETE_NIFTI_EVENT.equals(event.getEventType())) {
 			sendSseEventsToUI(saved);
 		}
 	}

shanoir-ng-users/src/main/java/org/shanoir/ng/tasks/AsyncTaskApiController.java

@@ -37,7 +37,7 @@ public class AsyncTaskApiController implements AsyncTaskApi {
 	public ResponseEntity<List<ShanoirEventLight>> findTasks() {
 		Long userId = KeycloakUtil.getTokenUserId();
 
-		List<ShanoirEventLight> taskList = taskService.getEventsByUserAndType(userId, ShanoirEventType.IMPORT_DATASET_EVENT, ShanoirEventType.COPY_DATASET_EVENT, ShanoirEventType.EXECUTION_MONITORING_EVENT, ShanoirEventType.CHECK_QUALITY_EVENT, ShanoirEventType.SOLR_INDEX_ALL_EVENT, ShanoirEventType.DOWNLOAD_STATISTICS_EVENT, ShanoirEventType.DELETE_EXAMINATION_EVENT);
+		List<ShanoirEventLight> taskList = taskService.getEventsByUserAndType(userId, ShanoirEventType.IMPORT_DATASET_EVENT, ShanoirEventType.COPY_DATASET_EVENT, ShanoirEventType.EXECUTION_MONITORING_EVENT, ShanoirEventType.CHECK_QUALITY_EVENT, ShanoirEventType.SOLR_INDEX_ALL_EVENT, ShanoirEventType.DOWNLOAD_STATISTICS_EVENT, ShanoirEventType.DELETE_EXAMINATION_EVENT, ShanoirEventType.DELETE_NIFTI_EVENT);
 
 		// Get only event with last updates < 7 days
 		Date now = new Date();