Batch backup restore

src/main/java/jasper/component/Backup.java

@@ -26,7 +26,9 @@
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.PlatformTransactionManager;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.transaction.support.TransactionTemplate;
 import org.springframework.util.StreamUtils;
 
 import javax.persistence.EntityManager;
@@ -65,6 +67,8 @@ public class Backup {
 	@Autowired
 	EntityManager entityManager;
 	@Autowired
+	PlatformTransactionManager transactionManager;
+	@Autowired
 	Storage storage;
 
 	@Async
@@ -179,19 +183,38 @@ public void restore(String origin, String id, BackupOptionsDto options) {
 
 	private <T extends Cursor> void restoreRepo(JpaRepository<T, ?> repo, String origin, InputStream file, Class<T> type) {
 		if (file == null) return; // Silently ignore missing files
-        var it = new JsonArrayStreamDataSupplier<>(file, type, objectMapper);
-		it.forEachRemaining(t -> {
-			try {
-				if (!isSubOrigin(origin, t.getOrigin())) t.setOrigin(origin);
-				repo.save(t);
-			} catch (Exception e) {
-				try {
-					logger.error("Skipping {} due to constraint violation", objectMapper.writeValueAsString(t), e);
-				} catch (JsonProcessingException ex) {
-					logger.error("Skipping {} due to constraint violation", t, e);
-				}
+		AtomicBoolean done = new AtomicBoolean(false);
+		var it = new JsonArrayStreamDataSupplier<>(file, type, objectMapper);
+		int count = 0;
+		try {
+			while (!done.get()) {
+				TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
+				transactionTemplate.execute(status -> {
+					for (var i = 0; i < this.props.getRestoreBatchSize(); i++) {
+						if (!it.hasNext()) {
+							done.set(true);
+							break;
+						}
+						var t = it.next();
+						try {
+							if (!isSubOrigin(origin, t.getOrigin())) t.setOrigin(origin);
+							repo.save(t);
+						} catch (Exception e) {
+							try {
+								logger.error("{} Skipping {} {} due to constraint violation", origin, type.getSimpleName(), objectMapper.writeValueAsString(t), e);
+							} catch (JsonProcessingException ex) {
+								logger.error("{} Skipping {} {} due to constraint violation", origin, type.getSimpleName(), type, e);
+							}
+						}
+					}
+					return null;
+				});
+				count++;
+				logger.info("{} {} {} restored...", origin, type.getSimpleName(), count * this.props.getRestoreBatchSize());
 			}
-		});
+		} catch (Exception e) {
+			logger.error("Failed to restore", e);
+		}
     }
 
 	@Timed(value = "jasper.backup", histogram = true)

src/main/java/jasper/config/Props.java

@@ -30,6 +30,7 @@ public class Props {
 	private boolean debug = false;
 	private int maxSources = 1000;
 	private int backupBufferSize = 1000000;
+	private int restoreBatchSize = 500;
 	private int maxEtagPageSize = 300;
 	private int ingestMaxRetry = 5;
 	private String[] modSeals;

src/main/java/jasper/repository/OriginMixin.java

@@ -1,7 +1,10 @@
 package jasper.repository;
 
+import org.springframework.data.jpa.repository.Modifying;
+
 import java.time.Instant;
 
 public interface OriginMixin {
+	@Modifying
 	void deleteByOriginAndModifiedLessThanEqual(String origin, Instant olderThan);
 }

src/main/java/jasper/service/OriginService.java

@@ -18,6 +18,7 @@
 import org.springframework.transaction.annotation.Transactional;
 
 import java.io.IOException;
+import java.time.Duration;
 import java.time.Instant;
 
 import static jasper.security.AuthoritiesConstants.ADMIN;
@@ -31,11 +32,11 @@ public class OriginService {
 	@Autowired
 	ExtRepository extRepository;
 	@Autowired
+	UserRepository userRepository;
+	@Autowired
 	PluginRepository pluginRepository;
 	@Autowired
 	TemplateRepository templateRepository;
-	@Autowired
-	UserRepository userRepository;
 
 	@Autowired
 	Replicator replicator;
@@ -63,6 +64,7 @@ public void pull(String url, String origin) throws FeedException, IOException {
 	@PreAuthorize("@auth.hasRole('MOD') and @auth.subOrigin(#origin)")
 	@Timed(value = "jasper.service", extraTags = {"service", "origin"}, histogram = true)
 	public void delete(String origin, Instant olderThan) {
+		var start = Instant.now();
 		logger.info("Deleting origin {} older than {}", origin, olderThan);
 		refRepository.deleteByOriginAndModifiedLessThanEqual(origin, olderThan);
 		extRepository.deleteByOriginAndModifiedLessThanEqual(origin, olderThan);
@@ -72,5 +74,6 @@ public void delete(String origin, Instant olderThan) {
 			templateRepository.deleteByOriginAndModifiedLessThanEqual(origin, olderThan);
 		}
 		logger.info("Finished deleting origin {} older than {}", origin, olderThan);
+		logger.info("Deleting Duration {}", Duration.between(start, Instant.now()));
 	}
 }