From 215542fd3e7c2af62d6299778405ff9ef2cddb75 Mon Sep 17 00:00:00 2001 From: alx652 Date: Sun, 11 Aug 2024 15:16:47 -0400 Subject: [PATCH 1/5] more experimenting --- .../substance/SubstanceCoreConfiguration.java | 4 +- ...SubstanceEasyLegacyBulkLoadController.java | 384 ++++++++++++++++++ 2 files changed, 387 insertions(+), 1 deletion(-) create mode 100644 gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/SubstanceCoreConfiguration.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/SubstanceCoreConfiguration.java index f3c9b96c..025ed2fa 100644 --- a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/SubstanceCoreConfiguration.java +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/SubstanceCoreConfiguration.java @@ -46,7 +46,9 @@ SubstanceHierarchyFinder.class, SubstanceHierarchyFinderConfig.class, ApprovalIdConfiguration.class,RendererOptionsConfig.class, MolWeightCalculatorProperties.class, //legacy bulk load - SubstanceBulkLoadService.class, SubstanceBulkLoadServiceConfiguration.class, SubstanceLegacyBulkLoadController.class, + SubstanceBulkLoadService.class, SubstanceBulkLoadServiceConfiguration.class, +SubstanceLegacyBulkLoadController.class, +SubstanceEasyLegacyBulkLoadController.class, ProcessingJobController.class, ProcessingJobEntityService.class, //used by bulk loader ConsoleFilterService.class, diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java new file mode 100644 index 00000000..679986ed --- /dev/null +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java @@ -0,0 +1,384 @@ +package gsrs.module.substance.controllers; + +import com.fasterxml.jackson.core.JsonProcessingException; +import gsrs.repository.UserProfileRepository; +import gsrs.security.AdminService; +import ix.ginas.models.v1.Substance; +import jdk.internal.loader.Loader; +import lombok.SneakyThrows; +import org.springframework.transaction.support.TransactionTemplate; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import gsrs.controller.GsrsControllerConfiguration; +import gsrs.module.substance.SubstanceEntityService; +import gsrs.module.substance.services.ProcessingJobEntityService; +import gsrs.module.substance.services.SubstanceBulkLoadService; +import gsrs.payload.PayloadController; +import gsrs.repository.PayloadRepository; +import gsrs.security.hasAdminRole; +import gsrs.service.GsrsEntityService; +import gsrs.service.PayloadService; +import ix.core.models.Payload; +import ix.core.models.ProcessingJob; +import ix.core.models.UserProfile; +import ix.core.processing.PayloadProcessor; +import ix.core.processing.TransformedRecord; +import ix.core.stats.Statistics; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; + +import java.io.*; +import java.nio.file.Files; +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + + +@RestController +@Slf4j +public class SubstanceEasyLegacyBulkLoadController { + @Autowired + private AdminService adminService; + + @Autowired + private UserProfileRepository userProfileRepository; + + @Autowired + private SubstanceEntityService substanceEntityService; + + @Autowired + private PayloadService payloadService; + + @Autowired + private PayloadRepository payloadRepository; + + @Autowired + private SubstanceBulkLoadService substanceBulkLoadService; + + @Autowired + private GsrsControllerConfiguration controllerConfiguration; + + @Autowired + private ProcessingJobEntityService processingJobService; + + @Autowired + private PlatformTransactionManager platformTransactionManager; + SubstanceEasyLegacyBulkLoadController(){ + + System.out.println("Inside controller ... SubstanceEasyLegacyBulkLoadController."); + } + @hasAdminRole + @GetMapping("/api/v1/testLoad") + public ResponseEntity testLoad(){ + String resultString = "Hello"; + Resource dataFile = new ClassPathResource( + "rep18.gsrs" + ); + + TransactionTemplate tx = new TransactionTemplate(platformTransactionManager); + tx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + Payload payload = tx.execute(status-> { +// createUser("admin2", Role.values()); + //the old json has user TYLER and FDA-SRS too +// createUser("TYLER", Role.values()); +// createUser("FDA_SRS", Role.values()); + try (InputStream in = dataFile.getInputStream()) { + return payloadService.createPayload(dataFile.getFilename(), "ignore", + in, PayloadService.PayloadPersistType.PERM); + }catch(IOException e){ + throw new UncheckedIOException(e); + } + }); + PayloadProcessor pp = substanceBulkLoadService.submit(SubstanceBulkLoadService.SubstanceBulkLoadParameters.builder() + .payload(payload) + .build()); + + String statKey = pp.key; + boolean done =false; + Statistics statistics=null; + while(!done){ + statistics = substanceBulkLoadService.getStatisticsFor(statKey); + + if(statistics._isDone()){ + System.out.println(statistics); + break; + } + // Thread.sleep(1000); + } + return ResponseEntity.ok(resultString); + + } + @hasAdminRole + @GetMapping("/api/v1/testLoad3") + public ResponseEntity testLoad3(){ + String resultString = "hello 3"; + TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); + Resource dataFile = new ClassPathResource( + "rep18.gsrs" + ); + File f = null; + try { + f = dataFile.getFile(); + } catch (IOException e) { + f = null; + e.printStackTrace(); + } + + // String pathToLoadFile = System.getProperty("ix.ginas.load.file"); + //&& substanceRepository.count()==0 + if (f.getAbsolutePath() != null ) { + // File f = new File(pathToLoadFile); + if(f.exists()) { + transactionTemplate.executeWithoutResult(status-> { + UserProfile up = userProfileRepository.findByUser_UsernameIgnoreCase("admin").standardize(); + + Authentication auth = new UsernamePasswordAuthenticationToken(up.user.username, null, + up.getRoles().stream().map(r -> new SimpleGrantedAuthority("ROLE_" + r.name())).collect(Collectors.toList())); + SecurityContextHolder.getContext().setAuthentication(auth); + }); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { + String line; + Pattern sep = Pattern.compile("\t"); + ObjectMapper mapper = new ObjectMapper(); + int i = 0; + long start = System.currentTimeMillis(); + while ((line = reader.readLine()) != null) { + + String[] cols = sep.split(line); +// System.out.println(cols[2]); + JsonNode json = mapper.readTree(cols[2]); + try { + transactionTemplate.executeWithoutResult(status-> { + try { + Substance s = substanceEntityService.createEntity(json, true).getCreatedEntity(); + if(s != null) { + String uuid = s.uuid.toString(); + uuid = (uuid==null) ? "UUID_NULL": uuid; + System.out.println("Loaded: " + uuid); + } else { + System.out.println("Loaded: " + "SUBSTANCE_NULL"); + } + } catch (IOException e) { + e.printStackTrace(); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + t.printStackTrace(); + } + + + + + + i++; + if (i % 100 == 0) { + System.out.println("loaded record " + i); + } + + } + System.out.println("done loading file"); + long finish = System.currentTimeMillis(); + long millisElapsed = finish - start; + long minutesElapsed = millisElapsed / (60 * 1000); + System.out.println("# Loaded: " + i); + System.out.println("Millis elapsed: " + millisElapsed); + System.out.println("Minutes elapsed: " + minutesElapsed); + + }catch(Throwable t){ + t.printStackTrace(); + } + }else{ + System.err.println("could not find GSRS file: " + f.getAbsolutePath()); + } + } + + return ResponseEntity.ok(resultString); + } + + @hasAdminRole + @GetMapping("/api/v1/testLoad4") + public ResponseEntity testLoad4(){ + String resultString = "hello 4"; + final TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); + transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + Resource dataFile = new ClassPathResource("rep18.gsrs"); + Authentication auth = adminService.getCurrentAdminAuth(); + File f = null; + try { + f = dataFile.getFile(); + } catch (IOException e) { + f = null; + e.printStackTrace(); + } + if (f.getAbsolutePath() != null ) { + if(f.exists()) { +// int parallelism = Runtime.getRuntime().availableProcessors(); + int parallelism = 4; + + List tasks = new ArrayList<>(); + ExecutorService executorService = Executors.newFixedThreadPool(parallelism); +// Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { + reader.lines().forEach(l -> { + // processItem(l,transactionTemplate); + + Future task = executorService.submit(() -> { + Runnable r = ()->{ + _countrun.incrementAndGet(); + processItem(l,transactionTemplate); + }; + adminService.runAs(auth, r); + + }); + tasks.add(task); + }); + + for (Future task : tasks) { + try { + task.get(); + + + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + System.out.println("_countrun:" + _countrun); + System.out.println("_countpi:" + _countpi); + + } catch (IOException e) { + System.out.println("IO Exception during buffered read ..."); + } + + }else{ + System.err.println("could not find GSRS file: " + f.getAbsolutePath()); + } + + } + + return ResponseEntity.ok(resultString); + } + + class MyRunnable implements Runnable { + + String line; + TransactionTemplate transactionTemplate; + AdminService adminService; + Authentication auth; + SubstanceEasyLegacyBulkLoadController parent; + + public MyRunnable( + String line, + TransactionTemplate transactionTemplate, + AdminService adminService, + Authentication auth, + SubstanceEasyLegacyBulkLoadController parent + ) {} + + @Override + public void run() { + Runnable r = ()->{ + parent.processItem(line,transactionTemplate); + }; + adminService.runAs(auth, r); + } + } + + private final String SEP = "\t"; + private final ObjectMapper MAPPER = new ObjectMapper(); + private final AtomicInteger _countrun = new AtomicInteger(0); + private final AtomicInteger _countpi = new AtomicInteger(0); + + + private void processItem(String line, TransactionTemplate transactionTemplate) { + + String[] cols = line.split(SEP); + try { + transactionTemplate.executeWithoutResult(status -> { + _countpi.incrementAndGet(); + try { + JsonNode json = MAPPER.readTree(cols[2]); + Substance s = substanceEntityService.createEntity(json, true).getCreatedEntity(); + if (s != null) { + String uuid = s.uuid.toString(); + uuid = (uuid == null) ? "UUID_NULL" : uuid; + System.out.println("Loaded: " + uuid); + } else { + System.out.println("Loaded: " + "SUBSTANCE_NULL"); + } + } catch (IOException e) { + System.out.println(e.getMessage()); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + System.out.println("THROWABLE: " + t.getMessage()); + } + } + +// class LoaderTask implements Future { +// String line; +// Authentication auth; +// ExecutorService executorService; +// +// LoaderTask(String line, Authentication auth, ExecutorService executorService) { +// this.line = line; +// this.auth = auth; +// Future task = executorService.submit(() -> { +// Runnable r = ()->{ +// processItem(l,transactionTemplate); +// }; +// adminService.runAs(auth, r); +// }); +// +// } +// +// +// @Override +// public boolean cancel(boolean mayInterruptIfRunning) { +// return false; +// } +// +// @Override +// public boolean isCancelled() { +// return false; +// } +// +// @Override +// public boolean isDone() { +// return false; +// } +// +// @Override +// public Object get() throws InterruptedException, ExecutionException { +// return null; +// } +// +// @Override +// public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { +// return null; +// } +// } + +} + + From d689924386c7c7cfd892571cb632c2ba88ddadf9 Mon Sep 17 00:00:00 2001 From: alx652 Date: Mon, 12 Aug 2024 14:51:01 -0400 Subject: [PATCH 2/5] problems with user profile --- ...SubstanceEasyLegacyBulkLoadController.java | 307 +++----------- ...ceEasyLegacyBulkLoadControllerOld.java.old | 400 ++++++++++++++++++ 2 files changed, 465 insertions(+), 242 deletions(-) create mode 100644 gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadControllerOld.java.old diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java index 679986ed..f8ce57dc 100644 --- a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java @@ -1,11 +1,9 @@ package gsrs.module.substance.controllers; -import com.fasterxml.jackson.core.JsonProcessingException; -import gsrs.repository.UserProfileRepository; +// import gsrs.repository.UserProfileRepository; +import gov.nih.ncats.common.executors.BlockingSubmitExecutor; import gsrs.security.AdminService; import ix.ginas.models.v1.Substance; -import jdk.internal.loader.Loader; -import lombok.SneakyThrows; import org.springframework.transaction.support.TransactionTemplate; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -13,40 +11,23 @@ import gsrs.module.substance.SubstanceEntityService; import gsrs.module.substance.services.ProcessingJobEntityService; import gsrs.module.substance.services.SubstanceBulkLoadService; -import gsrs.payload.PayloadController; import gsrs.repository.PayloadRepository; import gsrs.security.hasAdminRole; -import gsrs.service.GsrsEntityService; import gsrs.service.PayloadService; -import ix.core.models.Payload; -import ix.core.models.ProcessingJob; -import ix.core.models.UserProfile; -import ix.core.processing.PayloadProcessor; -import ix.core.processing.TransformedRecord; -import ix.core.stats.Statistics; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.ClassPathResource; import org.springframework.core.io.Resource; import org.springframework.http.ResponseEntity; -import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.core.Authentication; -import org.springframework.security.core.authority.SimpleGrantedAuthority; -import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.transaction.PlatformTransactionManager; import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.support.TransactionTemplate; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; import java.io.*; -import java.nio.file.Files; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; -import java.util.regex.Pattern; -import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; @@ -56,8 +37,8 @@ public class SubstanceEasyLegacyBulkLoadController { @Autowired private AdminService adminService; - @Autowired - private UserProfileRepository userProfileRepository; +// @Autowired +// private UserProfileRepository userProfileRepository; @Autowired private SubstanceEntityService substanceEntityService; @@ -65,151 +46,29 @@ public class SubstanceEasyLegacyBulkLoadController { @Autowired private PayloadService payloadService; - @Autowired - private PayloadRepository payloadRepository; +// @Autowired +// private PayloadRepository payloadRepository; - @Autowired - private SubstanceBulkLoadService substanceBulkLoadService; +// @Autowired +// private SubstanceBulkLoadService substanceBulkLoadService; - @Autowired - private GsrsControllerConfiguration controllerConfiguration; +// @Autowired +// private GsrsControllerConfiguration controllerConfiguration; - @Autowired - private ProcessingJobEntityService processingJobService; +// @Autowired +// private ProcessingJobEntityService processingJobService; @Autowired private PlatformTransactionManager platformTransactionManager; - SubstanceEasyLegacyBulkLoadController(){ - System.out.println("Inside controller ... SubstanceEasyLegacyBulkLoadController."); - } - @hasAdminRole - @GetMapping("/api/v1/testLoad") - public ResponseEntity testLoad(){ - String resultString = "Hello"; - Resource dataFile = new ClassPathResource( - "rep18.gsrs" - ); - - TransactionTemplate tx = new TransactionTemplate(platformTransactionManager); - tx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); - Payload payload = tx.execute(status-> { -// createUser("admin2", Role.values()); - //the old json has user TYLER and FDA-SRS too -// createUser("TYLER", Role.values()); -// createUser("FDA_SRS", Role.values()); - try (InputStream in = dataFile.getInputStream()) { - return payloadService.createPayload(dataFile.getFilename(), "ignore", - in, PayloadService.PayloadPersistType.PERM); - }catch(IOException e){ - throw new UncheckedIOException(e); - } - }); - PayloadProcessor pp = substanceBulkLoadService.submit(SubstanceBulkLoadService.SubstanceBulkLoadParameters.builder() - .payload(payload) - .build()); - - String statKey = pp.key; - boolean done =false; - Statistics statistics=null; - while(!done){ - statistics = substanceBulkLoadService.getStatisticsFor(statKey); - - if(statistics._isDone()){ - System.out.println(statistics); - break; - } - // Thread.sleep(1000); - } - return ResponseEntity.ok(resultString); +// private Authentication auth; - } - @hasAdminRole - @GetMapping("/api/v1/testLoad3") - public ResponseEntity testLoad3(){ - String resultString = "hello 3"; - TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); - Resource dataFile = new ClassPathResource( - "rep18.gsrs" - ); - File f = null; - try { - f = dataFile.getFile(); - } catch (IOException e) { - f = null; - e.printStackTrace(); - } - - // String pathToLoadFile = System.getProperty("ix.ginas.load.file"); - //&& substanceRepository.count()==0 - if (f.getAbsolutePath() != null ) { - // File f = new File(pathToLoadFile); - if(f.exists()) { - transactionTemplate.executeWithoutResult(status-> { - UserProfile up = userProfileRepository.findByUser_UsernameIgnoreCase("admin").standardize(); - - Authentication auth = new UsernamePasswordAuthenticationToken(up.user.username, null, - up.getRoles().stream().map(r -> new SimpleGrantedAuthority("ROLE_" + r.name())).collect(Collectors.toList())); - SecurityContextHolder.getContext().setAuthentication(auth); - }); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { - String line; - Pattern sep = Pattern.compile("\t"); - ObjectMapper mapper = new ObjectMapper(); - int i = 0; - long start = System.currentTimeMillis(); - while ((line = reader.readLine()) != null) { - - String[] cols = sep.split(line); -// System.out.println(cols[2]); - JsonNode json = mapper.readTree(cols[2]); - try { - transactionTemplate.executeWithoutResult(status-> { - try { - Substance s = substanceEntityService.createEntity(json, true).getCreatedEntity(); - if(s != null) { - String uuid = s.uuid.toString(); - uuid = (uuid==null) ? "UUID_NULL": uuid; - System.out.println("Loaded: " + uuid); - } else { - System.out.println("Loaded: " + "SUBSTANCE_NULL"); - } - } catch (IOException e) { - e.printStackTrace(); - status.setRollbackOnly(); - } - }); - } catch (Throwable t) { - t.printStackTrace(); - } - - - - - - i++; - if (i % 100 == 0) { - System.out.println("loaded record " + i); - } - - } - System.out.println("done loading file"); - long finish = System.currentTimeMillis(); - long millisElapsed = finish - start; - long minutesElapsed = millisElapsed / (60 * 1000); - System.out.println("# Loaded: " + i); - System.out.println("Millis elapsed: " + millisElapsed); - System.out.println("Minutes elapsed: " + minutesElapsed); - - }catch(Throwable t){ - t.printStackTrace(); - } - }else{ - System.err.println("could not find GSRS file: " + f.getAbsolutePath()); - } - } - - return ResponseEntity.ok(resultString); + SubstanceEasyLegacyBulkLoadController( + // AdminService adminService + ){ + this.adminService = adminService; + // Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + // Authentication auth = adminService.getAnyAdmin(); } @hasAdminRole @@ -219,7 +78,7 @@ public ResponseEntity testLoad4(){ final TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); Resource dataFile = new ClassPathResource("rep18.gsrs"); - Authentication auth = adminService.getCurrentAdminAuth(); + File f = null; try { f = dataFile.getFile(); @@ -230,23 +89,57 @@ public ResponseEntity testLoad4(){ if (f.getAbsolutePath() != null ) { if(f.exists()) { // int parallelism = Runtime.getRuntime().availableProcessors(); - int parallelism = 4; + int parallelism = 2; List tasks = new ArrayList<>(); - ExecutorService executorService = Executors.newFixedThreadPool(parallelism); + // ExecutorService executorService = Executors.newFixedThreadPool(parallelism); + ExecutorService executorService = BlockingSubmitExecutor.newFixedThreadPool(parallelism, 5); + + // Authentication auth = SecurityContextHolder.getContext().getAuthentication(); try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { reader.lines().forEach(l -> { - // processItem(l,transactionTemplate); - + Authentication auth = adminService.getAnyAdmin(); Future task = executorService.submit(() -> { - Runnable r = ()->{ - _countrun.incrementAndGet(); - processItem(l,transactionTemplate); - }; - adminService.runAs(auth, r); + adminService.runAs(auth, + (Runnable) ()->{ + System.out.println("AUTH: " + auth.toString()); + _countrun.incrementAndGet(); + + + // processItem(l, auth, transactionTemplate); + + long threadId = Thread.currentThread().getId(); + System.out.println("Thread # " + threadId + " is doing this task"); + String[] cols = l.split(SEP); + try { + transactionTemplate.executeWithoutResult(status -> { + _countpi.incrementAndGet(); + try { + JsonNode json = MAPPER.readTree(cols[2]); + // Substance s = + substanceEntityService.createEntity(json, true); //.getCreatedEntity(); +// if (s != null) { +// String uuid = s.uuid.toString(); +// uuid = (uuid == null) ? "UUID_NULL" : uuid; +// System.out.println("Loaded: " + uuid); +// } else { +// System.out.println("Loaded: " + "SUBSTANCE_NULL"); +// } + } catch (IOException e) { + System.out.println(e.getMessage()); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + long threadId2 = Thread.currentThread().getId(); + System.out.println("THROWABLE -- " + "Thread # " + threadId + " ... " + t.getMessage()); + } + System.out.println("Auth Service"); + System.out.println(auth.toString()); + }); }); tasks.add(task); }); @@ -254,8 +147,6 @@ public ResponseEntity testLoad4(){ for (Future task : tasks) { try { task.get(); - - } catch (InterruptedException e) { e.printStackTrace(); } catch (ExecutionException e) { @@ -278,30 +169,6 @@ public ResponseEntity testLoad4(){ return ResponseEntity.ok(resultString); } - class MyRunnable implements Runnable { - - String line; - TransactionTemplate transactionTemplate; - AdminService adminService; - Authentication auth; - SubstanceEasyLegacyBulkLoadController parent; - - public MyRunnable( - String line, - TransactionTemplate transactionTemplate, - AdminService adminService, - Authentication auth, - SubstanceEasyLegacyBulkLoadController parent - ) {} - - @Override - public void run() { - Runnable r = ()->{ - parent.processItem(line,transactionTemplate); - }; - adminService.runAs(auth, r); - } - } private final String SEP = "\t"; private final ObjectMapper MAPPER = new ObjectMapper(); @@ -309,8 +176,8 @@ public void run() { private final AtomicInteger _countpi = new AtomicInteger(0); - private void processItem(String line, TransactionTemplate transactionTemplate) { - + private void processItem(String line, Authentication auth, TransactionTemplate transactionTemplate) { + System.out.println("AUTH:" + auth.getName()); String[] cols = line.split(SEP); try { transactionTemplate.executeWithoutResult(status -> { @@ -335,50 +202,6 @@ private void processItem(String line, TransactionTemplate transactionTemplate) } } -// class LoaderTask implements Future { -// String line; -// Authentication auth; -// ExecutorService executorService; -// -// LoaderTask(String line, Authentication auth, ExecutorService executorService) { -// this.line = line; -// this.auth = auth; -// Future task = executorService.submit(() -> { -// Runnable r = ()->{ -// processItem(l,transactionTemplate); -// }; -// adminService.runAs(auth, r); -// }); -// -// } -// -// -// @Override -// public boolean cancel(boolean mayInterruptIfRunning) { -// return false; -// } -// -// @Override -// public boolean isCancelled() { -// return false; -// } -// -// @Override -// public boolean isDone() { -// return false; -// } -// -// @Override -// public Object get() throws InterruptedException, ExecutionException { -// return null; -// } -// -// @Override -// public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { -// return null; -// } -// } - } diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadControllerOld.java.old b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadControllerOld.java.old new file mode 100644 index 00000000..a0294a68 --- /dev/null +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadControllerOld.java.old @@ -0,0 +1,400 @@ +package gsrs.module.substance.controllers; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import gsrs.controller.GsrsControllerConfiguration; +import gsrs.module.substance.SubstanceEntityService; +import gsrs.module.substance.services.ProcessingJobEntityService; +import gsrs.module.substance.services.SubstanceBulkLoadService; +import gsrs.repository.PayloadRepository; +import gsrs.repository.UserProfileRepository; +import gsrs.security.AdminService; +import gsrs.security.hasAdminRole; +import gsrs.service.PayloadService; +import ix.core.models.Payload; +import ix.core.models.UserProfile; +import ix.core.processing.PayloadProcessor; +import ix.core.stats.Statistics; +import ix.ginas.models.v1.Substance; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.Resource; +import org.springframework.http.ResponseEntity; +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.authority.SimpleGrantedAuthority; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.zip.GZIPInputStream; + + +@RestController +@Slf4j +public class SubstanceEasyLegacyBulkLoadControllerOld { + @Autowired + private AdminService adminService; + + @Autowired + private UserProfileRepository userProfileRepository; + + @Autowired + private SubstanceEntityService substanceEntityService; + + @Autowired + private PayloadService payloadService; + + @Autowired + private PayloadRepository payloadRepository; + + @Autowired + private SubstanceBulkLoadService substanceBulkLoadService; + + @Autowired + private GsrsControllerConfiguration controllerConfiguration; + + @Autowired + private ProcessingJobEntityService processingJobService; + + @Autowired + private PlatformTransactionManager platformTransactionManager; + + private Authentication auth; + + SubstanceEasyLegacyBulkLoadControllerOld( + AdminService adminService + ){ + this.adminService = adminService; + // Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + Authentication auth = adminService.getAnyAdmin(); + System.out.println("Auth Service"); + System.out.println(auth.toString()); + System.out.println("Inside controller ... SubstanceEasyLegacyBulkLoadController."); + } + @hasAdminRole + @GetMapping("/api/v1/testLoad") + public ResponseEntity testLoad(){ + String resultString = "Hello"; + Resource dataFile = new ClassPathResource( + "rep18.gsrs" + ); + + TransactionTemplate tx = new TransactionTemplate(platformTransactionManager); + tx.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + Payload payload = tx.execute(status-> { +// createUser("admin2", Role.values()); + //the old json has user TYLER and FDA-SRS too +// createUser("TYLER", Role.values()); +// createUser("FDA_SRS", Role.values()); + try (InputStream in = dataFile.getInputStream()) { + return payloadService.createPayload(dataFile.getFilename(), "ignore", + in, PayloadService.PayloadPersistType.PERM); + }catch(IOException e){ + throw new UncheckedIOException(e); + } + }); + PayloadProcessor pp = substanceBulkLoadService.submit(SubstanceBulkLoadService.SubstanceBulkLoadParameters.builder() + .payload(payload) + .build()); + + String statKey = pp.key; + boolean done =false; + Statistics statistics=null; + while(!done){ + statistics = substanceBulkLoadService.getStatisticsFor(statKey); + + if(statistics._isDone()){ + System.out.println(statistics); + break; + } + // Thread.sleep(1000); + } + return ResponseEntity.ok(resultString); + + } + @hasAdminRole + @GetMapping("/api/v1/testLoad3") + public ResponseEntity testLoad3(){ + String resultString = "hello 3"; + TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); + Resource dataFile = new ClassPathResource( + "rep18.gsrs" + ); + File f = null; + try { + f = dataFile.getFile(); + } catch (IOException e) { + f = null; + e.printStackTrace(); + } + + // String pathToLoadFile = System.getProperty("ix.ginas.load.file"); + //&& substanceRepository.count()==0 + if (f.getAbsolutePath() != null ) { + // File f = new File(pathToLoadFile); + if(f.exists()) { + transactionTemplate.executeWithoutResult(status-> { + UserProfile up = userProfileRepository.findByUser_UsernameIgnoreCase("admin").standardize(); + + Authentication auth = new UsernamePasswordAuthenticationToken(up.user.username, null, + up.getRoles().stream().map(r -> new SimpleGrantedAuthority("ROLE_" + r.name())).collect(Collectors.toList())); + SecurityContextHolder.getContext().setAuthentication(auth); + }); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { + String line; + Pattern sep = Pattern.compile("\t"); + ObjectMapper mapper = new ObjectMapper(); + int i = 0; + long start = System.currentTimeMillis(); + while ((line = reader.readLine()) != null) { + + String[] cols = sep.split(line); +// System.out.println(cols[2]); + JsonNode json = mapper.readTree(cols[2]); + try { + transactionTemplate.executeWithoutResult(status-> { + try { + Substance s = substanceEntityService.createEntity(json, true).getCreatedEntity(); + if(s != null) { + String uuid = s.uuid.toString(); + uuid = (uuid==null) ? "UUID_NULL": uuid; + System.out.println("Loaded: " + uuid); + } else { + System.out.println("Loaded: " + "SUBSTANCE_NULL"); + } + } catch (IOException e) { + e.printStackTrace(); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + t.printStackTrace(); + } + + + + + + i++; + if (i % 100 == 0) { + System.out.println("loaded record " + i); + } + + } + System.out.println("done loading file"); + long finish = System.currentTimeMillis(); + long millisElapsed = finish - start; + long minutesElapsed = millisElapsed / (60 * 1000); + System.out.println("# Loaded: " + i); + System.out.println("Millis elapsed: " + millisElapsed); + System.out.println("Minutes elapsed: " + minutesElapsed); + + }catch(Throwable t){ + t.printStackTrace(); + } + }else{ + System.err.println("could not find GSRS file: " + f.getAbsolutePath()); + } + } + + return ResponseEntity.ok(resultString); + } + + @hasAdminRole + @GetMapping("/api/v1/testLoad4") + public ResponseEntity testLoad4(){ + String resultString = "hello 4"; + final TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); + transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + Resource dataFile = new ClassPathResource("rep18.gsrs"); + + File f = null; + try { + f = dataFile.getFile(); + } catch (IOException e) { + f = null; + e.printStackTrace(); + } + if (f.getAbsolutePath() != null ) { + if(f.exists()) { +// int parallelism = Runtime.getRuntime().availableProcessors(); + int parallelism = 4; + + List tasks = new ArrayList<>(); + ExecutorService executorService = Executors.newFixedThreadPool(parallelism); +// Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { + reader.lines().forEach(l -> { + // processItem(l,transactionTemplate); + +// Future task = executorService.submit(() -> { +// Runnable r = ()->{ +// _countrun.incrementAndGet(); +// processItem(l,transactionTemplate); +// }; +// adminService.runAs(auth, r); +// +// }); +// tasks.add(task); + Future task = executorService.submit(() -> { + Runnable r = ()->{ + _countrun.incrementAndGet(); + processItem(l,auth, transactionTemplate); + }; + System.out.println("Auth Service"); + System.out.println(auth.toString()); + adminService.runAs(auth, r); + + }); + tasks.add(task); + + + }); + + for (Future task : tasks) { + try { + task.get(); + + + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + System.out.println("_countrun:" + _countrun); + System.out.println("_countpi:" + _countpi); + + } catch (IOException e) { + System.out.println("IO Exception during buffered read ..."); + } + + }else{ + System.err.println("could not find GSRS file: " + f.getAbsolutePath()); + } + + } + + return ResponseEntity.ok(resultString); + } + + class MyRunnable implements Runnable { + + String line; + TransactionTemplate transactionTemplate; + AdminService adminService; + Authentication auth; + SubstanceEasyLegacyBulkLoadControllerOld parent; + + public MyRunnable( + String line, + TransactionTemplate transactionTemplate, + AdminService adminService, + Authentication auth, + SubstanceEasyLegacyBulkLoadControllerOld parent + ) {} + + @Override + public void run() { + Runnable r = ()->{ + parent.processItem(line,auth, transactionTemplate); + }; + adminService.runAs(auth, r); + } + } + + private final String SEP = "\t"; + private final ObjectMapper MAPPER = new ObjectMapper(); + private final AtomicInteger _countrun = new AtomicInteger(0); + private final AtomicInteger _countpi = new AtomicInteger(0); + + + private void processItem(String line, Authentication auth, TransactionTemplate transactionTemplate) { + System.out.println("AUTH:" + auth.getName()); + String[] cols = line.split(SEP); + try { + transactionTemplate.executeWithoutResult(status -> { + _countpi.incrementAndGet(); + try { + JsonNode json = MAPPER.readTree(cols[2]); + Substance s = substanceEntityService.createEntity(json, true).getCreatedEntity(); + if (s != null) { + String uuid = s.uuid.toString(); + uuid = (uuid == null) ? "UUID_NULL" : uuid; + System.out.println("Loaded: " + uuid); + } else { + System.out.println("Loaded: " + "SUBSTANCE_NULL"); + } + } catch (IOException e) { + System.out.println(e.getMessage()); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + System.out.println("THROWABLE: " + t.getMessage()); + } + } + +// class LoaderTask implements Future { +// String line; +// Authentication auth; +// ExecutorService executorService; +// +// LoaderTask(String line, Authentication auth, ExecutorService executorService) { +// this.line = line; +// this.auth = auth; +// Future task = executorService.submit(() -> { +// Runnable r = ()->{ +// processItem(l,transactionTemplate); +// }; +// adminService.runAs(auth, r); +// }); +// +// } +// +// +// @Override +// public boolean cancel(boolean mayInterruptIfRunning) { +// return false; +// } +// +// @Override +// public boolean isCancelled() { +// return false; +// } +// +// @Override +// public boolean isDone() { +// return false; +// } +// +// @Override +// public Object get() throws InterruptedException, ExecutionException { +// return null; +// } +// +// @Override +// public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { +// return null; +// } +// } + +} + + From f576ceac5487c12d9569481e4e4aeee10a677e90 Mon Sep 17 00:00:00 2001 From: alx652 Date: Tue, 20 Aug 2024 18:24:48 -0400 Subject: [PATCH 3/5] debugging --- .../SubstanceEasyLegacyBulkLoadController.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java index f8ce57dc..ba7feeff 100644 --- a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java @@ -89,10 +89,17 @@ public ResponseEntity testLoad4(){ if (f.getAbsolutePath() != null ) { if(f.exists()) { // int parallelism = Runtime.getRuntime().availableProcessors(); - int parallelism = 2; + int parallelism = 6; +// when there is more than one thread getting "THROWABLE(s)" +// # throwables = threads - 1 +// may be related to PrincipalServiceImpl and transaction syncronization ? + List tasks = new ArrayList<>(); // ExecutorService executorService = Executors.newFixedThreadPool(parallelism); + + + ExecutorService executorService = BlockingSubmitExecutor.newFixedThreadPool(parallelism, 5); @@ -100,7 +107,8 @@ public ResponseEntity testLoad4(){ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { reader.lines().forEach(l -> { - Authentication auth = adminService.getAnyAdmin(); + final Authentication auth = adminService.getCurrentAdminAuth(); + Future task = executorService.submit(() -> { adminService.runAs(auth, (Runnable) ()->{ From 7fad501c8fcaf87db8898001cbd35601771216f9 Mon Sep 17 00:00:00 2001 From: alx652 Date: Wed, 28 Aug 2024 10:47:36 -0400 Subject: [PATCH 4/5] more testing --- ...SubstanceEasyLegacyBulkLoadController.java | 86 +++++++++++-------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java index ba7feeff..a1f74767 100644 --- a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java @@ -4,6 +4,7 @@ import gov.nih.ncats.common.executors.BlockingSubmitExecutor; import gsrs.security.AdminService; import ix.ginas.models.v1.Substance; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.transaction.support.TransactionTemplate; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -64,7 +65,7 @@ public class SubstanceEasyLegacyBulkLoadController { // private Authentication auth; SubstanceEasyLegacyBulkLoadController( - // AdminService adminService + // AdminService adminService ){ this.adminService = adminService; // Authentication auth = SecurityContextHolder.getContext().getAuthentication(); @@ -95,8 +96,9 @@ public ResponseEntity testLoad4(){ // may be related to PrincipalServiceImpl and transaction syncronization ? - List tasks = new ArrayList<>(); - // ExecutorService executorService = Executors.newFixedThreadPool(parallelism); +// List tasks = new ArrayList<>(); + List tasks = Collections.synchronizedList(new ArrayList()); + // ExecutorService executorService = Executors.newFixedThreadPool(parallelism); @@ -107,27 +109,28 @@ public ResponseEntity testLoad4(){ try (BufferedReader reader = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(f.getAbsolutePath()))))) { reader.lines().forEach(l -> { - final Authentication auth = adminService.getCurrentAdminAuth(); + final Authentication auth = adminService.getAnyAdmin(); Future task = executorService.submit(() -> { + adminService.runAs(auth, - (Runnable) ()->{ - System.out.println("AUTH: " + auth.toString()); - _countrun.incrementAndGet(); - - - // processItem(l, auth, transactionTemplate); - - long threadId = Thread.currentThread().getId(); - System.out.println("Thread # " + threadId + " is doing this task"); - String[] cols = l.split(SEP); - try { - transactionTemplate.executeWithoutResult(status -> { - _countpi.incrementAndGet(); - try { - JsonNode json = MAPPER.readTree(cols[2]); - // Substance s = - substanceEntityService.createEntity(json, true); //.getCreatedEntity(); + (Runnable) ()->{ + System.out.println("AUTH: " + auth.toString()); + _countrun.incrementAndGet(); + + + // processItem(l, auth, transactionTemplate); + + long threadId = Thread.currentThread().getId(); + System.out.println("Thread # " + threadId + " is doing this task"); + String[] cols = l.split(SEP); + try { + transactionTemplate.executeWithoutResult(status -> { + _countpi.incrementAndGet(); + try { + JsonNode json = MAPPER.readTree(cols[2]); + // Substance s = + substanceEntityService.createEntity(json, true); //.getCreatedEntity(); // if (s != null) { // String uuid = s.uuid.toString(); // uuid = (uuid == null) ? "UUID_NULL" : uuid; @@ -135,23 +138,31 @@ public ResponseEntity testLoad4(){ // } else { // System.out.println("Loaded: " + "SUBSTANCE_NULL"); // } - } catch (IOException e) { - System.out.println(e.getMessage()); - status.setRollbackOnly(); - } - }); - } catch (Throwable t) { - long threadId2 = Thread.currentThread().getId(); - System.out.println("THROWABLE -- " + "Thread # " + threadId + " ... " + t.getMessage()); - } - System.out.println("Auth Service"); - System.out.println(auth.toString()); - - }); + + } catch (IOException e) { + System.out.println(e.getMessage()); + status.setRollbackOnly(); + } + }); + } catch (Throwable t) { + long threadId2 = Thread.currentThread().getId(); + System.out.println("THROWABLE -- " + "Thread # " + threadId + " ... " + t.getMessage()); + } + System.out.println("Auth Service"); + System.out.println(auth.toString()); + + }); }); tasks.add(task); }); + synchronized (tasks) { + for (Future task: (List) tasks) { + task.get(); + } + } + +/* for (Future task : tasks) { try { task.get(); @@ -161,11 +172,16 @@ public ResponseEntity testLoad4(){ e.printStackTrace(); } } +*/ System.out.println("_countrun:" + _countrun); System.out.println("_countpi:" + _countpi); - } catch (IOException e) { + } catch (IOException e) { System.out.println("IO Exception during buffered read ..."); + } catch (ExecutionException e) { + e.printStackTrace(); + } catch (InterruptedException e) { + e.printStackTrace(); } }else{ From b751be908942ac71070659e196c321985d4b23b6 Mon Sep 17 00:00:00 2001 From: alx652 Date: Mon, 28 Oct 2024 10:05:51 -0400 Subject: [PATCH 5/5] note --- .../controllers/SubstanceEasyLegacyBulkLoadController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java index a1f74767..408cb9d7 100644 --- a/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java +++ b/gsrs-module-substances-core/src/main/java/gsrs/module/substance/controllers/SubstanceEasyLegacyBulkLoadController.java @@ -75,6 +75,7 @@ public class SubstanceEasyLegacyBulkLoadController { @hasAdminRole @GetMapping("/api/v1/testLoad4") public ResponseEntity testLoad4(){ + System.out.println("Starting testLoad4"); String resultString = "hello 4"; final TransactionTemplate transactionTemplate = new TransactionTemplate(platformTransactionManager); transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW);