diff --git a/gsrs-controlled-vocabulary/src/main/java/gsrs/cv/ControlledVocabularyEntityServiceImpl.java b/gsrs-controlled-vocabulary/src/main/java/gsrs/cv/ControlledVocabularyEntityServiceImpl.java index e5cebc2f..f7a27942 100644 --- a/gsrs-controlled-vocabulary/src/main/java/gsrs/cv/ControlledVocabularyEntityServiceImpl.java +++ b/gsrs-controlled-vocabulary/src/main/java/gsrs/cv/ControlledVocabularyEntityServiceImpl.java @@ -10,7 +10,9 @@ import gsrs.events.AbstractEntityUpdatedEvent; import gsrs.repository.ControlledVocabularyRepository; import gsrs.service.AbstractGsrsEntityService; +import ix.core.util.EntityUtils.Key; import ix.ginas.models.v1.ControlledVocabulary; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.context.annotation.ScopedProxyMode; @@ -22,6 +24,8 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.UUID; +import java.util.stream.Collectors; @Scope(proxyMode = ScopedProxyMode.INTERFACES) @Service public class ControlledVocabularyEntityServiceImpl extends AbstractGsrsEntityService implements ControlledVocabularyEntityService { @@ -154,6 +158,11 @@ protected Optional flexLookupIdOnly(String someKindOfId) { return Optional.empty(); } + @Override + public List getIDs() { + return repository.getAllIDs(); + } + // private SearchResult parseQueryIntoMatch(String query, SearchSession session) { // Pattern pattern = Pattern.compile("(\\S+):(\\S+)"); @@ -247,6 +256,11 @@ protected Optional flexLookupIdOnly(String someKindOfId) { // } - + @Override + public List getKeys() { + List IDs = repository.getAllIds(); + List keys = IDs.stream().map(id->Key.ofStringId(ControlledVocabulary.class, Long.toString(id))).collect(Collectors.toList()); + return keys; + } } diff --git a/gsrs-controlled-vocabulary/src/main/java/gsrs/repository/ControlledVocabularyRepository.java b/gsrs-controlled-vocabulary/src/main/java/gsrs/repository/ControlledVocabularyRepository.java index 5fe29c9c..f0077c42 100644 --- a/gsrs-controlled-vocabulary/src/main/java/gsrs/repository/ControlledVocabularyRepository.java +++ b/gsrs-controlled-vocabulary/src/main/java/gsrs/repository/ControlledVocabularyRepository.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Repository; import java.util.List; +import java.util.UUID; @Repository public interface ControlledVocabularyRepository extends GsrsVersionedRepository { @@ -17,6 +18,9 @@ public interface ControlledVocabularyRepository extends GsrsVersionedRepository< List findByDomain(String domain); @Query(value = "select * from ControlledVocabulary where id=:id", nativeQuery = true) List foo(String id); + + @Query("select cv.id from ControlledVocabulary cv") + List getAllIds(); /** * Summary of a ControlledVocabulary with only a few fields. @@ -26,4 +30,8 @@ interface ControlledVocabularySummary { String getDomain(); } + + @Query("select cv.id from ControlledVocabulary cv") + List getAllIDs(); + } diff --git a/gsrs-core-entities/src/main/java/gsrs/events/ReindexEntityEvent.java b/gsrs-core-entities/src/main/java/gsrs/events/ReindexEntityEvent.java index 76943e4b..e0f016f3 100644 --- a/gsrs-core-entities/src/main/java/gsrs/events/ReindexEntityEvent.java +++ b/gsrs-core-entities/src/main/java/gsrs/events/ReindexEntityEvent.java @@ -14,18 +14,23 @@ public class ReindexEntityEvent implements ReindexEvent { private UUID reindexId; private EntityUtils.Key entityKey; - private boolean requiresDelete=false; + private boolean requiresDelete = false; + private boolean excludeExternal = false; private Optional> optionalEntityWrapper = Optional.empty(); - - public ReindexEntityEvent(UUID reindexId, EntityUtils.Key entityKey, Optional> of, boolean b) { + public ReindexEntityEvent(UUID reindexId, EntityUtils.Key entityKey, Optional> of, boolean requiresDelete, boolean excludeExternal) { this.reindexId=reindexId; this.entityKey=entityKey; this.optionalEntityWrapper=of; - this.requiresDelete=b; + this.requiresDelete=requiresDelete; + this.excludeExternal=excludeExternal; + } + + public ReindexEntityEvent(UUID reindexId, EntityUtils.Key entityKey, Optional> of, boolean b) { + this(reindexId,entityKey,of,b,false); } public ReindexEntityEvent(UUID reindexId, EntityUtils.Key entityKey, Optional> of) { - this(reindexId,entityKey,of,false); + this(reindexId,entityKey,of,false,false); } diff --git a/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEntityListener.java b/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEntityListener.java index 4f3252f1..fbb58781 100644 --- a/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEntityListener.java +++ b/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEntityListener.java @@ -57,13 +57,17 @@ public void updateEntity(Object obj) { } public void reindexEntity(Object obj, boolean deleteFirst) { + reindexEntity(obj, deleteFirst, false); + } + + public void reindexEntity(Object obj, boolean deleteFirst, boolean excludeExternal) { autowireIfNeeded(); EntityUtils.EntityWrapper ew = EntityUtils.EntityWrapper.of(obj); if(ew.shouldIndex()) { IndexerEventFactory indexerFactoryFor = indexerEventFactoryFactory.getIndexerFactoryFor(obj); if(indexerFactoryFor !=null) { // log.error("ew before publishing event :" + ew.toString()); - applicationEventPublisher.publishEvent(indexerFactoryFor.newReindexEventFor(ew,deleteFirst)); + applicationEventPublisher.publishEvent(indexerFactoryFor.newReindexEventFor(ew,deleteFirst,excludeExternal)); } } diff --git a/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEventFactory.java b/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEventFactory.java index 1b8ecda2..f02885dd 100644 --- a/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEventFactory.java +++ b/gsrs-core-entities/src/main/java/gsrs/indexer/IndexerEventFactory.java @@ -53,6 +53,10 @@ default Object newReindexEventFor(EntityUtils.EntityWrapper ew, boolean deleteFi return new ReindexEntityEvent(UUID.randomUUID(), ew.getKey() ,Optional.of(ew),deleteFirst); } + default Object newReindexEventFor(EntityUtils.EntityWrapper ew, boolean deleteFirst, boolean excludeExternal){ + return new ReindexEntityEvent(UUID.randomUUID(), ew.getKey() ,Optional.of(ew),deleteFirst,excludeExternal); + } + /** * Create a new UpdateIndexEvent object for the given wrapped Entity. * The returned Object will be published to the {@link org.springframework.context.ApplicationEventPublisher}. diff --git a/gsrs-core-entities/src/main/java/gsrs/repository/EditRepository.java b/gsrs-core-entities/src/main/java/gsrs/repository/EditRepository.java index 299c2706..5191f65d 100644 --- a/gsrs-core-entities/src/main/java/gsrs/repository/EditRepository.java +++ b/gsrs-core-entities/src/main/java/gsrs/repository/EditRepository.java @@ -7,6 +7,7 @@ import java.util.UUID; import java.util.stream.Collectors; +import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; import ix.core.models.Edit; @@ -23,6 +24,9 @@ public interface EditRepository extends GsrsRepository { List findByRefidAndVersion(String refId, String version); + @Query("select e.id from Edit e") + List getAllIDs(); + default Optional findFirstByRefidOrderByCreatedDescAndKinds(String refId, Set kinds){ return findByRefidOrderByCreatedDesc(refId) .stream() diff --git a/gsrs-core-entities/src/main/java/gsrs/repository/NamespaceRepository.java b/gsrs-core-entities/src/main/java/gsrs/repository/NamespaceRepository.java index 077ac0b8..18437e62 100644 --- a/gsrs-core-entities/src/main/java/gsrs/repository/NamespaceRepository.java +++ b/gsrs-core-entities/src/main/java/gsrs/repository/NamespaceRepository.java @@ -2,9 +2,16 @@ import ix.core.models.Namespace; import ix.core.models.Principal; + +import java.util.List; + +import org.springframework.data.jpa.repository.Query; import org.springframework.stereotype.Repository; @Repository public interface NamespaceRepository extends GsrsRepository { Namespace findByName(String name); + + @Query("select ns.id from Namespace ns") + List getAllIDs(); } diff --git a/gsrs-data-exchange/src/main/java/gsrs/dataexchange/services/ImportMetadataReindexer.java b/gsrs-data-exchange/src/main/java/gsrs/dataexchange/services/ImportMetadataReindexer.java index 8a2a3049..98293e0d 100644 --- a/gsrs-data-exchange/src/main/java/gsrs/dataexchange/services/ImportMetadataReindexer.java +++ b/gsrs-data-exchange/src/main/java/gsrs/dataexchange/services/ImportMetadataReindexer.java @@ -237,7 +237,7 @@ public void incrementReindexEvent(IncrementReindexEvent event){ public static void indexOneItem(UUID reindexId, Consumer eventConsumer, EntityUtils.Key key, EntityUtils.EntityWrapper wrappedEntity) { log.trace("indexOneItem will process reindex of key {}", key); - ReindexEntityEvent event = new ReindexEntityEvent(reindexId, key, Optional.of(wrappedEntity), true); + ReindexEntityEvent event = new ReindexEntityEvent(reindexId, key, Optional.of(wrappedEntity), false); eventConsumer.accept(event); log.trace("submitted index event"); } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractGsrsEntityController.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractGsrsEntityController.java index b2bdbe99..4ee2d7e5 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractGsrsEntityController.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractGsrsEntityController.java @@ -14,6 +14,7 @@ import java.util.function.BiFunction; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.servlet.http.HttpServletRequest; @@ -400,6 +401,16 @@ public long getCount(){ return getEntityService().count(); } + @Override + @GetGsrsRestApiMapping("/@keys") + public List getKeys(){ + List IDs = getEntityService().getIDs(); +// System.out.println("GET IDS!"); +// IDs.forEach(id -> System.out.println("ID " + id.toString())); + List keys = IDs.stream().map(id->Key.ofStringId(getEntityService().getEntityClass(), id.toString())).collect(Collectors.toList()); + return keys; + } + @Override @GetGsrsRestApiMapping("") @Transactional(readOnly = true) diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractLegacyTextSearchGsrsEntityController.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractLegacyTextSearchGsrsEntityController.java index 7d6773c7..42988367 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractLegacyTextSearchGsrsEntityController.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/AbstractLegacyTextSearchGsrsEntityController.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -33,10 +34,16 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.client.RestTemplate; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Sets; +import com.google.common.collect.Sets.SetView; import gov.nih.ncats.common.util.TimeUtil; import gsrs.DefaultDataSourceConfig; @@ -66,6 +73,8 @@ import ix.core.search.bulk.UserSavedListService; import ix.core.search.bulk.UserSavedListService.Operation; import ix.core.search.text.FacetMeta; +import ix.core.search.text.RestrictedIVMSpecification; +import ix.core.search.text.RestrictedIVMSpecification.RestrictedType; import ix.core.search.text.TextIndexer; import ix.core.search.text.TextIndexerFactory; import ix.core.util.EntityUtils.EntityWrapper; @@ -95,7 +104,8 @@ public abstract class AbstractLegacyTextSearchGsrsEntityController{ return gsrsControllerConfiguration.handleNotFound(queryParameters); }); - } + } @hasAdminRole @PostGsrsRestApiMapping(value="/@reindexBulk", apiVersions = 1) - public ResponseEntity bulkReindex(@RequestBody String ids, @RequestParam Map queryParameters){ + public ResponseEntity bulkReindex(@RequestBody String ids, + @RequestParam(value= "excludeExternal", defaultValue = "false") boolean excludeExternal, + @RequestParam Map queryParameters){ List queries = Arrays.asList(ids.split("\n")); List list = queries.stream() @@ -194,6 +206,45 @@ public ResponseEntity bulkReindex(@RequestBody String ids, @RequestParam Mapq.length()>0) .distinct() .collect(Collectors.toList()); +// ReindexStatus stat = new ReindexStatus(); +// stat.statusID = UUID.randomUUID(); +// stat.done=false; +// stat.status="initializing"; +// stat.ids=list; +// stat.start = TimeUtil.getCurrentTimeMillis(); +// stat.total=list.size(); +// reindexing.put(stat.statusID.toString(), stat); +// +// executor.execute(()->{ +// int[] r = new int[] {0}; +// stat.ids.forEach(id->{ +// r[0]++; +// stat.setStatus("indexing record " + r[0] + " of " + stat.total); +// //TODO: Should change how this works probably to not use REST endpoint +// try { +// Optional entityID = getEntityService().getEntityIdOnlyBySomeIdentifier(id).map(ii->ii.toString()); +// Class eclass = getEntityService().getEntityClass(); +// Key k = Key.ofStringId(eclass, entityID.get()); +// Object o = EntityFetcher.of(k).call(); +// getlegacyGsrsSearchService().reindex(o, true); +// stat.indexed++; +// +// }catch(Exception e) { +// log.warn("trouble reindexing id: " + id, e); +// stat.failed++; +// } +// +// }); +// stat.setStatus("finished"); +// stat.done=true; +// stat.finshed = TimeUtil.getCurrentTimeMillis(); +// }); + + return new ResponseEntity<>(bulkReindexListOfIDs(list, excludeExternal), HttpStatus.OK); + } + + private ReindexStatus bulkReindexListOfIDs(List list, boolean excludeExternal) { + ReindexStatus stat = new ReindexStatus(); stat.statusID = UUID.randomUUID(); stat.done=false; @@ -214,7 +265,7 @@ public ResponseEntity bulkReindex(@RequestBody String ids, @RequestParam Map(stat, HttpStatus.OK); + return stat; } @GetGsrsRestApiMapping(value="/@reindexBulk", apiVersions = 1) @@ -256,13 +307,14 @@ public ResponseEntity bulkReindexClear(@RequestParam Map queryPa @PostGsrsRestApiMapping(value="({id})/@reindex", apiVersions = 1) public ResponseEntity reindex(@PathVariable("id") String id, + @RequestParam(value= "excludeExternal", defaultValue = "false") boolean excludeExternal, @RequestParam Map queryParameters){ //this needs to trigger a reindex Optional obj = getEntityService().getEntityBySomeIdentifier(id); if(obj.isPresent()){ Key k = EntityWrapper.of(obj.get()).getKey(); - getlegacyGsrsSearchService().reindex(obj.get(), true /*delete first*/); + getlegacyGsrsSearchService().reindex(obj.get(), true /*delete first*/, excludeExternal); //TODO: invent a better response? return getIndexData(k.getIdString(),queryParameters); } @@ -387,10 +439,11 @@ public ResponseEntity searchV1(@RequestParam("q") Optional query top.ifPresent( t-> builder.top(t)); skip.ifPresent( t-> builder.skip(t)); fdim.ifPresent( t-> builder.fdim(t)); - + + Map parameterMap = request.getParameterMap(); SearchRequest searchRequest = builder.withParameters(request.getParameterMap()) .build(); - + this.instrumentSearchRequest(searchRequest); SearchResult result = null; @@ -434,10 +487,89 @@ public ResponseEntity searchV1(@RequestParam("q") Optional query ResponseEntity ret= new ResponseEntity<>(createSearchResponse(results, result, request), HttpStatus.OK); return ret; } -// -// protected abstract Object createSearchResponse(List results, SearchResult result, HttpServletRequest request); + + public List searchEntityInIndex(){ + + final int defaultTop = 999999; + SearchRequest.Builder builder = new SearchRequest.Builder() + .query(null) + .kind(getEntityService().getEntityClass()); + builder.top(defaultTop); + + Map searchMap = new HashMap<>(); + searchMap.put("simpleSearchOnly", new String[]{"true"}); + searchMap.put("view",new String[]{"key"}); + + SearchRequest searchRequest = builder.withParameters(searchMap).build(); + this.instrumentSearchRequest(searchRequest); + SearchResult result = null; + + try { + result = getlegacyGsrsSearchService().search(searchRequest.getQuery(), searchRequest.getOptions() ); + } catch (Exception e) { + return new ArrayList(); + } + + SearchResult fresult=result; + + TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager); + transactionTemplate.setReadOnly(true); + List results = (List) transactionTemplate.execute(stauts -> { + List klist=new ArrayList<>(Math.min(fresult.getCount(),1000)); + fresult.copyKeysTo(klist, 0, defaultTop, true); + return klist; + + }); + return results; + } + + + @GetGsrsRestApiMapping(value = "/@databaseIndexDiff", apiVersions = 1) + public ResponseEntity getDifferenceBetweenDatabaseAndIndexes(HttpServletRequest request) throws JsonMappingException, JsonProcessingException{ + + List keysInDatabase = getKeys(); + System.out.println("List keys from database"); + keysInDatabase.forEach(System.out::println); + + List keysInIndex = searchEntityInIndex(); + System.out.println("List from index"); + keysInIndex.forEach(System.out::println); + + + ObjectMapper mapper = new ObjectMapper(); + Set extraInDatabase = Sets.difference(new HashSet(keysInDatabase), new HashSet(keysInIndex)); + Set extraInIndex = Sets.difference(new HashSet(keysInIndex), new HashSet(keysInDatabase)); + + ObjectNode baseNode = mapper.createObjectNode(); + baseNode.put("databaseOnly", mapper.writeValueAsString(extraInDatabase)); + baseNode.put("indexOnly", mapper.writeValueAsString(extraInIndex)); + + return new ResponseEntity<>(baseNode, HttpStatus.OK); + } + + @PostGsrsRestApiMapping(value = "/@databaseIndexSync", apiVersions = 1) + public ResponseEntity syncIndexesWithDatabase(HttpServletRequest request) throws JsonMappingException, JsonProcessingException{ + + List keysInDatabase = getKeys(); + System.out.println("List keys from database"); + keysInDatabase.forEach(System.out::println); + + List keysInIndex = searchEntityInIndex(); + //Todo for Lihui: Remove after testing + keysInIndex.remove(0); + //Todo for Lihui: Remove after testing + System.out.println("Lsit from index"); + keysInIndex.forEach(System.out::println); + + + + Set extraInDatabase = Sets.difference(new HashSet(keysInDatabase), new HashSet(keysInIndex)); + List list = extraInDatabase.stream().map(format->format.getIdString()).collect(Collectors.toList()); + return new ResponseEntity<>(bulkReindexListOfIDs(list, true), HttpStatus.OK); + } + @PostGsrsRestApiMapping(value="/@bulkQuery") public ResponseEntity saveQueryList(@RequestBody String query, @RequestParam("top") Optional top, @@ -1215,10 +1347,7 @@ private void reIndexWithKeys(UserListStatus status, List keyIds) { //TODO: should use indexing event instead TextIndexerFactory tif = StaticContextAccessor.getBean(TextIndexerFactory.class); - Set userFieldSet = new HashSet<>(); - userFieldSet.add("User List"); - - + keyIds.parallelStream().forEach(id->{ try { @@ -1229,7 +1358,7 @@ private void reIndexWithKeys(UserListStatus status, List keyIds) { Key k = Key.ofStringId(eclass, entityID.get()); Object o = EntityFetcher.of(k).call(); - tif.getDefaultInstance().updateFields(EntityWrapper.of(o), userFieldSet); + tif.getDefaultInstance().updateFields(EntityWrapper.of(o), RestrictedIVMSpecification.getRestrictedIVMSpecs(RestrictedType.INCLUDE_USER_LIST)); //getlegacyGsrsSearchService().reindex(o, true); }else { @@ -1260,5 +1389,5 @@ private String generateResultIDJson(String id) { ObjectNode node = mapper.createObjectNode(); node.put("id", id); return node.toPrettyString(); - } + } } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/EditEntityService.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/EditEntityService.java index c3591f0c..20ad3b6d 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/EditEntityService.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/EditEntityService.java @@ -7,6 +7,8 @@ import gsrs.repository.EditRepository; import gsrs.service.AbstractGsrsEntityService; import ix.core.models.Edit; + +import org.aspectj.apache.bcel.Repository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; @@ -123,4 +125,9 @@ public Class getEntityClass() { public Page page(Pageable pageable) { return editRepository.findAll(pageable); } + + @Override + public List getIDs() { + return editRepository.getAllIDs(); + } } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/GsrsEntityController.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/GsrsEntityController.java index 55a27824..d3e667aa 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/GsrsEntityController.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/GsrsEntityController.java @@ -1,6 +1,8 @@ package gsrs.controller; import com.fasterxml.jackson.databind.JsonNode; + +import ix.core.util.EntityUtils.Key; import ix.core.validator.ValidationResponse; import lombok.Data; import org.springframework.hateoas.Link; @@ -37,6 +39,9 @@ ResponseEntity updateEntity(@RequestBody JsonNode updatedEntityJson, @GetGsrsRestApiMapping("/@count") long getCount(); + + @GetGsrsRestApiMapping("/@keys") + List getKeys(); @GetGsrsRestApiMapping("") ResponseEntity page(@RequestParam(value = "top", defaultValue = "16") long top, @@ -66,4 +71,5 @@ class EntityExists{ private String url; } + } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/NamespaceEntityService.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/NamespaceEntityService.java index 852e5def..89391bb8 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/NamespaceEntityService.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/controller/NamespaceEntityService.java @@ -105,4 +105,9 @@ public Class getEntityClass() { public Page page(Pageable pageable) { return namespaceRepository.findAll(pageable); } + + @Override + public List getIDs() { + return namespaceRepository.getAllIDs(); + } } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/imports/ImportUtilities.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/imports/ImportUtilities.java index c7240f55..f4b53285 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/imports/ImportUtilities.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/imports/ImportUtilities.java @@ -270,7 +270,7 @@ public JsonNode handleActionsAsync(StagingAreaService stagingAreaService, log.trace("about to call runnable to call processOneRecord"); adminService.runAs(auth, runnable); log.trace("got back singleReturn {}", singleReturn[0].toPrettyString()); - } catch (Exception e) { + } catch (Throwable e) { log.error("error during import processing: ", e); singleReturn[0] = JsonNodeFactory.instance.objectNode(); singleReturn[0].put("id", stagingAreaId); @@ -296,7 +296,12 @@ public JsonNode handleActionsAsync(StagingAreaService stagingAreaService, String entityId = n.get("PersistedEntityId").asText(); Unchecked.ThrowingRunnable runnable2 = ()-> stagingAreaService.synchronizeRecord(entityId, entityClass.getName(), contextName); log.trace("About to call runnable2 to call synchronizeRecord"); - adminService.runAs(auth, runnable2); + try { + adminService.runAs(auth, runnable2); + } catch (Throwable e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } else { log.warn("entity id not found!!"); } diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/service/GsrsEntityService.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/service/GsrsEntityService.java index 3be0170d..d3bbf8d0 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/service/GsrsEntityService.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/service/GsrsEntityService.java @@ -6,6 +6,7 @@ import gsrs.EntityPersistAdapter; import gsrs.controller.OffsetBasedPageRequest; import gsrs.security.*; +import ix.core.util.EntityUtils.Key; import ix.core.validator.ValidationResponse; import ix.core.validator.ValidatorCategory; import lombok.Builder; @@ -15,7 +16,10 @@ import org.springframework.data.domain.Pageable; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; +import java.util.UUID; /** * Contains all the business logic for converting JSON into an Entity, reading and writing @@ -46,7 +50,15 @@ public interface GsrsEntityService { * @return a number ≥0. */ long count(); + + default List getKeys(){ + return new ArrayList(); + } + default List getIDs(){ + return new ArrayList(); + } + /** * Remove the given entity from the repository. * @param id the id of the entity to delete. diff --git a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/stagingarea/service/DefaultStagingAreaService.java b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/stagingarea/service/DefaultStagingAreaService.java index 21bbbc12..0d256701 100644 --- a/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/stagingarea/service/DefaultStagingAreaService.java +++ b/gsrs-spring-boot-autoconfigure/src/main/java/gsrs/stagingarea/service/DefaultStagingAreaService.java @@ -257,7 +257,7 @@ private void handleIndexing(ImportMetadata importMetadata){ log.trace("Here is where we index facets for the ImportMetadata object"); EntityUtils.EntityWrapper entityWrapper = EntityUtils.EntityWrapper.of(importMetadata); UUID reindexUuid = UUID.randomUUID(); - ReindexEntityEvent event = new ReindexEntityEvent(reindexUuid, entityWrapper.getKey(), Optional.of(entityWrapper), true); + ReindexEntityEvent event = new ReindexEntityEvent(reindexUuid, entityWrapper.getKey(), Optional.of(entityWrapper)); applicationEventPublisher.publishEvent(event); log.trace("published event for metadata"); } diff --git a/gsrs-spring-legacy-indexer/src/main/java/gsrs/legacy/LegacyGsrsSearchService.java b/gsrs-spring-legacy-indexer/src/main/java/gsrs/legacy/LegacyGsrsSearchService.java index d005c896..ea3a4933 100644 --- a/gsrs-spring-legacy-indexer/src/main/java/gsrs/legacy/LegacyGsrsSearchService.java +++ b/gsrs-spring-legacy-indexer/src/main/java/gsrs/legacy/LegacyGsrsSearchService.java @@ -202,6 +202,10 @@ public void reindexAndWait(boolean wipeIndexFirst){ } public void reindex(Object entity, boolean deleteFirst){ + reindex(entity, deleteFirst, false); + } + + public void reindex(Object entity, boolean deleteFirst, boolean excludeExternal){ //this ensures that the reindexing is done recursively //TODO: technically this will not handle the cases where // a child element which is indexed at root had been deleted via the database @@ -225,7 +229,7 @@ public void reindex(Object entity, boolean deleteFirst){ if (isEntity && isRootIndexed) { try { - indexerEntityListener.reindexEntity(wrapped.getRawValue(), deleteFirst); + indexerEntityListener.reindexEntity(wrapped.getRawValue(), deleteFirst, excludeExternal); } catch (Throwable t) { log.warn("indexing error handling:" + wrapped, t); } diff --git a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/CombinedIndexValueMaker.java b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/CombinedIndexValueMaker.java index cb8a3ffa..d233bb4e 100644 --- a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/CombinedIndexValueMaker.java +++ b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/CombinedIndexValueMaker.java @@ -1,5 +1,7 @@ package ix.core.search.text; +import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Consumer; @@ -7,6 +9,8 @@ import org.apache.commons.collections4.SetUtils; +import com.google.common.collect.Sets; + public class CombinedIndexValueMaker implements IndexValueMaker { private final List> list; private final Class clazz; @@ -27,11 +31,37 @@ public Set getFieldNames(){ @Override - public IndexValueMaker restrictedForm(Set fields){ - List> filteredList=list.stream() - .filter(ivm->ivm.getFieldNames().stream().anyMatch(fn->fields.contains(fn))) - .map(ivm->(IndexValueMaker)ivm.restrictedForm(fields)) - .collect(Collectors.toList()); + public Set getTags() { + return list.stream().flatMap(ivm->ivm.getTags().stream()).collect(Collectors.toSet()); + } + + @Override + public IndexValueMaker restrictedForm(Set tags, boolean include){ + + if(tags.size() ==0) return this; + List> filteredList; + + if(include) { + System.out.println("in include combined"); + filteredList=list.stream() + .filter(ivm->ivm.getTags().stream().anyMatch(tag->tags.contains(tag))) + .map(ivm->(IndexValueMaker)ivm.restrictedForm(tags,include)) + .collect(Collectors.toList()); + }else { + System.out.println("in exclude combined"); + filteredList=list.stream() + .filter(ivm->!ivm.getTags().stream().anyMatch(tn->tags.contains(tn))) + .map(ivm->(IndexValueMaker)ivm.restrictedForm(tags,include)) + .collect(Collectors.toList()); + + } + + //Todo for Lihui: Remove after testing + Set> filteredOut = Sets.difference(new HashSet(list), new HashSet(filteredList)); + if(filteredOut.size()>0) { + System.out.println("IVMs are filtered out in combined index value maker: "); + filteredOut.forEach(ivm->System.out.println(ivm.getClass().getSimpleName())); + } return new CombinedIndexValueMaker(clazz,filteredList); } diff --git a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/IndexValueMaker.java b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/IndexValueMaker.java index 6b008978..76d03714 100644 --- a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/IndexValueMaker.java +++ b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/IndexValueMaker.java @@ -1,6 +1,7 @@ package ix.core.search.text; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.function.Consumer; @@ -25,6 +26,9 @@ public interface IndexValueMaker { * * @since 3.0 */ + // Todo: + // Each ivm should specify the indexable fields and input fields. + // Have a flag to show if the IVM has dynamic input or indexable fields. Class getIndexedEntityClass(); /** * Creates IndexableValues out of the given Entity T, and returns them @@ -40,12 +44,57 @@ public interface IndexValueMaker { default Set getFieldNames(){ return new HashSet<>(); + } + + default boolean areInputFieldsDynamic() { + return false; } - default IndexValueMaker restrictedForm(Set fields){ - return this; + default boolean areIndexableFieldsDynamic() { + return false; } + default IndexValueMaker restrictedForm(Set tags, boolean include){ + + // include this ivm + if(include) { + if(tags.size()>0 && this.getTags().size()>0) { + System.out.println("in include"); + // include tags match the current ivm tags + if(this.getTags().stream().anyMatch(tag->tags.contains(tag))) { + return this; + } + } + }else{ // not exclude this ivm + + System.out.println("in exclude"); + if(tags.size()==0 || this.getTags().size() ==0 || + !this.getTags().stream().anyMatch(tag->tags.contains(tag))) { + return this; + } + } + + return new IndexValueMaker() { + + @Override + public Class getIndexedEntityClass() { + // TODO Auto-generated method stub + return this.getIndexedEntityClass(); + } + + @Override + public void createIndexableValues(Object t, Consumer consumer) { + // TODO Auto-generated method stub + } + + }; + } + + // external, chemical ... + default Set getTags() { + return Collections.emptySet(); + } + /** * Combine 2 IndexValueMakers together, so that * each is called sequentially. This accumulation diff --git a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/RestrictedIVMSpecification.java b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/RestrictedIVMSpecification.java new file mode 100644 index 00000000..c36be251 --- /dev/null +++ b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/RestrictedIVMSpecification.java @@ -0,0 +1,36 @@ +package ix.core.search.text; + +import java.util.Collections; +import java.util.Set; + +import ix.utils.Util; +import lombok.Data; + +@Data +public class RestrictedIVMSpecification { + + boolean include; + Set tags; + + public enum RestrictedType{ + EXCLUDE_EXTERNAL, + INCLUDE_USER_LIST + }; + + public RestrictedIVMSpecification(boolean include, Set tags) { + this.include = include; + this.tags = tags; + } + + public static RestrictedIVMSpecification getRestrictedIVMSpecs(RestrictedType type) { + switch(type) { + case EXCLUDE_EXTERNAL: + return new RestrictedIVMSpecification(false, Util.toSet("external")); + case INCLUDE_USER_LIST: + return new RestrictedIVMSpecification(true, Util.toSet("user_list")); + default: + return new RestrictedIVMSpecification(false, Collections.emptySet()); + } + } + +} diff --git a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexer.java b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexer.java index 7e733831..367a3b85 100644 --- a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexer.java +++ b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexer.java @@ -169,6 +169,7 @@ import ix.core.search.SuggestResult; import ix.core.search.bulk.UserSavedListService; import ix.core.search.bulk.UserSavedListService.UserListIndexedValue; +import ix.core.search.text.RestrictedIVMSpecification.RestrictedType; import ix.core.util.EntityUtils; import ix.core.util.EntityUtils.EntityInfo; import ix.core.util.EntityUtils.EntityWrapper; @@ -2937,7 +2938,7 @@ public JsonNode getDocJson(Key k) throws Exception { } public void update(EntityWrapper ew) throws IOException{ - log.trace("starting in update, key: {}", ew.getKey()); + log.error("starting in update, key: {}", ew.getKey()); Lock l = stripedLock.get(ew.getKey()); l.lock(); try { @@ -2951,29 +2952,55 @@ public void update(EntityWrapper ew) throws IOException{ } - public void updateFields(EntityWrapper ew, Set fields) throws IOException{ - log.trace("starting in update, key: {}", ew.getKey()); + public void update(EntityWrapper ew, RestrictedType type) throws IOException{ + log.error("starting in update with restricted IVMs, key: {}", ew.getKey()); Lock l = stripedLock.get(ew.getKey()); l.lock(); - + try { + //Removal is now done right before the final add to reduce time where the document is not + //found in the index + //remove(ew); + add(ew, true, RestrictedIVMSpecification.getRestrictedIVMSpecs(type)); + }finally{ + l.unlock(); + } + + } + + //This is currently only used for including specified IVMs + public void updateFields(EntityWrapper ew, RestrictedIVMSpecification ivmSpecs) throws IOException{ + log.error("starting in updateFields, key: {}", ew.getKey()); + Lock l = stripedLock.get(ew.getKey()); + l.lock(); + try{ + + if(!ivmSpecs.isInclude()) { + throw new Exception("This only handle include type of restricted forms of IVMs."); + } + IndexValueMaker valueMaker= indexValueMakerFactory.createIndexValueMakerFor(ew) + .restrictedForm(ivmSpecs.getTags(), true); //retrieve the indexed record as it was before so it can be directly //modified - IndexRecord ix =getIndexRecord(ew.getKey()); + IndexRecord ix =getIndexRecord(ew.getKey()); + Set fields = valueMaker.getFieldNames(); //remove the fields and facets which have the supplied name + + ix.facets=ix.facets.stream().filter(iff->!fields.contains(iff.getFacetName())).collect(Collectors.toList()); ix.fields=ix.fields.stream().filter(iff->!fields.contains(iff.getFieldName())).collect(Collectors.toList()); ix.suggest=ix.suggest.stream().filter(iff->!fields.contains(iff.getSuggestName())).collect(Collectors.toList()); + //No longer remove the documents here. Now remove the document right before saving the new one to reduce //the time where the index is incomplete // remove(ew); - add(ew,fields,ix, true); + add(ew, ivmSpecs, ix, true); } catch (Exception e) { log.warn("problem doing partial index update", e); //fallback to normal indexing @@ -2984,12 +3011,12 @@ public void updateFields(EntityWrapper ew, Set fields) throws IOExceptio } - - /** + /** * recursively index any object annotated with Entity, only use selected IVMs and store the delta * */ - private void add(EntityWrapper ew, Set filters,IndexRecord ix, boolean removeOld) throws IOException { + private void add(EntityWrapper ew, RestrictedIVMSpecification ivmSpecs, IndexRecord ix, boolean removeOld) throws IOException { + log.error("in add with Restricted IVMs, key: {}", ew.getKey()); ew.toInternalJson(); Document doc = new Document(); Document docExact = new Document(); @@ -3005,13 +3032,13 @@ private void add(EntityWrapper ew, Set filters,IndexRecord ix, boolean r }); } - Key kk = ew.getKey().toRootKey(); - - + Key kk = ew.getKey().toRootKey(); //make a version that only does the fields in question IndexValueMaker valueMaker= indexValueMakerFactory.createIndexValueMakerFor(ew) - .restrictedForm(filters); + .restrictedForm(ivmSpecs.getTags(), ivmSpecs.isInclude()); + Set filters = valueMaker.getFieldNames(); + valueMaker.createIndexableValues(ew.getValue(), iv->{ this.instrumentIndexableValue(ix, iv, ie->{ return filters.contains(ie.getIndexFieldName()); @@ -3094,8 +3121,183 @@ private void add(EntityWrapper ew, Set filters,IndexRecord ix, boolean r } } + - public void add(EntityWrapper ew) throws IOException { + /** + * recursively index any object annotated with Entity, only use selected IVMs and store the delta + * + */ + private void add(EntityWrapper ew, boolean force, boolean removeFirst, RestrictedIVMSpecification ivmSpecs) throws IOException { + log.error("in add method, with general restricted IVMs key: {}", ew.getKey()); + if(ivmSpecs.getTags().size()==0) { + add(ew, force, removeFirst); + return; + } + + if(!textIndexerConfig.isEnabled()){ + return; + } + Objects.requireNonNull(ew); + + if( !force){ + return; + } + + Lock l = stripedLock.get(ew.getKey()); + l.lock(); + try{ + ew.toInternalJson(); + Document doc = new Document(); + Document docExact = new Document(); + if(textIndexerConfig.isShouldLog()){ + LogUtil.debug(()->{ + String beanId; + if(ew.hasIdField()){ + beanId = ew.getKey().toString(); + }else{ + beanId = ew.toString(); + } + return "[LOG_INDEX] =======================\nINDEXING BEAN "+ beanId; + }); + + } + IndexRecord ix = new IndexRecord(); + Key kk = ew.getKey().toRootKey(); + ix.kind=kk.getKind(); + ix.id=kk.getIdString(); + ix.idField=kk.getEntityInfo().getInternalIdField(); + ix.deepAnalyzed=textIndexerConfig.isFieldsuggest() && deepKindFunction.apply(ew) && ew.hasKey(); + //flag the kind of document + IndexValueMaker valueMaker = indexValueMakerFactory.createIndexValueMakerFor(ew).restrictedForm(ivmSpecs.getTags(), ivmSpecs.isInclude()); +// log.error("ew.getValue(): " + ew.getValue() + " ew.getValue().getClass(): " + ew.getValue().getClass()); + + Set filterFields = valueMaker.getFieldNames(); + if(ivmSpecs.isInclude()) { + ix.facets=ix.facets.stream().filter(iff->!filterFields.contains(iff.getFacetName())).collect(Collectors.toList()); + ix.fields=ix.fields.stream().filter(iff->!filterFields.contains(iff.getFieldName())).collect(Collectors.toList()); + ix.suggest=ix.suggest.stream().filter(iff->!filterFields.contains(iff.getSuggestName())).collect(Collectors.toList()); + + valueMaker.createIndexableValues(ew.getValue(), iv->{ +// log.error("KK: " + kk + " iv name: " + iv.name() + " iv value: " + iv.value()); + this.instrumentIndexableValue(ix, iv, ie->{ + return filterFields.contains(ie.getIndexFieldName()); + }); + }); + }else { + ix.facets=ix.facets.stream().filter(iff->filterFields.contains(iff.getFacetName())).collect(Collectors.toList()); + ix.fields=ix.fields.stream().filter(iff->filterFields.contains(iff.getFieldName())).collect(Collectors.toList()); + ix.suggest=ix.suggest.stream().filter(iff->filterFields.contains(iff.getSuggestName())).collect(Collectors.toList()); + + valueMaker.createIndexableValues(ew.getValue(), iv->{ +// log.error("KK: " + kk + " iv name: " + iv.name() + " iv value: " + iv.value()); + this.instrumentIndexableValue(ix, iv, ie->{ + return !filterFields.contains(ie.getIndexFieldName()); + }); + }); + + ix.fields.add(IndexedField.builder() + .fieldName(FIELD_KIND) + .fieldValue(kk.getKind()) + .type(IndexedFieldType.STRING) + .stored(true) + .build()); + ix.fields.add(IndexedField.builder() + .fieldName(ANALYZER_MARKER_FIELD) + .fieldValue("false") + .type(IndexedFieldType.STRING) + .stored(true) + .build()); + } + + + + // Now that the record is built, + // we need to process it + IndexRecordProcessor irp = new IndexRecordProcessor(doc, ix, kk.getEntityInfo(), this); + irp.process(); + +// fieldCollector.accept(new StringField(FIELD_KIND, ew.getKind(), YES)); +// fieldCollector.accept(new StringField(ANALYZER_MARKER_FIELD, "false", YES)); + + irp.numericFieldList.forEach((n,v)->{ + if(v.size()==1) { + doc.add(v.get(0)); + }else { + v.forEach(iff->{ + double d =iff.numericValue().doubleValue(); + doc.add(new DoubleField(n, d, NO)); + }); + } + }); + + if(removeFirst) { + remove(ew); + } + + // now index + addDoc(doc); + + + Tuple luceneKey = kk.asLuceneIdTuple(); + + //ID + docExact.add(new StringField(FULL_DOC_PREFIX + luceneKey.k() , luceneKey.v(),YES)); + + docExact.add(new StringField(FIELD_KIND, FULL_DOC_PREFIX + kk.getKind(),YES)); + docExact.add(new SortedDocValuesField(FIELD_KIND,new BytesRef(FULL_DOC_PREFIX + kk.getKind()))); + docExact.add(new StoredField(FIELD_KIND, new BytesRef(FULL_DOC_PREFIX + kk.getKind()))); + + docExact.add(new StoredField("FULL_INDEX", new BytesRef(EntityWrapper.of(ix).toInternalJson()))); + docExact.add(new StringField(ANALYZER_MARKER_FIELD, "false",YES)); + + indexerService.addDocument(docExact); + + if(ix.isDeepAnalyzed()){ + + if(!kk.getIdString().equals("")){ //probably not needed + StringField toAnalyze=new StringField(FIELD_KIND, ANALYZER_VAL_PREFIX + kk.getKind(),YES); + SortedDocValuesField toAnalyze2= new SortedDocValuesField(FIELD_KIND,new BytesRef(ANALYZER_VAL_PREFIX + kk.getKind())); + StoredField toAnalyze3= new StoredField(FIELD_KIND, new BytesRef(ANALYZER_VAL_PREFIX + kk.getKind())); + StringField analyzeMarker=new StringField(ANALYZER_MARKER_FIELD, "true",YES); + + + StringField docParent=new StringField(ANALYZER_VAL_PREFIX+luceneKey.k(),luceneKey.v(),YES); + FacetField docParentFacet =new FacetField(ANALYZER_VAL_PREFIX+luceneKey.k(),luceneKey.v()); + //This is a test of a terrible idea, which just. might. work. + irp.fullText.forEach((name,group)->{ + try{ + Document fielddoc = new Document(); + fielddoc.add(toAnalyze); + fielddoc.add(toAnalyze2); + fielddoc.add(toAnalyze3); + fielddoc.add(analyzeMarker); + fielddoc.add(docParent); + fielddoc.add(docParentFacet); + fielddoc.add(new FacetField(ANALYZER_FIELD,name)); + for(String f:group){ + fielddoc.add(new TextField(FULL_TEXT_FIELD, f, NO)); + } + addDoc(fielddoc); + }catch(Exception e){ + log.error("Analyzing index failed", e); + } + }); + } + } + + +// if (DEBUG(2)) { +// log.debug("<<< " + ew.getValue()); +// } + }catch(Exception e){ + log.error("Error indexing record [" + ew.toString() + "] This may cause consistency problems", e); + }finally{ + l.unlock(); + } + } + + + public void add(EntityWrapper ew) throws IOException { add(ew, false); } @@ -3108,10 +3310,23 @@ public void add(EntityWrapper ew, boolean removeFirst) throws IOException { (textIndexerConfig.isRootIndexOnly() && !ew.isRootIndex()) || (isReindexing.get() && !alreadySeenDuringReindexingMode.add(ew.getKey().toString())); - add(ew,!shouldNotAdd, removeFirst); + add(ew, !shouldNotAdd, removeFirst); } + public void add(EntityWrapper ew, boolean removeFirst, RestrictedIVMSpecification ivmSpecs) throws IOException { + //Don't index if any of the following: + // 1. The entity doesn't have an Indexable annotation OR + // 2. The config is set to only index things with Indexable Root annotation and the entity doesn't have that annotation + // 3. Reindexing is happening and the entity has already been indexed + boolean shouldNotAdd= !ew.shouldIndex() || + (textIndexerConfig.isRootIndexOnly() && !ew.isRootIndex()) || + (isReindexing.get() && !alreadySeenDuringReindexingMode.add(ew.getKey().toString())); + + add(ew, !shouldNotAdd, removeFirst, ivmSpecs); + + } + private static boolean shouldIndexAsIdentifier(EntityInfo ei, String field) { // Identifiers are fields considered worth matching exactly, as opposed to a general text field. // Allows for searches to have an easy way to search identifier-level things (e.g. names, codes, uuids, inchis) @@ -3460,6 +3675,7 @@ private void processIndexedElement(IndexedElement fe) { * recursively index any object annotated with Entity */ private void add(EntityWrapper ew, boolean force, boolean removeFirst) throws IOException { + log.error("in adding entity key: {}", ew.getKey()); if(!textIndexerConfig.isEnabled()){ return; } @@ -3598,6 +3814,8 @@ private void add(EntityWrapper ew, boolean force, boolean removeFirst) throws IO } } + + //One more thing: // 1. need list of fields indexed. // 2. that's easy! At index time, just also index each field diff --git a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexerEntityListener.java b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexerEntityListener.java index 08bca0f0..1628e0a4 100644 --- a/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexerEntityListener.java +++ b/gsrs-spring-legacy-indexer/src/main/java/ix/core/search/text/TextIndexerEntityListener.java @@ -20,6 +20,7 @@ import gsrs.indexer.IndexUpdateEntityEvent; import gsrs.springUtils.AutowireHelper; import ix.core.EntityFetcher; +import ix.core.search.text.RestrictedIVMSpecification.RestrictedType; import ix.core.util.EntityUtils; import ix.core.util.EntityUtils.EntityWrapper; import ix.core.util.EntityUtils.Key; @@ -90,7 +91,13 @@ public void reindexEntity(ReindexEntityEvent event) throws IOException { if(opt.isPresent()){ if(event.isRequiresDelete()) { log.trace("updating"); - textIndexerFactory.getDefaultInstance().update(opt.get()); + if(event.isExcludeExternal()) { + log.info("updating excluding external"); + textIndexerFactory.getDefaultInstance().update(opt.get(), RestrictedType.EXCLUDE_EXTERNAL); + }else { + log.info("updating including external"); + textIndexerFactory.getDefaultInstance().update(opt.get()); + } }else { log.trace("adding"); textIndexerFactory.getDefaultInstance().add(opt.get()); @@ -140,9 +147,10 @@ public void updateEntity(IndexUpdateEntityEvent event) { } TextIndexer indexer = textIndexerFactory.getDefaultInstance(); if(indexer !=null) { + log.warn("In update Entity IndexUpdateEntityEvent"); try { EntityUtils.EntityWrapper ew = event.getOptionalFetchedEntity().orElse(null); - indexer.update(ew); + indexer.update(ew, RestrictedType.EXCLUDE_EXTERNAL); // exclude external }catch(Throwable t){ log.warn("trouble updating index for:" + event.getSource().toString(), t); } diff --git a/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/TextIndexerIndexedRecordTest.java b/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/TextIndexerIndexedRecordTest.java index 6c538a8e..b1fd15bb 100644 --- a/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/TextIndexerIndexedRecordTest.java +++ b/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/TextIndexerIndexedRecordTest.java @@ -109,7 +109,7 @@ public void testIndexRecordGetsSavedAndCanBeRetrieved() throws NoSuchElementExce TestEntity addTest = TestEntity.builder().id(1l).field("demo").build(); EntityWrapper wrapped = EntityWrapper.of(addTest); - ti.add(wrapped); + ti.add(wrapped, true); IndexRecord ir = ti.getIndexRecord(wrapped.getKey()); Assertions.assertNotNull(ir); Assertions.assertEquals(addTest.getId().toString(),ir.getId()); @@ -123,7 +123,7 @@ public void testIndexRecordGetsSavedAndCanBeRetrievedAndRemoved() throws NoSuchE TextIndexer ti=getNewTextIndexer(); TestEntity addTest = TestEntity.builder().id(1l).field("demo").build(); EntityWrapper wrapped = EntityWrapper.of(addTest); - ti.add(wrapped); + ti.add(wrapped, true); IndexRecord ir = ti.getIndexRecord(wrapped.getKey()); Assertions.assertNotNull(ir); Assertions.assertEquals(addTest.getId().toString(),ir.getId()); @@ -140,7 +140,7 @@ public void testIndexRecordGetsSavedAndCanBeSearched() throws NoSuchElementExcep TextIndexer ti=getNewTextIndexer(); TestEntity addTest = TestEntity.builder().id(1l).field("demo").build(); EntityWrapper wrapped = EntityWrapper.of(addTest); - ti.add(wrapped); + ti.add(wrapped, true); //Simple searches SearchResult srMiss = ti.search(null, "foofoo:bar", 50); diff --git a/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/text/RestrictedFormIVMTest.java b/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/text/RestrictedFormIVMTest.java new file mode 100644 index 00000000..5aa17987 --- /dev/null +++ b/gsrs-spring-legacy-indexer/src/test/java/ix/core/search/text/RestrictedFormIVMTest.java @@ -0,0 +1,178 @@ +package ix.core.search.text; + +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.persistence.Entity; +import javax.persistence.Id; + +import org.assertj.core.util.Arrays; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.ArgumentMatchers; +import org.mockito.Mockito; + +import gsrs.cache.GsrsCache; +import gsrs.indexer.IndexValueMakerFactory; +import ix.core.search.TextIndexerIndexedRecordTest; +import ix.core.search.TextIndexerIndexedRecordTest.TestEntity; +import ix.core.search.bulk.UserSavedListService; +import ix.core.search.text.TextIndexer.IndexRecord; +import ix.core.util.EntityUtils.EntityWrapper; +import ix.utils.Util; +import lombok.Builder; +import lombok.Data; + +public class RestrictedFormIVMTest { + + private IndexValueMakerFactory restrictedFromIVMMakerFactory; + + @Entity + @Data + @Builder + public static class TestEntity{ + @Id + public Long id; + public String name; + public String code; + } + + @TempDir + static File file; + + class TestCodeIndexValueMaker implements IndexValueMaker{ + + @Override + public Class getIndexedEntityClass() { + return TestEntity.class; + } + + @Override + public Set getFieldNames(){ + return Util.toSet("CAS"); + } + + @Override + public Set getTags(){ + return Util.toSet("CAS_code"); + } + + @Override + public void createIndexableValues(TestEntity t, Consumer consumer) { + consumer.accept(IndexableValue.simpleStringValue("CAS", "161622-14-6")); + } + } + + class TestNameIndexValueMaker implements IndexValueMaker{ + + @Override + public Class getIndexedEntityClass() { + return TestEntity.class; + } + + @Override + public Set getFieldNames(){ + return Util.toSet("std name"); + } + + @Override + public Set getTags(){ + return Util.toSet("std_name"); + } + + @Override + public void createIndexableValues(TestEntity t, Consumer consumer) { + consumer.accept(IndexableValue.simpleStringValue("std name", "test_name")); + } + } + + + private TextIndexer getNewTextIndexer() throws IOException { + Lucene4IndexServiceFactory fac = new Lucene4IndexServiceFactory(); + TextIndexerConfig conf = new TextIndexerConfig(); + conf.setEnabled(true); + conf.setFieldsuggest(true); + conf.setShouldLog(false); + UserSavedListService userSavedListService = mock(UserSavedListService.class); + Mockito.when(userSavedListService.getUserSearchResultLists(ArgumentMatchers.anyString(), ArgumentMatchers.anyString())).thenReturn(new ArrayList()); + + + IndexValueMakerFactory restrictedIVMMaker = new IndexValueMakerFactory() { + + @Override + public IndexValueMaker createIndexValueMakerFor(EntityWrapper ew) { + + IndexValueMaker nameIndexer = new TestNameIndexValueMaker(); + IndexValueMaker codeIndexer = new TestCodeIndexValueMaker(); + + List> list = new ArrayList>(); + list.add(nameIndexer); + list.add(codeIndexer); + Class cls = ew.getEntityClass(); + + return new CombinedIndexValueMaker(cls, list); + } + }; + + this.restrictedFromIVMMakerFactory = restrictedIVMMaker; + GsrsCache cache = mock(GsrsCache.class); + TextIndexer ti= new TextIndexer(file, fac, fac.createForDir(file), conf, restrictedIVMMaker, cache, (ee)->false, userSavedListService); + return ti; + } + + + @Test + public void testStrictedIVMApplied() throws NoSuchElementException, Exception { + TextIndexer ti=getNewTextIndexer(); + TestEntity addTest = TestEntity.builder().id(1l).name("demo1").code("161622-14-2").build(); + EntityWrapper wrapped = EntityWrapper.of(addTest); + + + IndexValueMaker includeCodeIVMmaker = restrictedFromIVMMakerFactory.createIndexValueMakerFor(wrapped) + .restrictedForm(Util.toSet("CAS_code"), true); +// System.out.println("include code "+includeIVMmaker.getFieldNames()); + Assertions.assertTrue(includeCodeIVMmaker.getFieldNames().contains("CAS")); + Assertions.assertEquals(includeCodeIVMmaker.getFieldNames().size(),1); + + + IndexValueMaker excludeCodeIVMmaker = restrictedFromIVMMakerFactory.createIndexValueMakerFor(wrapped) + .restrictedForm(Util.toSet("CAS_code"), false); +// System.out.println("exclude code "+includeIVMmaker.getFieldNames()); + Assertions.assertFalse(excludeCodeIVMmaker.getFieldNames().contains("CAS")); + Assertions.assertEquals(excludeCodeIVMmaker.getFieldNames().size(),1); + + IndexValueMaker includeNameIVMmaker = restrictedFromIVMMakerFactory.createIndexValueMakerFor(wrapped) + .restrictedForm(Util.toSet("std_name"), true); +// System.out.println("include code "+includeIVMmaker.getFieldNames()); + Assertions.assertTrue(includeNameIVMmaker.getFieldNames().contains("std name")); + Assertions.assertEquals(includeNameIVMmaker.getFieldNames().size(),1); + + + IndexValueMaker excludeNameIVMmaker = restrictedFromIVMMakerFactory.createIndexValueMakerFor(wrapped) + .restrictedForm(Util.toSet("std_name"), false); +// System.out.println("exclude code "+includeIVMmaker.getFieldNames()); + Assertions.assertFalse(excludeNameIVMmaker.getFieldNames().contains("std name")); + Assertions.assertEquals(excludeNameIVMmaker.getFieldNames().size(),1); + + + RestrictedIVMSpecification specs = new RestrictedIVMSpecification(true, Util.toSet("CAS_code")); + ti.add(wrapped, true, specs); + IndexRecord ir = ti.getIndexRecord(wrapped.getKey()); + Assertions.assertNotNull(ir); + Assertions.assertEquals(addTest.getId().toString(),ir.getId()); + ti.remove(wrapped.getKey()); + IndexRecord ir2 = ti.getIndexRecord(wrapped.getKey()); + Assertions.assertNull(ir2); + } + +} diff --git a/gsrs-spring-starter-tests/src/test/java/gsrs/dataexchange/area/ImportDataSearchTest.java b/gsrs-spring-starter-tests/src/test/java/gsrs/dataexchange/area/ImportDataSearchTest.java index 36cda6bd..57bf856a 100644 --- a/gsrs-spring-starter-tests/src/test/java/gsrs/dataexchange/area/ImportDataSearchTest.java +++ b/gsrs-spring-starter-tests/src/test/java/gsrs/dataexchange/area/ImportDataSearchTest.java @@ -122,7 +122,7 @@ private void addSomeDocuments() throws IOException { importMetadata.setSourceName("Funky File"); importMetadata.setImportStatus(ImportMetadata.RecordImportStatus.accepted); EntityUtils.EntityWrapper wrapper = EntityUtils.EntityWrapper.of(importMetadata); - indexer.add(wrapper); + indexer.add(wrapper, true); ImportMetadata importMetadata2 = new ImportMetadata(); UUID recordId = UUID.randomUUID(); @@ -133,7 +133,7 @@ private void addSomeDocuments() throws IOException { instanceId = UUID.randomUUID(); importMetadata2.setInstanceId(instanceId); EntityUtils.EntityWrapper wrapper2 = EntityUtils.EntityWrapper.of(importMetadata2); - indexer.add(wrapper2); + indexer.add(wrapper2,true); ImportMetadata importMetadata3 = new ImportMetadata(); UUID recordId3 = UUID.randomUUID(); @@ -145,7 +145,7 @@ private void addSomeDocuments() throws IOException { instanceId = UUID.randomUUID(); importMetadata3.setInstanceId(instanceId); EntityUtils.EntityWrapper wrapper3 = EntityUtils.EntityWrapper.of(importMetadata3); - indexer.add(wrapper3); + indexer.add(wrapper3, true); ImportMetadata importMetadata4 = new ImportMetadata(); UUID recordId4 = UUID.randomUUID(); @@ -159,6 +159,6 @@ private void addSomeDocuments() throws IOException { instanceId = UUID.randomUUID(); importMetadata4.setInstanceId(instanceId); EntityUtils.EntityWrapper wrapper4 = EntityUtils.EntityWrapper.of(importMetadata4); - indexer.add(wrapper4); + indexer.add(wrapper4, true); } } diff --git a/gsrs-spring-starter-tests/src/test/java/gsrs/startertests/controller/MyEntityService.java b/gsrs-spring-starter-tests/src/test/java/gsrs/startertests/controller/MyEntityService.java index 78071cbc..8155d8f9 100644 --- a/gsrs-spring-starter-tests/src/test/java/gsrs/startertests/controller/MyEntityService.java +++ b/gsrs-spring-starter-tests/src/test/java/gsrs/startertests/controller/MyEntityService.java @@ -15,6 +15,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.UUID; @@ -118,4 +119,11 @@ protected AbstractEntityUpdatedEvent newUpdateEvent(MyEntity updatedEn protected AbstractEntityCreatedEvent newCreationEvent(MyEntity createdEntity) { return null; } + + + @Override + public List getIDs() { + // TODO Auto-generated method stub + return new ArrayList() ; + } }