diff --git a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java index 419c2df19..1207f1f4a 100644 --- a/src/main/java/org/codelibs/fess/helper/IndexingHelper.java +++ b/src/main/java/org/codelibs/fess/helper/IndexingHelper.java @@ -21,12 +21,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.search.TotalHits; import org.codelibs.fess.es.client.SearchEngineClient; import org.codelibs.fess.mylasta.direction.FessConfig; import org.codelibs.fess.thumbnail.ThumbnailManager; import org.codelibs.fess.util.ComponentUtil; import org.codelibs.fess.util.DocList; import org.codelibs.fess.util.MemoryUtil; +import org.opensearch.action.admin.indices.refresh.RefreshResponse; import org.opensearch.action.search.SearchResponse; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; @@ -52,19 +54,23 @@ public void sendDocuments(final SearchEngineClient searchEngineClient, final Doc try { if (fessConfig.isThumbnailCrawlerEnabled()) { final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager(); + final String thumbnailField = fessConfig.getIndexFieldThumbnail(); docList.stream().forEach(doc -> { if (!thumbnailManager.offer(doc)) { if (logger.isDebugEnabled()) { - logger.debug("Removing {}={} from doc[{}]", fessConfig.getIndexFieldThumbnail(), - doc.get(fessConfig.getIndexFieldThumbnail()), doc.get(fessConfig.getIndexFieldUrl())); + logger.debug("Removing {}={} from doc[{}]", thumbnailField, doc.get(thumbnailField), + doc.get(fessConfig.getIndexFieldUrl())); } - doc.remove(fessConfig.getIndexFieldThumbnail()); + doc.remove(thumbnailField); } }); } final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper(); synchronized (searchEngineClient) { - deleteOldDocuments(searchEngineClient, docList); + final long deletedDocCount = deleteOldDocuments(searchEngineClient, docList); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} old docs", deletedDocCount); + } searchEngineClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), docList, (doc, builder) -> { final String configId = (String) doc.get(fessConfig.getIndexFieldConfigId()); crawlingConfigHelper.getPipeline(configId).ifPresent(s -> builder.setPipeline(s)); @@ -85,7 +91,7 @@ public void sendDocuments(final SearchEngineClient searchEngineClient, final Doc } } - private void deleteOldDocuments(final SearchEngineClient searchEngineClient, final DocList docList) { + protected long deleteOldDocuments(final SearchEngineClient searchEngineClient, final DocList docList) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); final List docIdList = new ArrayList<>(); @@ -108,7 +114,7 @@ private void deleteOldDocuments(final SearchEngineClient searchEngineClient, fin new String[] { fessConfig.getIndexFieldId(), fessConfig.getIndexFieldDocId() }); for (final Map doc : docs) { final Object oldIdValue = doc.get(fessConfig.getIndexFieldId()); - if (!idValue.equals(oldIdValue) && oldIdValue != null) { + if (oldIdValue != null && !idValue.equals(oldIdValue)) { final Object oldDocIdValue = doc.get(fessConfig.getIndexFieldDocId()); if (oldDocIdValue != null) { docIdList.add(oldDocIdValue.toString()); @@ -120,9 +126,10 @@ private void deleteOldDocuments(final SearchEngineClient searchEngineClient, fin } } if (!docIdList.isEmpty()) { - searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), + return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(), QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIdList.stream().toArray(n -> new String[n]))); } + return 0L; } public boolean updateDocument(final SearchEngineClient searchEngineClient, final String id, final String field, final Object value) { @@ -137,19 +144,23 @@ public boolean deleteDocument(final SearchEngineClient searchEngineClient, final public long deleteDocumentByUrl(final SearchEngineClient searchEngineClient, final String url) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), + return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(), QueryBuilders.termQuery(fessConfig.getIndexFieldUrl(), url)); } public long deleteDocumentsByDocId(final SearchEngineClient searchEngineClient, final List docIdList) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), + return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(), QueryBuilders.termsQuery(fessConfig.getIndexFieldDocId(), docIdList.stream().toArray(n -> new String[n]))); } public long deleteDocumentByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), queryBuilder); + return deleteDocumentByQuery(searchEngineClient, fessConfig.getIndexDocumentUpdateIndex(), queryBuilder); + } + + protected long deleteDocumentByQuery(final SearchEngineClient searchEngineClient, final String index, final QueryBuilder queryBuilder) { + return searchEngineClient.deleteByQuery(index, queryBuilder); } public Map getDocument(final SearchEngineClient searchEngineClient, final String id, final String[] fields) { @@ -168,9 +179,9 @@ public List> getDocumentListByPrefixId(final SearchEngineCli return getDocumentListByQuery(searchEngineClient, queryBuilder, fields); } - public void deleteChildDocument(final SearchEngineClient searchEngineClient, final String id) { + public long deleteChildDocument(final SearchEngineClient searchEngineClient, final String id) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), + return searchEngineClient.deleteByQuery(fessConfig.getIndexDocumentUpdateIndex(), QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id)); } @@ -185,9 +196,7 @@ protected List> getDocumentListByQuery(final SearchEngineCli final String[] fields) { final FessConfig fessConfig = ComponentUtil.getFessConfig(); - final SearchResponse countResponse = searchEngineClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex()) - .setQuery(queryBuilder).setSize(0).execute().actionGet(fessConfig.getIndexSearchTimeout()); - final long numFound = countResponse.getHits().getTotalHits().value; + final long numFound = getDocumentSizeByQuery(searchEngineClient, queryBuilder, fessConfig); final long maxSearchDocSize = fessConfig.getIndexerMaxSearchDocSizeAsInteger().longValue(); final boolean exceeded = numFound > maxSearchDocSize; if (exceeded) { @@ -217,35 +226,74 @@ protected List> getDocumentListByQuery(final SearchEngineCli }); } + protected long getDocumentSizeByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder, + final FessConfig fessConfig) { + final SearchResponse countResponse = searchEngineClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex()) + .setQuery(queryBuilder).setSize(0).setTrackTotalHits(true).execute().actionGet(fessConfig.getIndexSearchTimeout()); + final TotalHits totalHits = countResponse.getHits().getTotalHits(); + if (totalHits != null) { + return totalHits.value; + } + return 0; + } + public long deleteBySessionId(final String sessionId) { + final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient(); final FessConfig fessConfig = ComponentUtil.getFessConfig(); final String index = fessConfig.getIndexDocumentUpdateIndex(); + return deleteBySessionId(searchEngineClient, index, sessionId); + } + + public long deleteBySessionId(final SearchEngineClient searchEngineClient, final String index, final String sessionId) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldSegment(), sessionId); - return deleteByQueryBuilder(index, queryBuilder); + return deleteByQueryBuilder(searchEngineClient, index, queryBuilder); } public long deleteByConfigId(final String configId) { + final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient(); final FessConfig fessConfig = ComponentUtil.getFessConfig(); final String index = fessConfig.getIndexDocumentUpdateIndex(); + return deleteByConfigId(searchEngineClient, index, configId); + } + + public long deleteByConfigId(final SearchEngineClient searchEngineClient, final String index, final String configId) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldConfigId(), configId); - return deleteByQueryBuilder(index, queryBuilder); + return deleteByQueryBuilder(searchEngineClient, index, queryBuilder); } public long deleteByVirtualHost(final String virtualHost) { + final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient(); final FessConfig fessConfig = ComponentUtil.getFessConfig(); final String index = fessConfig.getIndexDocumentUpdateIndex(); + return deleteByVirtualHost(searchEngineClient, index, virtualHost); + } + + public long deleteByVirtualHost(final SearchEngineClient searchEngineClient, final String index, final String virtualHost) { + final FessConfig fessConfig = ComponentUtil.getFessConfig(); final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldVirtualHost(), virtualHost); - return deleteByQueryBuilder(index, queryBuilder); + return deleteByQueryBuilder(searchEngineClient, index, queryBuilder); } - protected long deleteByQueryBuilder(final String index, final QueryBuilder queryBuilder) { - final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient(); - searchEngineClient.admin().indices().prepareRefresh(index).execute().actionGet(); + protected long deleteByQueryBuilder(final SearchEngineClient searchEngineClient, final String index, final QueryBuilder queryBuilder) { + refreshIndex(searchEngineClient, index); final long numOfDeleted = searchEngineClient.deleteByQuery(index, queryBuilder); - logger.info("Deleted {} old docs.", numOfDeleted); + if (logger.isDebugEnabled()) { + logger.debug("Deleted {} old docs.", numOfDeleted); + } return numOfDeleted; } + protected int refreshIndex(final SearchEngineClient searchEngineClient, final String index) { + final RefreshResponse response = searchEngineClient.admin().indices().prepareRefresh(index).execute().actionGet(); + if (logger.isDebugEnabled()) { + logger.debug("[{}] refresh status: {} ({}/{}/{})", index, response.getStatus(), response.getTotalShards(), + response.getSuccessfulShards(), response.getFailedShards()); + } + return response.getStatus().getStatus(); + } + public long calculateDocumentSize(final Map dataMap) { return MemoryUtil.sizeOf(dataMap); } diff --git a/src/test/java/org/codelibs/fess/helper/IndexingHelperTest.java b/src/test/java/org/codelibs/fess/helper/IndexingHelperTest.java new file mode 100644 index 000000000..4d064ef65 --- /dev/null +++ b/src/test/java/org/codelibs/fess/helper/IndexingHelperTest.java @@ -0,0 +1,533 @@ +/* + * Copyright 2012-2023 CodeLibs Project and the Others. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied. See the License for the specific language + * governing permissions and limitations under the License. + */ +package org.codelibs.fess.helper; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; + +import org.codelibs.fess.app.service.WebConfigService; +import org.codelibs.fess.es.client.SearchEngineClient; +import org.codelibs.fess.es.config.exentity.WebConfig; +import org.codelibs.fess.mylasta.direction.FessConfig; +import org.codelibs.fess.thumbnail.ThumbnailManager; +import org.codelibs.fess.unit.UnitFessTestCase; +import org.codelibs.fess.util.ComponentUtil; +import org.codelibs.fess.util.DocList; +import org.dbflute.optional.OptionalEntity; +import org.opensearch.action.index.IndexAction; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchAction; +import org.opensearch.action.search.SearchRequestBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; + +public class IndexingHelperTest extends UnitFessTestCase { + private IndexingHelper indexingHelper; + + private long documentSizeByQuery = 0L; + + @Override + public void setUp() throws Exception { + super.setUp(); + indexingHelper = new IndexingHelper() { + @Override + protected long getDocumentSizeByQuery(final SearchEngineClient searchEngineClient, final QueryBuilder queryBuilder, + final FessConfig fessConfig) { + return documentSizeByQuery; + } + + @Override + protected int refreshIndex(final SearchEngineClient searchEngineClient, final String index) { + return 200; + } + }; + indexingHelper.setMaxRetryCount(5); + indexingHelper.setDefaultRowSize(100); + indexingHelper.setRequestInterval(500); + ComponentUtil.register(indexingHelper, "indexingHelper"); + ComponentUtil.register(new ThumbnailManager(), "thumbnailManager"); + final CrawlingConfigHelper crawlingConfigHelper = new CrawlingConfigHelper(); + crawlingConfigHelper.init(); + ComponentUtil.register(crawlingConfigHelper, "crawlingConfigHelper"); + ComponentUtil.register(new WebConfigService() { + @Override + public OptionalEntity getWebConfig(final String id) { + final WebConfig webConfig = new WebConfig(); + webConfig.setId(id); + return OptionalEntity.of(webConfig); + } + }, WebConfigService.class.getCanonicalName()); + } + + public void test_sendDocuments() { + documentSizeByQuery = 0L; + final AtomicReference sentIndex = new AtomicReference<>(); + final List> sentDocList = new ArrayList<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public String[] addAll(final String index, final List> docList, + final BiConsumer, IndexRequestBuilder> options) { + sentIndex.set(index); + docList.forEach(x -> options.accept(x, new IndexRequestBuilder(this, IndexAction.INSTANCE))); + sentDocList.addAll(docList); + return docList.stream().map(x -> (String) x.get("id")).toArray(n -> new String[n]); + } + + @Override + public List> getDocumentList(final String index, final SearchCondition condition) { + return Collections.emptyList(); + } + }; + ComponentUtil.register(client, "searchEngineClient"); + final DocList docList = new DocList(); + indexingHelper.sendDocuments(client, docList); + assertEquals(0, docList.size()); + assertEquals(0, sentDocList.size()); + assertNull(sentIndex.get()); + + sentIndex.set(null); + sentDocList.clear(); + docList.add(new HashMap<>(Map.of(// + "_id", "001", // + "config_id", "W01", // + "url", "http://test.com/001"// + ))); + docList.add(new HashMap<>(Map.of(// + "_id", "002", // + "thumbnail", "http://test.com/002", // + "url", "http://test.com/002"// + ))); + indexingHelper.sendDocuments(client, docList); + assertEquals(0, docList.size()); + assertEquals(2, sentDocList.size()); + assertEquals("fess.update", sentIndex.get()); + } + + public void test_deleteOldDocuments() { + documentSizeByQuery = 0L; + final List deletedDocIdList = new ArrayList<>(); + final List> oldDocList = new ArrayList<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public List> getDocumentList(final String index, final SearchCondition condition) { + return oldDocList; + } + + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermsQueryBuilder termsQueryBuilder) { + termsQueryBuilder.values().stream().forEach(o -> deletedDocIdList.add(o.toString())); + } + return deletedDocIdList.size(); + } + }; + ComponentUtil.register(client, "searchEngineClient"); + final DocList docList = new DocList(); + assertEquals(0, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(0, docList.size()); + assertEquals(0, deletedDocIdList.size()); + + docList.clear(); + deletedDocIdList.clear(); + docList.add(new HashMap<>(Map.of(// + "config_id", "W01", // + "doc_id", "1", // + "url", "http://test.com/001"// + ))); + documentSizeByQuery = oldDocList.size(); + assertEquals(0, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(1, docList.size()); + assertEquals(0, deletedDocIdList.size()); + + docList.clear(); + deletedDocIdList.clear(); + docList.add(new HashMap<>(Map.of(// + "_id", "001", // + "doc_id", "1", // + "url", "http://test.com/001"// + ))); + documentSizeByQuery = oldDocList.size(); + assertEquals(0, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(1, docList.size()); + assertEquals(0, deletedDocIdList.size()); + + docList.clear(); + deletedDocIdList.clear(); + docList.add(new HashMap<>(Map.of(// + "_id", "001", // + "config_id", "W01", // + "doc_id", "1", // + "url", "http://test.com/001"// + ))); + documentSizeByQuery = oldDocList.size(); + assertEquals(0, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(1, docList.size()); + assertEquals(0, deletedDocIdList.size()); + + deletedDocIdList.clear(); + oldDocList.add(new HashMap<>(Map.of(// + "_id", "001", // same id + "config_id", "W01", // + "doc_id", "1", // + "url", "http://test.com/001"// + ))); + documentSizeByQuery = oldDocList.size(); + assertEquals(0, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(1, docList.size()); + assertEquals(0, deletedDocIdList.size()); + + deletedDocIdList.clear(); + oldDocList.clear(); + oldDocList.add(new HashMap<>(Map.of(// + "_id", "001_OLD", // same id + "config_id", "W01", // + "doc_id", "1", // + "url", "http://test.com/001"// + ))); + documentSizeByQuery = oldDocList.size(); + assertEquals(1, indexingHelper.deleteOldDocuments(client, docList)); + assertEquals(1, docList.size()); + assertEquals(1, deletedDocIdList.size()); + assertEquals("1", deletedDocIdList.get(0)); + } + + public void test_updateDocument() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public boolean update(final String index, final String id, final String field, final Object value) { + resultMap.put("index", index); + resultMap.put("id", id); + resultMap.put("field", field); + resultMap.put("value", value.toString()); + return true; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String id = "001"; + final String field = "content"; + final String value = "test"; + + assertTrue(indexingHelper.updateDocument(client, id, field, value)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(id, resultMap.get("id")); + assertEquals(field, resultMap.get("field")); + assertEquals(value, resultMap.get("value")); + } + + public void test_deleteDocument() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public boolean delete(final String index, final String id) { + resultMap.put("index", index); + resultMap.put("id", id); + return true; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String id = "001"; + + assertTrue(indexingHelper.deleteDocument(client, id)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(id, resultMap.get("id")); + } + + public void test_deleteDocumentByUrl() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("url", termQueryBuilder.value().toString()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String url = "http://test.com/"; + + assertEquals(1, indexingHelper.deleteDocumentByUrl(client, url)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(url, resultMap.get("url")); + } + + public void test_deleteDocumentsByDocId() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermsQueryBuilder termsQueryBuilder) { + resultMap.put("index", index); + resultMap.put("ids", termsQueryBuilder.values().stream().map(Object::toString).collect(Collectors.toList())); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final List docList = new ArrayList<>(); + + indexingHelper.deleteDocumentsByDocId(client, docList); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(0, ((List) resultMap.get("ids")).size()); + + docList.add("001"); + indexingHelper.deleteDocumentsByDocId(client, docList); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(1, ((List) resultMap.get("ids")).size()); + assertEquals("001", ((List) resultMap.get("ids")).get(0)); + } + + public void test_deleteDocumentByQuery() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("url", termQueryBuilder.value().toString()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String url = "http://test.com/"; + + assertEquals(1, indexingHelper.deleteDocumentByQuery(client, QueryBuilders.termQuery("fess.update", url))); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(url, resultMap.get("url")); + } + + public void test_getDocument() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + + @Override + public OptionalEntity> getDocument(final String index, + final SearchCondition condition) { + final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE); + condition.build(builder); + resultMap.put("index", index); + resultMap.put("query", builder.toString()); + return OptionalEntity.of(Map.of("_id", "001", "title", "test1", "content", "test2")); + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final Map document = indexingHelper.getDocument(client, "001", new String[] { "title", "content" }); + assertEquals("fess.update", resultMap.get("index")); + assertEquals( + "{\"query\":{\"ids\":{\"values\":[\"001\"],\"boost\":1.0}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}", + resultMap.get("query")); + assertEquals(3, document.size()); + } + + public void test_getDocumentListByPrefixId() { + documentSizeByQuery = 1L; + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public List> getDocumentList(final String index, final SearchCondition condition) { + final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE); + condition.build(builder); + resultMap.put("index", index); + resultMap.put("query", builder.toString()); + final List> docList = new ArrayList<>(); + docList.add(Map.of("_id", "001", "title", "test1", "content", "test2")); + return docList; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final List> documents = + indexingHelper.getDocumentListByPrefixId(client, "001", new String[] { "title", "content" }); + assertEquals("fess.update", resultMap.get("index")); + assertEquals( + "{\"size\":1,\"query\":{\"prefix\":{\"_id\":{\"value\":\"001\",\"boost\":1.0}}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}", + resultMap.get("query")); + assertEquals(1, documents.size()); + } + + public void test_deleteChildDocument() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("id", termQueryBuilder.value().toString()); + resultMap.put("field", termQueryBuilder.fieldName()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String id = "001"; + + assertEquals(1, indexingHelper.deleteChildDocument(client, id)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals(id, resultMap.get("id")); + assertEquals("parent_id", resultMap.get("field")); + } + + public void test_getChildDocumentList() { + documentSizeByQuery = 1L; + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public List> getDocumentList(final String index, final SearchCondition condition) { + final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE); + condition.build(builder); + resultMap.put("index", index); + resultMap.put("query", builder.toString()); + final List> docList = new ArrayList<>(); + docList.add(Map.of("_id", "001", "title", "test1", "content", "test2")); + return docList; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final List> documents = indexingHelper.getChildDocumentList(client, "001", new String[] { "title", "content" }); + assertEquals("fess.update", resultMap.get("index")); + assertEquals( + "{\"size\":1,\"query\":{\"term\":{\"parent_id\":{\"value\":\"001\",\"boost\":1.0}}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}", + resultMap.get("query")); + assertEquals(1, documents.size()); + } + + public void test_getDocumentListByQuery() { + documentSizeByQuery = 1L; + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public List> getDocumentList(final String index, final SearchCondition condition) { + final SearchRequestBuilder builder = new SearchRequestBuilder(this, SearchAction.INSTANCE); + condition.build(builder); + resultMap.put("index", index); + resultMap.put("query", builder.toString()); + final List> docList = new ArrayList<>(); + docList.add(Map.of("_id", "001", "title", "test1", "content", "test2")); + return docList; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final List> documents = + indexingHelper.getDocumentListByQuery(client, QueryBuilders.idsQuery().addIds("001"), new String[] { "title", "content" }); + assertEquals("fess.update", resultMap.get("index")); + assertEquals( + "{\"size\":1,\"query\":{\"ids\":{\"values\":[\"001\"],\"boost\":1.0}},\"_source\":{\"includes\":[\"title\",\"content\"],\"excludes\":[]}}", + resultMap.get("query")); + assertEquals(1, documents.size()); + } + + public void test_deleteBySessionId() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("field", termQueryBuilder.fieldName()); + resultMap.put("value", termQueryBuilder.value().toString()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String sessionId = "session1"; + + assertEquals(1, indexingHelper.deleteBySessionId(sessionId)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals("segment", resultMap.get("field")); + assertEquals(sessionId, resultMap.get("value")); + } + + public void test_deleteByConfigId() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("field", termQueryBuilder.fieldName()); + resultMap.put("value", termQueryBuilder.value().toString()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String configId = "W01"; + + assertEquals(1, indexingHelper.deleteByConfigId(configId)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals("config_id", resultMap.get("field")); + assertEquals(configId, resultMap.get("value")); + } + + public void test_deleteByVirtualHost() { + final Map resultMap = new HashMap<>(); + final SearchEngineClient client = new SearchEngineClient() { + @Override + public long deleteByQuery(final String index, final QueryBuilder queryBuilder) { + if (queryBuilder instanceof final TermQueryBuilder termQueryBuilder) { + resultMap.put("index", index); + resultMap.put("field", termQueryBuilder.fieldName()); + resultMap.put("value", termQueryBuilder.value().toString()); + return 1; + } + return 0; + } + }; + ComponentUtil.register(client, "searchEngineClient"); + + final String virtualHost = "aaa"; + + assertEquals(1, indexingHelper.deleteByVirtualHost(virtualHost)); + assertEquals("fess.update", resultMap.get("index")); + assertEquals("virtual_host", resultMap.get("field")); + assertEquals(virtualHost, resultMap.get("value")); + } + + public void test_calculateDocumentSize() { + assertEquals(0, indexingHelper.calculateDocumentSize(Collections.emptyMap())); + assertEquals(118, indexingHelper.calculateDocumentSize(Map.of("id", "test"))); + assertEquals(249, indexingHelper.calculateDocumentSize(Map.of("id", "test", "url", "http://test.com/"))); + } +}