Skip to content
This repository has been archived by the owner on Oct 27, 2020. It is now read-only.

Feature/search 1388 #510

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Feature/search 1388 #510

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
565d674
[SEARCH-1388] http solr MLT query building
alessandrobenedetti Jul 5, 2019
1cfe4ae
[SEARCH-1388] minor revert
alessandrobenedetti Jul 5, 2019
d2db7bc
Merge remote-tracking branch 'origin/master' into feature/SEARCH-1388
alessandrobenedetti Jul 9, 2019
e305795
[SEARCH-1388] encoding added + exception ingestion fixed
alessandrobenedetti Jul 19, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 52 additions & 33 deletions src/main/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@
*/
public class SolrQueryHTTPClient extends AbstractSolrQueryHTTPClient implements SolrQueryClient
{
protected static final String ORIGINAL_SHARDS = "originalShards";
private static final String MLT_ENDPOINT = "getSimilar";
static Log s_logger = LogFactory.getLog(SolrQueryHTTPClient.class);

private DictionaryService dictionaryService;
Expand Down Expand Up @@ -428,6 +430,7 @@ public ResultSet executeQuery(final SearchParameters searchParameters, String la
url.append(httpClientAndBaseUrl.getSecond());

String languageUrlFragment = SolrClientUtil.extractLanguageFragment(languageMappings, language);
boolean moreLikeThisQuery = languageUrlFragment.equals(MLT_ENDPOINT);
if(!url.toString().endsWith("/"))
{
url.append("/");
Expand Down Expand Up @@ -464,32 +467,7 @@ else if(searchParameters.getLimitBy() == LimitBy.FINAL_SIZE && searchParameters.
}
url.append("&rows=").append(String.valueOf(maxResults));

if((searchParameters.getStores().size() > 1) || (mapping.isSharded()))
{
boolean requiresSeparator = false;
url.append("&shards=");
for(StoreRef storeRef : searchParameters.getStores())
{
SolrStoreMappingWrapper storeMapping =
SolrClientUtil.extractMapping(storeRef,
mappingLookup, shardRegistry,
useDynamicShardRegistration, beanFactory);

if(requiresSeparator)
{
url.append(',');
}
else
{
requiresSeparator = true;
}

url.append(storeMapping.getShards());

}
}

buildUrlParameters(searchParameters, mapping.isSharded(), encoder, url);
buildUrlParameters(searchParameters, mapping.isSharded(), encoder, url, moreLikeThisQuery);

final String searchTerm = searchParameters.getSearchTerm();
String spellCheckQueryStr = null;
Expand All @@ -503,7 +481,9 @@ else if(searchParameters.getLimitBy() == LimitBy.FINAL_SIZE && searchParameters.
}

JSONObject body = new JSONObject();
body.put("query", searchParameters.getQuery());
if (!moreLikeThisQuery) {
body.put("query", searchParameters.getQuery());
}


// Authorities go over as is - and tenant mangling and query building takes place on the SOLR side
Expand Down Expand Up @@ -586,19 +566,23 @@ else if(searchParameters.getLimitBy() == LimitBy.FINAL_SIZE && searchParameters.
}
catch (UnsupportedEncodingException e)
{
throw new LuceneQueryParserException("", e);
s_logger.error(e.getMessage(),e);
throw new LuceneQueryParserException(e.getMessage(), e);
}
catch (HttpException e)
{
throw new LuceneQueryParserException("", e);
s_logger.error(e.getMessage(),e);
throw new LuceneQueryParserException(e.getMessage(), e);
}
catch (IOException e)
{
throw new LuceneQueryParserException("", e);
s_logger.error(e.getMessage(),e);
throw new LuceneQueryParserException(e.getMessage(), e);
}
catch (JSONException e)
{
throw new LuceneQueryParserException("", e);
s_logger.error(e.getMessage(),e);
throw new LuceneQueryParserException(e.getMessage(), e);
}
}

Expand All @@ -608,9 +592,10 @@ else if(searchParameters.getLimitBy() == LimitBy.FINAL_SIZE && searchParameters.
* @param isSharded
* @param encoder
* @param url
* @param isMoreLikeThisQuery
* @throws UnsupportedEncodingException
*/
public void buildUrlParameters(SearchParameters searchParameters, boolean isSharded, URLCodec encoder, StringBuilder url)
public void buildUrlParameters(SearchParameters searchParameters, boolean isSharded, URLCodec encoder, StringBuilder url, boolean isMoreLikeThisQuery)
throws UnsupportedEncodingException
{
Locale locale = SolrClientUtil.extractLocale(searchParameters);
Expand All @@ -622,7 +607,7 @@ public void buildUrlParameters(SearchParameters searchParameters, boolean isShar
url.append("&").append(SearchParameters.ALTERNATIVE_DICTIONARY).append("=").append(alternativeDictionary);
for(String paramName : searchParameters.getExtraParameters().keySet())
{
url.append("&").append(paramName).append("=").append(searchParameters.getExtraParameters().get(paramName));
url.append("&").append(paramName).append("=").append(encoder.encode(searchParameters.getExtraParameters().get(paramName), "UTF-8"));
}
StringBuffer sortBuffer = buildSortParameters(searchParameters, encoder);
url.append(sortBuffer);
Expand Down Expand Up @@ -652,6 +637,36 @@ public void buildUrlParameters(SearchParameters searchParameters, boolean isShar
url.append("&fq=").append(encoder.encode(filterQuery, "UTF-8"));
}

StringBuilder shards = new StringBuilder();
if ((searchParameters.getStores().size() > 1) || (isSharded)) {
boolean requiresSeparator = false;
url.append("&shards=");

for (StoreRef storeRef : searchParameters.getStores()) {
SolrStoreMappingWrapper storeMapping =
SolrClientUtil.extractMapping(storeRef,
mappingLookup, shardRegistry,
useDynamicShardRegistration, beanFactory);

if (requiresSeparator) {
shards.append(',');
} else {
requiresSeparator = true;
}

shards.append(storeMapping.getShards());

}
url.append(shards.toString());
}

if (isMoreLikeThisQuery) {
url.append("&" + ORIGINAL_SHARDS + "=");
url.append(shards);
url.append("&docId=");
url.append(searchParameters.getQuery());
}

buildFacetParameters(searchParameters, isSharded, encoder, url);
buildPivotParameters(searchParameters, encoder, url);
buildStatsParameters(searchParameters, encoder, url);
Expand Down Expand Up @@ -1410,4 +1425,8 @@ public int getMaximumResultsFromUnlimitedQuery()
{
return maximumResultsFromUnlimitedQuery;
}

public void setMappingLookup(HashMap<StoreRef, SolrStoreMappingWrapper> mappingLookup) {
this.mappingLookup = mappingLookup;
}
}
15 changes: 15 additions & 0 deletions src/main/resources/alfresco/subsystems/Search/solr/solr-search-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
<entry key="solr-cmis" value="cmis" />
<entry key="fts-alfresco" value="afts" />
<entry key="solr-fts-alfresco" value="afts" />
<entry key="solr-mlt" value="getSimilar" />
</map>
</property>
<property name="storeMappings">
Expand Down Expand Up @@ -219,6 +220,20 @@
<ref bean="search.solrQueryHTTPCLient" />
</property>
</bean>

<bean id="search.solr.mlt" class="org.alfresco.repo.search.impl.solr.SolrQueryLanguage" >
<property name="factories">
<list>
<ref bean="search.indexerAndSearcherFactory" />
</list>
</property>
<property name="name">
<value>solr-mlt</value>
</property>
<property name="solrQueryHTTPClient">
<ref bean="search.solrQueryHTTPCLient" />
</property>
</bean>

<bean id="search.indexerAndSearcherFactory" class="org.alfresco.repo.service.StoreRedirectorProxyFactory">
<property name="proxyInterface">
Expand Down
15 changes: 15 additions & 0 deletions src/main/resources/alfresco/subsystems/Search/solr6/solr-search-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
<entry key="solr-cmis" value="cmis" />
<entry key="fts-alfresco" value="afts" />
<entry key="solr-fts-alfresco" value="afts" />
<entry key="solr-mlt" value="getSimilar" />
</map>
</property>
<property name="storeMappings">
Expand Down Expand Up @@ -270,6 +271,20 @@
<ref bean="search.solrQueryHTTPCLient" />
</property>
</bean>

<bean id="search.solr.mlt" class="org.alfresco.repo.search.impl.solr.SolrQueryLanguage" >
<property name="factories">
<list>
<ref bean="search.indexerAndSearcherFactory" />
</list>
</property>
<property name="name">
<value>solr-mlt</value>
</property>
<property name="solrQueryHTTPClient">
<ref bean="search.solrQueryHTTPCLient" />
</property>
</bean>

<bean id="search.indexerAndSearcherFactory" class="org.alfresco.repo.service.StoreRedirectorProxyFactory">
<property name="proxyInterface">
Expand Down
39 changes: 32 additions & 7 deletions src/test/java/org/alfresco/repo/search/impl/solr/SolrQueryHTTPClientTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import org.alfresco.service.cmr.dictionary.DataTypeDefinition;
import org.alfresco.service.cmr.dictionary.DictionaryService;
import org.alfresco.service.cmr.repository.NodeService;
import org.alfresco.service.cmr.repository.StoreRef;
import org.alfresco.service.cmr.search.FieldHighlightParameters;
import org.alfresco.service.cmr.search.GeneralHighlightParameters;
import org.alfresco.service.cmr.search.Interval;
Expand Down Expand Up @@ -97,6 +98,7 @@ public static void setUpBeforeClass() throws Exception
languageMappings.put("solr-alfresco", "alfresco");
languageMappings.put("solr-fts-alfresco", "afts");
languageMappings.put("solr-cmis", "cmis");
languageMappings.put("solr-mlt", "getSimilar");

NamespaceDAO namespaceDAO = mock(NamespaceDAO.class);
DictionaryService dictionaryService = mock(DictionaryService.class);
Expand Down Expand Up @@ -181,42 +183,65 @@ public void testBuildTimezone() throws UnsupportedEncodingException
SearchParameters params = new SearchParameters();
params.setTimezone("");
StringBuilder urlBuilder = new StringBuilder();
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
String url = urlBuilder.toString();
assertFalse(url.contains("&TZ"));

params.setTimezone("bob");
urlBuilder = new StringBuilder();
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
url = urlBuilder.toString();

//Timezone formats are not validated here so its just passing a string.
assertTrue(url.contains("&TZ=bob"));;
}

@Test
public void testMLTQuery() throws UnsupportedEncodingException {
SolrStoreMappingWrapper storeMapping = mock(SolrStoreMappingWrapper.class);
when(storeMapping.getShards()).thenReturn("shard1,shard2");
StoreRef sampleStoreRef = new StoreRef("test","test");
HashMap<StoreRef,SolrStoreMappingWrapper> mappingLookupTest = new HashMap<>();
mappingLookupTest.put(sampleStoreRef,storeMapping);
client.setMappingLookup(mappingLookupTest);

SearchParameters params = new SearchParameters();
params.addStore(sampleStoreRef);
params.setLanguage("solr-mlt");
params.setQuery("id1");
params.addExtraParameter("qf","{http://www.alfresco.org/model/content/1.0}content^500.0,{http://www.alfresco.org/model/content/1.0}title");
StringBuilder urlBuilder = new StringBuilder();
client.buildUrlParameters(params, true, encoder, urlBuilder, true);
String url = urlBuilder.toString();
assertNotNull(url);
assertTrue(url.contains("qf=%7Bhttp%3A%2F%2Fwww.alfresco.org%2Fmodel%2Fcontent%2F1.0%7Dcontent%5E500.0%2C%7Bhttp%3A%2F%2Fwww.alfresco.org%2Fmodel%2Fcontent%2F1.0%7Dtitle"));
assertTrue(url.contains("docId=id1"));
assertTrue(url.contains("&shards=shard1,shard2&originalShards=shard1,shard2"));
}

@Test
public void testBuildHighlightQuery() throws UnsupportedEncodingException
{
SearchParameters params = new SearchParameters();
params.setSearchTerm("bob");
StringBuilder urlBuilder = new StringBuilder();
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
String url = urlBuilder.toString();
assertNotNull(url);
assertFalse(url.contains("&hl"));

urlBuilder = new StringBuilder();
GeneralHighlightParameters highlightParameters = new GeneralHighlightParameters(null, null, null, null, null, null, null, null);
params.setHighlight(highlightParameters);
client.buildUrlParameters(params, true, encoder, urlBuilder);
client.buildUrlParameters(params, true, encoder, urlBuilder, false);
url = urlBuilder.toString();
assertTrue(url.contains("&hl=true"));
assertTrue(url.contains("&hl.q=bob"));

urlBuilder = new StringBuilder();
highlightParameters = new GeneralHighlightParameters(5, 10, false, "{", "}", 20, true, null);
params.setHighlight(highlightParameters);
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
url = urlBuilder.toString();
assertTrue(url.contains("&hl=true"));
assertTrue(url.contains("&hl.q=bob"));
Expand All @@ -236,7 +261,7 @@ public void testBuildHighlightQuery() throws UnsupportedEncodingException

try
{
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
fail();
}
catch (IllegalArgumentException iae)
Expand All @@ -249,7 +274,7 @@ public void testBuildHighlightQuery() throws UnsupportedEncodingException
urlBuilder = new StringBuilder();
highlightParameters = new GeneralHighlightParameters(5, 10, false, "{", "}", 20, true, fields);
params.setHighlight(highlightParameters);
client.buildUrlParameters(params, false, encoder, urlBuilder);
client.buildUrlParameters(params, false, encoder, urlBuilder, false);
url = urlBuilder.toString();
assertTrue(url.contains("&hl=true"));
assertTrue(url.contains("&hl.fl=" + encoder.encode("desc,title", "UTF-8")));
Expand Down