VIVO-742 Base and Solr Implementations of SearchEngine

This commit is contained in:
Jim Blake 2014-04-22 10:40:17 -04:00
parent 6329343465
commit 5d653ebc9c
9 changed files with 923 additions and 0 deletions

View file

@ -0,0 +1,55 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.ArrayList;
import java.util.List;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
/**
* A foundation class for implementing SearchFacetField.
*/
public class BaseSearchFacetField implements SearchFacetField {
private final String name;
private final List<Count> values;
public BaseSearchFacetField(String name, List<? extends Count> values) {
this.name = name;
this.values = new ArrayList<>(values);
}
@Override
public String getName() {
return name;
}
@Override
public List<Count> getValues() {
return values;
}
/**
* A foundation class for implementing SearchFacetField.Count.
*/
public static class BaseCount implements SearchFacetField.Count {
private final String name;
private final long count;
public BaseCount(String name, long count) {
this.name = name;
this.count = count;
}
@Override
public String getName() {
return name;
}
@Override
public long getCount() {
return count;
}
}
}

View file

@ -0,0 +1,77 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
/**
* A foundation class for implementing SearchInputDocument.
*/
public class BaseSearchInputDocument implements SearchInputDocument {
private final Map<String, SearchInputField> fieldMap = new HashMap<>();
private float documentBoost = 1.0F;
@Override
public void addField(SearchInputField field) {
fieldMap.put(field.getName(), field);
}
@Override
public void addField(String name, Object... values) {
addField(name, 1.0F, Arrays.asList(values));
}
@Override
public void addField(String name, Collection<Object> values) {
addField(name, 1.0F, values);
}
@Override
public void addField(String name, float boost, Object... values) {
addField(name, boost, Arrays.asList(values));
}
@Override
public void addField(String name, float boost, Collection<Object> values) {
BaseSearchInputField field = new BaseSearchInputField(name);
field.setBoost(boost);
field.addValues(values);
fieldMap.put(name, field);
}
@Override
public void setDocumentBoost(float searchBoost) {
this.documentBoost = searchBoost;
}
@Override
public float getDocumentBoost() {
return this.documentBoost;
}
@Override
public SearchInputField getField(String name) {
return fieldMap.get(name);
}
@Override
public Map<String, SearchInputField> getFieldMap() {
return new HashMap<>(fieldMap);
}
/**
* Sub-classes should override this if the field requires special
* functionality.
*/
@Override
public SearchInputField createField(String name) {
return new BaseSearchInputField(name);
}
}

View file

@ -0,0 +1,64 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
/**
* A foundation class for implementing SearchInputField.
*/
public class BaseSearchInputField implements SearchInputField {
private final String name;
private final List<Object> valueList = new ArrayList<>();
private float boost = 1.0F;
public BaseSearchInputField(String name) {
this.name = name;
}
@Override
public void addValues(Object... values) {
addValues(Arrays.asList(values));
}
@Override
public void addValues(Collection<? extends Object> values) {
valueList.addAll(values);
}
@Override
public void setBoost(float boost) {
this.boost = boost;
}
@Override
public String getName() {
return name;
}
@Override
public float getBoost() {
return boost;
}
@Override
public Collection<Object> getValues() {
return new ArrayList<Object>(valueList);
}
@Override
public Object getFirstValue() {
if (valueList.isEmpty()) {
return null;
} else {
return valueList.get(0);
}
}
}

View file

@ -0,0 +1,179 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
/**
* A foundation class for implementing SearchQuery.
*/
public class BaseSearchQuery implements SearchQuery {
private String queryText;
private int start = 0;
private int rows = -1;
private final Set<String> fieldsToReturn = new HashSet<>();
private final Map<String, SearchQuery.Order> sortFields = new HashMap<>();
private final Set<String> filters = new HashSet<>();
private boolean faceting;
private final Set<String> facetFields = new HashSet<>();
private final Set<String> facetQueries = new HashSet<>();
private int facetMinCount = -1;
private final Map<String, List<String>> parameterMap = new HashMap<>();
@Override
public SearchQuery setQuery(String query) {
this.queryText = query;
return this;
}
@Override
public SearchQuery setStart(int start) {
this.start = start;
return this;
}
@Override
public SearchQuery setRows(int rows) {
this.rows = rows;
return this;
}
@Override
public SearchQuery addFields(String... names) {
return addFields(Arrays.asList(names));
}
@Override
public SearchQuery addFields(Collection<String> names) {
this.fieldsToReturn.addAll(names);
return this;
}
@Override
public SearchQuery addSortField(String name, Order order) {
sortFields.put(name, order);
return this;
}
@Override
public SearchQuery addFilterQuery(String filterQuery) {
filters.add(filterQuery);
return this;
}
@Override
public SearchQuery addFilterQueries(String... filterQueries) {
this.filters.addAll(Arrays.asList(filterQueries));
return this;
}
@Override
public SearchQuery setFaceting(boolean b) {
this.faceting = b;
return this;
}
@Override
public SearchQuery addFacetFields(String... fields) {
facetFields.addAll(Arrays.asList(fields));
return this;
}
@Override
public SearchQuery addFacetQueries(String... queries) {
facetQueries.addAll(Arrays.asList(queries));
return this;
}
@Override
public SearchQuery setFacetMinCount(int cnt) {
facetMinCount = cnt;
return this;
}
@Override
public SearchQuery addParameter(String name, String... values) {
parameterMap.put(name, Collections.unmodifiableList(new ArrayList<>(
Arrays.asList(values))));
return this;
}
@Override
public String getQuery() {
return queryText;
}
@Override
public int getStart() {
return start;
}
@Override
public int getRows() {
return rows;
}
@Override
public Set<String> getFieldsToReturn() {
return Collections.unmodifiableSet(fieldsToReturn);
}
@Override
public Map<String, SearchQuery.Order> getSortFields() {
return Collections.unmodifiableMap(sortFields);
}
@Override
public Set<String> getFilters() {
return Collections.unmodifiableSet(filters);
}
@Override
public boolean isFaceting() {
return faceting;
}
@Override
public Set<String> getFacetFields() {
return Collections.unmodifiableSet(facetFields);
}
@Override
public Set<String> getFacetQueries() {
return Collections.unmodifiableSet(facetQueries);
}
@Override
public int getFacetMinCount() {
return facetMinCount;
}
@Override
public Map<String, List<String>> getParameterMap() {
return Collections.unmodifiableMap(parameterMap);
}
@Override
public String toString() {
return "BaseSearchQuery [queryText=" + queryText + ", start=" + start
+ ", rows=" + rows + ", fieldsToReturn=" + fieldsToReturn
+ ", sortFields=" + sortFields + ", filters=" + filters
+ ", faceting=" + faceting + ", facetFields=" + facetFields
+ ", facetQueries=" + facetQueries + ", facetMinCount="
+ facetMinCount + ", parameterMap=" + parameterMap + "]";
}
}

View file

@ -0,0 +1,51 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
/**
* A foundation class for implementing SearchResponse;
*/
public class BaseSearchResponse implements SearchResponse {
private final Map<String, Map<String, List<String>>> highlighting;
private final Map<String, SearchFacetField> facetFields;
private final SearchResultDocumentList results;
public BaseSearchResponse(
Map<String, Map<String, List<String>>> highlighting,
Map<String, SearchFacetField> facetFields,
SearchResultDocumentList results) {
this.highlighting = highlighting;
this.facetFields = facetFields;
this.results = results;
}
@Override
public SearchResultDocumentList getResults() {
return results;
}
@Override
public Map<String, Map<String, List<String>>> getHighlighting() {
return Collections.unmodifiableMap(highlighting);
}
@Override
public SearchFacetField getFacetField(String name) {
return facetFields.get(name);
}
@Override
public List<SearchFacetField> getFacetFields() {
return new ArrayList<>(facetFields.values());
}
}

View file

@ -0,0 +1,75 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.base;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
/**
* A foundation class for implementing SearchResultDocument.
*/
public class BaseSearchResultDocument implements SearchResultDocument {
private final String uniqueId;
private final Map<String, Collection<Object>> fieldValuesMap;
public BaseSearchResultDocument(String uniqueId, Map<String, Collection<Object>> fieldValuesMap) {
this.uniqueId = uniqueId;
Map<String, Collection<Object>> map = new HashMap<>();
for (String name : fieldValuesMap.keySet()) {
map.put(name, Collections.unmodifiableList(new ArrayList<>(
fieldValuesMap.get(name))));
}
this.fieldValuesMap = Collections.unmodifiableMap(map);
}
@Override
public String getUniqueId() {
return uniqueId;
}
@Override
public Collection<String> getFieldNames() {
return fieldValuesMap.keySet();
}
@Override
public Object getFirstValue(String name) {
Collection<Object> values = fieldValuesMap.get(name);
if (values == null || values.isEmpty()) {
return null;
}
return values.iterator().next();
}
@Override
public String getStringValue(String name) {
Object o = getFirstValue(name);
if (o == null) {
return null;
} else {
return String.valueOf(o);
}
}
@Override
public Collection<Object> getFieldValues(String name) {
Collection<Object> values = fieldValuesMap.get(name);
if (values == null) {
return Collections.emptyList();
} else {
return values;
}
}
@Override
public Map<String, Collection<Object>> getFieldValuesMap() {
return fieldValuesMap;
}
}

View file

@ -0,0 +1,173 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.solr;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchFacetField;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchFacetField.BaseCount;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchResponse;
/**
* Utility method for converting from Solr-specific instances to Search-generic
* instances, and back.
*/
public class SolrConversionUtils {
// ----------------------------------------------------------------------
// Convert input documents to Solr-specific.
// ----------------------------------------------------------------------
static List<SolrInputDocument> convertToSolrInputDocuments(
Collection<SearchInputDocument> docs) {
List<SolrInputDocument> solrDocs = new ArrayList<>();
for (SearchInputDocument doc : docs) {
solrDocs.add(convertToSolrInputDocument(doc));
}
return solrDocs;
}
private static SolrInputDocument convertToSolrInputDocument(
SearchInputDocument doc) {
SolrInputDocument solrDoc = new SolrInputDocument(
convertToSolrInputFieldMap(doc.getFieldMap()));
solrDoc.setDocumentBoost(doc.getDocumentBoost());
return solrDoc;
}
private static Map<String, SolrInputField> convertToSolrInputFieldMap(
Map<String, SearchInputField> fieldMap) {
Map<String, SolrInputField> solrFieldMap = new HashMap<>();
for (String fieldName : fieldMap.keySet()) {
solrFieldMap.put(fieldName,
convertToSolrInputField(fieldMap.get(fieldName)));
}
return solrFieldMap;
}
private static SolrInputField convertToSolrInputField(
SearchInputField searchInputField) {
SolrInputField solrField = new SolrInputField(
searchInputField.getName());
solrField.addValue(searchInputField.getValues(),
searchInputField.getBoost());
return solrField;
}
// ----------------------------------------------------------------------
// Convert queries to Solr-specific.
// ----------------------------------------------------------------------
/**
* Convert from a SearchQuery to a SolrQuery, so the Solr server may execute
* it.
*/
static SolrQuery convertToSolrQuery(SearchQuery query) {
SolrQuery solrQuery = new SolrQuery(query.getQuery());
solrQuery.setStart(query.getStart());
int rows = query.getRows();
if (rows >= 0) {
solrQuery.setRows(rows);
}
for (String fieldToReturn : query.getFieldsToReturn()) {
solrQuery.addField(fieldToReturn);
}
Map<String, Order> sortFields = query.getSortFields();
for (String sortField : sortFields.keySet()) {
solrQuery.addSortField(sortField,
convertToSolrOrder(sortFields.get(sortField)));
}
for (String filter : query.getFilters()) {
solrQuery.addFilterQuery(filter);
}
solrQuery.setFacet(query.isFaceting());
for (String facetField : query.getFacetFields()) {
solrQuery.addFacetField(facetField);
}
for (String facetQuery : query.getFacetQueries()) {
solrQuery.addFacetQuery(facetQuery);
}
int minCount = query.getFacetMinCount();
if (minCount >= 0) {
solrQuery.setFacetMinCount(minCount);
}
Map<String, List<String>> parameterMap = query.getParameterMap();
for (String parameterName : parameterMap.keySet()) {
String[] values = parameterMap.get(parameterName).toArray(
new String[0]);
solrQuery.add(parameterName, values);
}
return solrQuery;
}
private static ORDER convertToSolrOrder(Order order) {
if (order == Order.DESC) {
return ORDER.desc;
} else {
return ORDER.asc;
}
}
// ----------------------------------------------------------------------
// Convert responses to Search-generic
// ----------------------------------------------------------------------
static SearchResponse convertToSearchResponse(QueryResponse response) {
return new BaseSearchResponse(response.getHighlighting(),
convertToSearchFacetFieldMap(response.getFacetFields()),
new SolrSearchResultDocumentList(response.getResults()));
}
private static Map<String, SearchFacetField> convertToSearchFacetFieldMap(
List<FacetField> facetFields) {
Map<String, SearchFacetField> map = new HashMap<>();
for (FacetField facetField : facetFields) {
map.put(facetField.getName(), convertToSearchFacetField(facetField));
}
return map;
}
private static SearchFacetField convertToSearchFacetField(
FacetField facetField) {
return new BaseSearchFacetField(facetField.getName(),
convertToSearchFacetFieldCounts(facetField.getValues()));
}
private static List<BaseCount> convertToSearchFacetFieldCounts(
List<FacetField.Count> solrCounts) {
List<BaseCount> searchCounts = new ArrayList<>();
for (FacetField.Count solrCount : solrCounts) {
searchCounts.add(new BaseCount(solrCount.getName(), solrCount
.getCount()));
}
return searchCounts;
}
}

View file

@ -0,0 +1,170 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.solr;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import javax.servlet.ServletContext;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.modules.Application;
import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery;
/**
* The Solr-based implementation of SearchEngine.
*/
public class SolrSearchEngine implements SearchEngine {
private HttpSolrServer server;
/**
* Set up the http connection with the solr server
*/
@Override
public void startup(Application application, ComponentStartupStatus css) {
ServletContext ctx = application.getServletContext();
String solrServerUrlString = ConfigurationProperties.getBean(ctx)
.getProperty("vitro.local.solr.url");
if (solrServerUrlString == null) {
css.fatal("Could not find vitro.local.solr.url in "
+ "runtime.properties. Vitro application needs the URL of "
+ "a solr server that it can use to index its data. It "
+ "should be something like http://localhost:${port}"
+ ctx.getContextPath() + "solr");
return;
}
try {
server = new HttpSolrServer(solrServerUrlString);
server.setSoTimeout(10000); // socket read timeout
server.setConnectionTimeout(10000);
server.setDefaultMaxConnectionsPerHost(100);
server.setMaxTotalConnections(100);
server.setMaxRetries(1);
css.info("Set up the Solr search engine; URL = '"
+ solrServerUrlString + "'.");
} catch (Exception e) {
css.fatal("Could not set up the Solr search engine", e);
}
}
@Override
public void shutdown(Application application) {
server.shutdown();
}
@Override
public void ping() throws SearchEngineException {
try {
server.ping();
} catch (SolrServerException | IOException e) {
throw new SearchEngineException(
"Solr server did not respont to ping.", e);
}
}
@Override
public SearchInputDocument createInputDocument() {
return new BaseSearchInputDocument();
}
@Override
public void add(SearchInputDocument... docs) throws SearchEngineException {
add(Arrays.asList(docs));
}
@Override
public void add(Collection<SearchInputDocument> docs)
throws SearchEngineException {
try {
server.add(SolrConversionUtils.convertToSolrInputDocuments(docs));
} catch (SolrServerException | IOException e) {
throw new SearchEngineException(
"Solr server failed to add documents " + docs, e);
}
}
@Override
public void commit() throws SearchEngineException {
try {
server.commit();
} catch (SolrServerException | IOException e) {
throw new SearchEngineException("Failed to commit to Solr server.",
e);
}
}
@Override
public void commit(boolean wait) throws SearchEngineException {
try {
server.commit(wait, wait);
} catch (SolrServerException | IOException e) {
throw new SearchEngineException("Failed to commit to Solr server.",
e);
}
}
@Override
public void deleteById(String... ids) throws SearchEngineException {
deleteById(Arrays.asList(ids));
}
@Override
public void deleteById(Collection<String> ids) throws SearchEngineException {
try {
server.deleteById(new ArrayList<>(ids));
} catch (SolrServerException | IOException e) {
throw new SearchEngineException(
"Solr server failed to delete documents: " + ids, e);
}
}
@Override
public void deleteByQuery(String query) throws SearchEngineException {
try {
server.deleteByQuery(query);
} catch (SolrServerException | IOException e) {
throw new SearchEngineException(
"Solr server failed to delete documents: " + query, e);
}
}
@Override
public SearchQuery createQuery() {
return new BaseSearchQuery();
}
@Override
public SearchQuery createQuery(String queryText) {
BaseSearchQuery query = new BaseSearchQuery();
query.setQuery(queryText);
return query;
}
@Override
public SearchResponse query(SearchQuery query) throws SearchEngineException {
try {
SolrQuery solrQuery = SolrConversionUtils.convertToSolrQuery(query);
QueryResponse response = server.query(solrQuery);
return SolrConversionUtils.convertToSearchResponse(response);
} catch (SolrServerException e) {
throw new SearchEngineException(
"Solr server failed to execute the query" + query, e);
}
}
}

View file

@ -0,0 +1,79 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.searchengine.solr;
import java.util.Iterator;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchResultDocument;
/**
* A Solr-based implementation of SearchResultDocumentList.
*
* It's necessary to use this instead of the base version, so the iterator can
* convert each document as it is requested.
*/
public class SolrSearchResultDocumentList implements SearchResultDocumentList {
private SolrDocumentList solrDocs;
public SolrSearchResultDocumentList(SolrDocumentList solrDocs) {
this.solrDocs = solrDocs;
}
@Override
public Iterator<SearchResultDocument> iterator() {
return new SearchResultDocumentIterator(solrDocs.iterator());
}
@Override
public long getNumFound() {
return solrDocs.getNumFound();
}
@Override
public int size() {
return solrDocs.size();
}
@Override
public SearchResultDocument get(int i) {
return convertToSearchResultDocument(solrDocs.get(i));
}
private static class SearchResultDocumentIterator implements
Iterator<SearchResultDocument> {
private final Iterator<SolrDocument> solrIterator;
public SearchResultDocumentIterator(Iterator<SolrDocument> solrIterator) {
this.solrIterator = solrIterator;
}
@Override
public boolean hasNext() {
return solrIterator.hasNext();
}
@Override
public SearchResultDocument next() {
return convertToSearchResultDocument(solrIterator.next());
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
}
private static SearchResultDocument convertToSearchResultDocument(
SolrDocument solrDoc) {
return new BaseSearchResultDocument(
(String) solrDoc.getFieldValue("DocId"),
solrDoc.getFieldValuesMap());
}
}