litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-873 create SearchEngine.documentCount()

Browse source 
Also, remove the vestigial IndexConstants class and two unused methods in PagedSearchController.
This commit is contained in:
Jim Blake 2015-01-06 16:04:19 -05:00
parent 6513d34d86
commit 8a0d8a2639
10 changed files with 162 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java
View file

@ -86,5 +86,9 @@ public interface SearchEngine extends Application.Module {
* Query the search index and return the results. Response is never null.
*/
SearchResponse query(SearchQuery query) throws SearchEngineException;
/**
* Find the number of documents in the search index.
*/
int documentCount() throws SearchEngineException;
}

6
webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java
View file

@ -18,6 +18,12 @@ public interface SearchResultDocumentList extends
*/
int size();
/**
* Retrieve the i'th document, starting with 0.
*
* @throws ArrayIndexOutOfBoundsException
* if i < 0 or i >= size()
*/
SearchResultDocument get(int i);
}

9
webapp/src/edu/cornell/mannlib/vitro/webapp/search/IndexConstants.java
View file

@ -1,9 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search;
public class IndexConstants {
public static String INDEX_REBUILD_REQUESTED_AT_STARTUP = "INDEX_REBUILD_REQUESTED_AT_STARTUP";
public static String SEARCH_DATAPROPERTY_BLACKLIST ="SEARCH_DATAPROPERTY_BLACKLIST";
public static String SEARCH_OBJECTPROPERTY_BLACKLIST = "SEARCH_OBJECTPROPERTY_BLACKLIST";
}

11
webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java
View file

@ -202,15 +202,10 @@ public class SearchIndexer implements IndexerIface {
* Returns true if there are documents in the index, false if there are none,
* and returns false on failure to connect to server.
*/
public boolean isIndexEmpty() {
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery("*:*");
@Override
public boolean isIndexEmpty() {
try {
SearchResponse rsp = server.query(query);
SearchResultDocumentList docs = rsp.getResults();
if(docs==null || docs.size()==0){
return true;
}
return server.documentCount() == 0;
} catch (SearchEngineException e) {
log.error("Could not connect to the search engine." ,e.getCause());
}

26
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java
View file

@ -48,7 +48,6 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.IndexConstants;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
@ -646,20 +645,6 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return rv;
}
@SuppressWarnings({ "unchecked", "unused" })
private HashSet<String> getDataPropertyBlacklist(){
HashSet<String>dpBlacklist = (HashSet<String>)
getServletContext().getAttribute(IndexConstants.SEARCH_DATAPROPERTY_BLACKLIST);
return dpBlacklist;
}
@SuppressWarnings({ "unchecked", "unused" })
private HashSet<String> getObjectPropertyBlacklist(){
HashSet<String>opBlacklist = (HashSet<String>)
getServletContext().getAttribute(IndexConstants.SEARCH_OBJECTPROPERTY_BLACKLIST);
return opBlacklist;
}
public static final int MAX_QUERY_LENGTH = 500;
protected boolean isRequestedFormatXml(VitroRequest req){
@ -707,8 +692,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
}
protected static Map<Format,Map<Result,String>> setupTemplateTable(){
Map<Format,Map<Result,String>> templateTable =
new HashMap<Format,Map<Result,String>>();
Map<Format,Map<Result,String>> table = new HashMap<>();
HashMap<Result,String> resultsToTemplates = new HashMap<Result,String>();
@ -716,7 +700,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
resultsToTemplates.put(Result.PAGED, "search-pagedResults.ftl");
resultsToTemplates.put(Result.ERROR, "search-error.ftl");
// resultsToTemplates.put(Result.BAD_QUERY, "search-badQuery.ftl");
templateTable.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates));
table.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates));
// set up XML format
resultsToTemplates = new HashMap<Result,String>();
@ -724,7 +708,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
resultsToTemplates.put(Result.ERROR, "search-xmlError.ftl");
// resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl");
templateTable.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates));
table.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates));
// set up CSV format
@ -733,9 +717,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
resultsToTemplates.put(Result.ERROR, "search-csvError.ftl");
// resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl");
templateTable.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates));
table.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates));
return Collections.unmodifiableMap(templateTable);
return Collections.unmodifiableMap(table);
}
}

1
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -463,7 +463,6 @@ public class IndexBuilder extends VitroBackgroundThread {
}
}
/* maybe ObjectSourceIface should be replaced with just an iterator. */
protected class UriToIndividualIterator implements Iterator<Individual>{
private final Iterator<String> uris;
private final WebappDaoFactory wdf;

72
webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java
View file

@ -7,6 +7,10 @@ import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.Lif
import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.LifecycleState.STOPPED;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -15,9 +19,12 @@ import edu.cornell.mannlib.vitro.webapp.modules.Application;
import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property;
import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation;
@ -206,4 +213,69 @@ public class InstrumentedSearchEngineWrapper implements SearchEngine {
}
}
@Override
public int documentCount() throws SearchEngineException {
try (SearchEngineLogger l = SearchEngineLogger.doCountQuery()) {
confirmActive();
int count = innerEngine.documentCount();
l.setSearchResponse(new SearchResponseForDocumentCount(count));
return count;
}
}
// ----------------------------------------------------------------------
// Helper classes
// ----------------------------------------------------------------------
private static class SearchResponseForDocumentCount implements SearchResponse {
private final int count;
public SearchResponseForDocumentCount(int count) {
this.count = count;
}
@Override
public SearchResultDocumentList getResults() {
return new EmptyDocumentListWithCount();
}
@Override
public Map<String, Map<String, List<String>>> getHighlighting() {
return Collections.emptyMap();
}
@Override
public SearchFacetField getFacetField(String name) {
return null;
}
@Override
public List<SearchFacetField> getFacetFields() {
return Collections.emptyList();
}
private class EmptyDocumentListWithCount implements SearchResultDocumentList {
@Override
public Iterator<SearchResultDocument> iterator() {
return Collections.emptyIterator();
}
@Override
public int size() {
return 0;
}
@Override
public long getNumFound() {
return count;
}
@Override
public SearchResultDocument get(int i) {
throw new ArrayIndexOutOfBoundsException(i);
}
}
}
}

63
webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java
View file

@ -82,6 +82,14 @@ public abstract class SearchEngineLogger implements AutoCloseable {
}
}
public static SearchEngineLogger doCountQuery() {
if (isEnabled(SEARCH_ENGINE_ENABLE)) {
return new CountQueryLogger();
} else {
return new DisabledLogger();
}
}
private static boolean isEnabled(Key enableKey) {
return log.isInfoEnabled()
&& DeveloperSettings.getInstance().getBoolean(enableKey);
@ -254,8 +262,8 @@ public abstract class SearchEngineLogger implements AutoCloseable {
QueryLogger(SearchQuery query) {
this.query = query;
this.stackTrace = new StackTraceUtility(InstrumentedSearchEngineWrapper.class,
true);
this.stackTrace = new StackTraceUtility(
InstrumentedSearchEngineWrapper.class, true);
this.passesRestrictions = passesQueryRestriction()
&& passesStackRestriction();
log.debug("QueryLogger: query=" + query + ", passes="
@ -312,8 +320,56 @@ public abstract class SearchEngineLogger implements AutoCloseable {
}
}
public static class CountQueryLogger extends SearchEngineLogger {
private final StackTraceUtility stackTrace;
private final boolean passesRestrictions;
private long count;
CountQueryLogger() {
this.stackTrace = new StackTraceUtility(
InstrumentedSearchEngineWrapper.class, true);
this.passesRestrictions = passesQueryRestriction()
&& passesStackRestriction();
log.debug("CountQueryLogger: passes=" + passesRestrictions);
}
private boolean passesStackRestriction() {
return stackTrace.passesStackRestriction(DeveloperSettings
.getInstance().getString(
Key.SEARCH_ENGINE_STACK_RESTRICTION));
}
/** Only passes if there is no restriction. */
private boolean passesQueryRestriction() {
String restriction = DeveloperSettings.getInstance().getString(
Key.SEARCH_ENGINE_QUERY_RESTRICTION);
return StringUtils.isEmpty(restriction);
}
@Override
public void setSearchResponse(SearchResponse response) {
this.count = response.getResults().getNumFound();
}
@Override
public void writeToLog() {
if (!passesRestrictions) {
return;
}
String results = "Document count query found " + count + " documents.\n";
String trace = stackTrace.format(showStackTrace());
log.info(String.format("%8.3f %s%s", elapsedSeconds(), results,
trace));
}
private boolean showStackTrace() {
return DeveloperSettings.getInstance().getBoolean(
Key.SEARCH_ENGINE_ADD_STACK_TRACE);
}
}
private static class DisabledLogger extends SearchEngineLogger {
@Override
public void setSearchResponse(SearchResponse response) {
// Does nothing.
@ -323,6 +379,5 @@ public abstract class SearchEngineLogger implements AutoCloseable {
protected void writeToLog() {
// Does nothing.
}
}
}

5
webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java
View file

@ -167,4 +167,9 @@ public class SolrSearchEngine implements SearchEngine {
}
}
@Override
public int documentCount() throws SearchEngineException {
SearchResponse response = query(createQuery("*:*"));
return (int) response.getResults().getNumFound();
}
}

22
webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java
View file

@ -22,9 +22,9 @@ public class SearchEngineStub implements SearchEngine {
// ----------------------------------------------------------------------
// Stub infrastructure
// ----------------------------------------------------------------------
Map<String, SearchResponseStub> queryResponses = new HashMap<>();
public void setQueryResponse(String queryText, SearchResponseStub response) {
queryResponses.put(queryText, response);
}
@ -58,77 +58,69 @@ public class SearchEngineStub implements SearchEngine {
return new BaseSearchInputDocument();
}
// ----------------------------------------------------------------------
// Un-implemented methods
// ----------------------------------------------------------------------
@Override
public void startup(Application application, ComponentStartupStatus ss) {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.startup() not implemented.");
}
@Override
public void shutdown(Application application) {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.shutdown() not implemented.");
}
@Override
public void ping() throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException("SearchEngineStub.ping() not implemented.");
}
@Override
public void add(SearchInputDocument... docs) throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException("SearchEngineStub.add() not implemented.");
}
@Override
public void add(Collection<SearchInputDocument> docs)
throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException("SearchEngineStub.add() not implemented.");
}
@Override
public void commit() throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException("SearchEngineStub.commit() not implemented.");
}
@Override
public void commit(boolean wait) throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException("SearchEngineStub.commit() not implemented.");
}
@Override
public void deleteById(String... ids) throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.deleteById() not implemented.");
}
@Override
public void deleteById(Collection<String> ids) throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.deleteById() not implemented.");
}
@Override
public void deleteByQuery(String query) throws SearchEngineException {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.deleteByQuery() not implemented.");
}
@Override
public int documentCount() throws SearchEngineException {
throw new RuntimeException(
"SearchEngineStub.documentCount() not implemented.");
}
}