diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java index 0b8e92d31..b44d77586 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java @@ -86,5 +86,9 @@ public interface SearchEngine extends Application.Module { * Query the search index and return the results. Response is never null. */ SearchResponse query(SearchQuery query) throws SearchEngineException; - + + /** + * Find the number of documents in the search index. + */ + int documentCount() throws SearchEngineException; } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java index 37563e45d..83915339a 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java @@ -18,6 +18,12 @@ public interface SearchResultDocumentList extends */ int size(); + /** + * Retrieve the i'th document, starting with 0. + * + * @throws ArrayIndexOutOfBoundsException + * if i < 0 or i >= size() + */ SearchResultDocument get(int i); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java index c704dd230..76e057489 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java @@ -43,6 +43,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceImpl; +import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; public abstract class RDFServiceJena extends RDFServiceImpl implements RDFService { @@ -106,8 +107,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic dumpOperation(model, modelChange); } if (modelChange.getOperation() == ModelChange.Operation.ADD) { - model.read(modelChange.getSerializedModel(), null, - getSerializationFormatString(modelChange.getSerializationFormat())); + Model addition = parseModel(modelChange); + model.add(addition); } else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) { Model removal = parseModel(modelChange); model.remove(removal); @@ -165,17 +166,13 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic String changeString = new String(changeBytes).replace('\n', ' '); - String graphUri = modelChange.getGraphURI(); - int delimHere = Math.max(graphUri.lastIndexOf('#'), - graphUri.lastIndexOf('/')); - String graphLocalName = graphUri.substring(delimHere + 1); - - String modelClassName = model.getClass().getSimpleName(); - - log.debug(String - .format(">>>>OPERATION: %3.3s %03dpunc, name='%s', class=%s, start=%.200s", - op, puncCount, graphLocalName, modelClassName, - changeString)); + log.debug(String.format( + ">>>>OPERATION: %3.3s %03dpunc, format=%s, graphUri='%s'\n" + + " start=%.200s\n" + " model=%s", + modelChange.getOperation(), puncCount, + modelChange.getSerializationFormat(), + modelChange.getGraphURI(), changeString, + ToString.modelToString(model))); } private void removeBlankNodesWithSparqlUpdate(Dataset dataset, Model model, String graphURI) { diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/IndexConstants.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/IndexConstants.java deleted file mode 100644 index d7066b5be..000000000 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/IndexConstants.java +++ /dev/null @@ -1,9 +0,0 @@ -/* $This file is distributed under the terms of the license in /doc/license.txt$ */ - -package edu.cornell.mannlib.vitro.webapp.search; - -public class IndexConstants { - public static String INDEX_REBUILD_REQUESTED_AT_STARTUP = "INDEX_REBUILD_REQUESTED_AT_STARTUP"; - public static String SEARCH_DATAPROPERTY_BLACKLIST ="SEARCH_DATAPROPERTY_BLACKLIST"; - public static String SEARCH_OBJECTPROPERTY_BLACKLIST = "SEARCH_OBJECTPROPERTY_BLACKLIST"; -} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java index aafef4ad3..8ab34a82e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java @@ -202,15 +202,10 @@ public class SearchIndexer implements IndexerIface { * Returns true if there are documents in the index, false if there are none, * and returns false on failure to connect to server. */ - public boolean isIndexEmpty() { - SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(); - query.setQuery("*:*"); + @Override + public boolean isIndexEmpty() { try { - SearchResponse rsp = server.query(query); - SearchResultDocumentList docs = rsp.getResults(); - if(docs==null || docs.size()==0){ - return true; - } + return server.documentCount() == 0; } catch (SearchEngineException e) { log.error("Could not connect to the search engine." ,e.getCause()); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java index c968390d6..be9d7491d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java @@ -48,7 +48,6 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList; -import edu.cornell.mannlib.vitro.webapp.search.IndexConstants; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult; @@ -646,20 +645,6 @@ public class PagedSearchController extends FreemarkerHttpServlet { return rv; } - @SuppressWarnings({ "unchecked", "unused" }) - private HashSet getDataPropertyBlacklist(){ - HashSetdpBlacklist = (HashSet) - getServletContext().getAttribute(IndexConstants.SEARCH_DATAPROPERTY_BLACKLIST); - return dpBlacklist; - } - - @SuppressWarnings({ "unchecked", "unused" }) - private HashSet getObjectPropertyBlacklist(){ - HashSetopBlacklist = (HashSet) - getServletContext().getAttribute(IndexConstants.SEARCH_OBJECTPROPERTY_BLACKLIST); - return opBlacklist; - } - public static final int MAX_QUERY_LENGTH = 500; protected boolean isRequestedFormatXml(VitroRequest req){ @@ -707,8 +692,7 @@ public class PagedSearchController extends FreemarkerHttpServlet { } protected static Map> setupTemplateTable(){ - Map> templateTable = - new HashMap>(); + Map> table = new HashMap<>(); HashMap resultsToTemplates = new HashMap(); @@ -716,7 +700,7 @@ public class PagedSearchController extends FreemarkerHttpServlet { resultsToTemplates.put(Result.PAGED, "search-pagedResults.ftl"); resultsToTemplates.put(Result.ERROR, "search-error.ftl"); // resultsToTemplates.put(Result.BAD_QUERY, "search-badQuery.ftl"); - templateTable.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates)); + table.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates)); // set up XML format resultsToTemplates = new HashMap(); @@ -724,7 +708,7 @@ public class PagedSearchController extends FreemarkerHttpServlet { resultsToTemplates.put(Result.ERROR, "search-xmlError.ftl"); // resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl"); - templateTable.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates)); + table.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates)); // set up CSV format @@ -733,9 +717,9 @@ public class PagedSearchController extends FreemarkerHttpServlet { resultsToTemplates.put(Result.ERROR, "search-csvError.ftl"); // resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl"); - templateTable.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates)); + table.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates)); - return Collections.unmodifiableMap(templateTable); + return Collections.unmodifiableMap(table); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index 9a9f38bc8..8e1835911 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -463,7 +463,6 @@ public class IndexBuilder extends VitroBackgroundThread { } } - /* maybe ObjectSourceIface should be replaced with just an iterator. */ protected class UriToIndividualIterator implements Iterator{ private final Iterator uris; private final WebappDaoFactory wdf; diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java index e4e88bbed..72228d940 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java @@ -7,6 +7,10 @@ import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.Lif import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.LifecycleState.STOPPED; import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -15,9 +19,12 @@ import edu.cornell.mannlib.vitro.webapp.modules.Application; import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList; import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property; import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation; @@ -206,4 +213,69 @@ public class InstrumentedSearchEngineWrapper implements SearchEngine { } } + @Override + public int documentCount() throws SearchEngineException { + try (SearchEngineLogger l = SearchEngineLogger.doCountQuery()) { + confirmActive(); + int count = innerEngine.documentCount(); + l.setSearchResponse(new SearchResponseForDocumentCount(count)); + return count; + } + } + + // ---------------------------------------------------------------------- + // Helper classes + // ---------------------------------------------------------------------- + + + private static class SearchResponseForDocumentCount implements SearchResponse { + private final int count; + + public SearchResponseForDocumentCount(int count) { + this.count = count; + } + + @Override + public SearchResultDocumentList getResults() { + return new EmptyDocumentListWithCount(); + } + + @Override + public Map>> getHighlighting() { + return Collections.emptyMap(); + } + + @Override + public SearchFacetField getFacetField(String name) { + return null; + } + + @Override + public List getFacetFields() { + return Collections.emptyList(); + } + + private class EmptyDocumentListWithCount implements SearchResultDocumentList { + @Override + public Iterator iterator() { + return Collections.emptyIterator(); + } + + @Override + public int size() { + return 0; + } + + @Override + public long getNumFound() { + return count; + } + + @Override + public SearchResultDocument get(int i) { + throw new ArrayIndexOutOfBoundsException(i); + } + } + } + } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java index b46cb6314..f3821ea77 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java @@ -82,6 +82,14 @@ public abstract class SearchEngineLogger implements AutoCloseable { } } + public static SearchEngineLogger doCountQuery() { + if (isEnabled(SEARCH_ENGINE_ENABLE)) { + return new CountQueryLogger(); + } else { + return new DisabledLogger(); + } + } + private static boolean isEnabled(Key enableKey) { return log.isInfoEnabled() && DeveloperSettings.getInstance().getBoolean(enableKey); @@ -254,8 +262,8 @@ public abstract class SearchEngineLogger implements AutoCloseable { QueryLogger(SearchQuery query) { this.query = query; - this.stackTrace = new StackTraceUtility(InstrumentedSearchEngineWrapper.class, - true); + this.stackTrace = new StackTraceUtility( + InstrumentedSearchEngineWrapper.class, true); this.passesRestrictions = passesQueryRestriction() && passesStackRestriction(); log.debug("QueryLogger: query=" + query + ", passes=" @@ -312,8 +320,56 @@ public abstract class SearchEngineLogger implements AutoCloseable { } } + public static class CountQueryLogger extends SearchEngineLogger { + private final StackTraceUtility stackTrace; + private final boolean passesRestrictions; + + private long count; + + CountQueryLogger() { + this.stackTrace = new StackTraceUtility( + InstrumentedSearchEngineWrapper.class, true); + this.passesRestrictions = passesQueryRestriction() + && passesStackRestriction(); + log.debug("CountQueryLogger: passes=" + passesRestrictions); + } + + private boolean passesStackRestriction() { + return stackTrace.passesStackRestriction(DeveloperSettings + .getInstance().getString( + Key.SEARCH_ENGINE_STACK_RESTRICTION)); + } + + /** Only passes if there is no restriction. */ + private boolean passesQueryRestriction() { + String restriction = DeveloperSettings.getInstance().getString( + Key.SEARCH_ENGINE_QUERY_RESTRICTION); + return StringUtils.isEmpty(restriction); + } + + @Override + public void setSearchResponse(SearchResponse response) { + this.count = response.getResults().getNumFound(); + } + + @Override + public void writeToLog() { + if (!passesRestrictions) { + return; + } + String results = "Document count query found " + count + " documents.\n"; + String trace = stackTrace.format(showStackTrace()); + log.info(String.format("%8.3f %s%s", elapsedSeconds(), results, + trace)); + } + + private boolean showStackTrace() { + return DeveloperSettings.getInstance().getBoolean( + Key.SEARCH_ENGINE_ADD_STACK_TRACE); + } + } + private static class DisabledLogger extends SearchEngineLogger { - @Override public void setSearchResponse(SearchResponse response) { // Does nothing. @@ -323,6 +379,5 @@ public abstract class SearchEngineLogger implements AutoCloseable { protected void writeToLog() { // Does nothing. } - } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java index 7a3dbf993..848de8acf 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java @@ -167,4 +167,9 @@ public class SolrSearchEngine implements SearchEngine { } } + @Override + public int documentCount() throws SearchEngineException { + SearchResponse response = query(createQuery("*:*")); + return (int) response.getResults().getNumFound(); + } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/DefaultTripleStoreQuirks.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/DefaultTripleStoreQuirks.java index 5365a8bb8..e84b2ff02 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/DefaultTripleStoreQuirks.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/DefaultTripleStoreQuirks.java @@ -3,6 +3,7 @@ package edu.cornell.mannlib.vitro.webapp.triplesource.impl; import com.hp.hpl.jena.rdf.model.Model; +import com.hp.hpl.jena.rdf.model.ModelFactory; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.TripleStoreQuirks; @@ -12,8 +13,15 @@ import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.TripleStoreQuirks; public class DefaultTripleStoreQuirks implements TripleStoreQuirks { @Override - public boolean hasFileGraphChanged(Model fromFile, Model previous, String graphURI) { - return !fromFile.isIsomorphicWith(previous); + public boolean hasFileGraphChanged(Model fromFile, Model previous, + String graphURI) { + /** + * The test for isomorphism involves a large number of ASK queries. It + * appears to be faster to read the previous graph data into memory than + * to run those queries against the RDFService. + */ + return !fromFile.isIsomorphicWith(ModelFactory.createDefaultModel() + .add(previous)); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/tdb/TDBTripleStoreQuirks.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/tdb/TDBTripleStoreQuirks.java index 936c93f03..2e2e88542 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/tdb/TDBTripleStoreQuirks.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/tdb/TDBTripleStoreQuirks.java @@ -47,7 +47,7 @@ public class TDBTripleStoreQuirks extends DefaultTripleStoreQuirks { Model mangled = ModelFactory.createDefaultModel(); mangled.read(mangleStream, null, "N-TRIPLE"); - return !mangled.isIsomorphicWith(previous); + return !isIsomorphic(previous, mangled); } catch (Exception e) { log.warn("Failed to test for changes in filegraph. " + "Change assumed.", e); @@ -55,4 +55,17 @@ public class TDBTripleStoreQuirks extends DefaultTripleStoreQuirks { } } + /** + * If we check isomorphism with a dbModel directly, we issue many ASK + * queries against the underlying RDFService. + * + * It's faster to read the entire model from the RDFService and then issue + * all of those ASKs against a memory-resident model. + */ + private boolean isIsomorphic(Model previous, Model mangled) { + Model previousInMemory = ModelFactory.createDefaultModel() + .add(previous); + return mangled.isIsomorphicWith(previousInMemory); + } + } diff --git a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java index 3df9f73e3..4b832952f 100644 --- a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java +++ b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java @@ -22,9 +22,9 @@ public class SearchEngineStub implements SearchEngine { // ---------------------------------------------------------------------- // Stub infrastructure // ---------------------------------------------------------------------- - + Map queryResponses = new HashMap<>(); - + public void setQueryResponse(String queryText, SearchResponseStub response) { queryResponses.put(queryText, response); } @@ -58,77 +58,69 @@ public class SearchEngineStub implements SearchEngine { return new BaseSearchInputDocument(); } - // ---------------------------------------------------------------------- // Un-implemented methods // ---------------------------------------------------------------------- - - @Override public void startup(Application application, ComponentStartupStatus ss) { - // TODO Auto-generated method stub throw new RuntimeException( "SearchEngineStub.startup() not implemented."); } @Override public void shutdown(Application application) { - // TODO Auto-generated method stub throw new RuntimeException( "SearchEngineStub.shutdown() not implemented."); } @Override public void ping() throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException("SearchEngineStub.ping() not implemented."); } @Override public void add(SearchInputDocument... docs) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException("SearchEngineStub.add() not implemented."); } @Override public void add(Collection docs) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException("SearchEngineStub.add() not implemented."); } @Override public void commit() throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException("SearchEngineStub.commit() not implemented."); } @Override public void commit(boolean wait) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException("SearchEngineStub.commit() not implemented."); } @Override public void deleteById(String... ids) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException( "SearchEngineStub.deleteById() not implemented."); } @Override public void deleteById(Collection ids) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException( "SearchEngineStub.deleteById() not implemented."); } @Override public void deleteByQuery(String query) throws SearchEngineException { - // TODO Auto-generated method stub throw new RuntimeException( "SearchEngineStub.deleteByQuery() not implemented."); } + @Override + public int documentCount() throws SearchEngineException { + throw new RuntimeException( + "SearchEngineStub.documentCount() not implemented."); + } }