Merge branch 'develop' into feature/searchIndexerTry

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngine.java

@@ -86,5 +86,9 @@ public interface SearchEngine extends Application.Module {
 	 * Query the search index and return the results. Response is never null.
 	 */
 	SearchResponse query(SearchQuery query) throws SearchEngineException;
-
+	
+	/**
+	 * Find the number of documents in the search index.
+	 */
+	int documentCount() throws SearchEngineException;
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchResultDocumentList.java

@@ -18,6 +18,12 @@ public interface SearchResultDocumentList extends
 	 */
 	int size();
 
+	/**
+	 * Retrieve the i'th document, starting with 0.
+	 * 
+	 * @throws ArrayIndexOutOfBoundsException
+	 *             if i < 0 or i >= size()
+	 */
 	SearchResultDocument get(int i);
 
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java

@@ -43,6 +43,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceImpl;
+import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString;
 
 public abstract class RDFServiceJena extends RDFServiceImpl implements RDFService {
 
@@ -106,8 +107,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
 				dumpOperation(model, modelChange);
 			}
             if (modelChange.getOperation() == ModelChange.Operation.ADD) {
-                model.read(modelChange.getSerializedModel(), null,
+            	Model addition = parseModel(modelChange);
-                        getSerializationFormatString(modelChange.getSerializationFormat()));  
+            	model.add(addition);
             } else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) {
                 Model removal = parseModel(modelChange);
                 model.remove(removal);
@@ -165,17 +166,13 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
 
 		String changeString = new String(changeBytes).replace('\n', ' ');
 
-		String graphUri = modelChange.getGraphURI();
+		log.debug(String.format(
-		int delimHere = Math.max(graphUri.lastIndexOf('#'),
+				">>>>OPERATION: %3.3s %03dpunc, format=%s, graphUri='%s'\n"
-				graphUri.lastIndexOf('/'));
+						+ "    start=%.200s\n" + "    model=%s",
-		String graphLocalName = graphUri.substring(delimHere + 1);
+				modelChange.getOperation(), puncCount,
-
+				modelChange.getSerializationFormat(),
-		String modelClassName = model.getClass().getSimpleName();
+				modelChange.getGraphURI(), changeString,
-
+				ToString.modelToString(model)));
-		log.debug(String
-				.format(">>>>OPERATION: %3.3s %03dpunc, name='%s', class=%s, start=%.200s",
-						op, puncCount, graphLocalName, modelClassName,
-						changeString));
 	}
 
 	private void removeBlankNodesWithSparqlUpdate(Dataset dataset, Model model, String graphURI) {

webapp/src/edu/cornell/mannlib/vitro/webapp/search/IndexConstants.java

@@ -1,9 +0,0 @@
-/* $This file is distributed under the terms of the license in /doc/license.txt$ */
-
-package edu.cornell.mannlib.vitro.webapp.search;
-
-public class IndexConstants {
-    public static String INDEX_REBUILD_REQUESTED_AT_STARTUP = "INDEX_REBUILD_REQUESTED_AT_STARTUP";
-    public static String SEARCH_DATAPROPERTY_BLACKLIST ="SEARCH_DATAPROPERTY_BLACKLIST";
-    public static String SEARCH_OBJECTPROPERTY_BLACKLIST = "SEARCH_OBJECTPROPERTY_BLACKLIST";
-}

webapp/src/edu/cornell/mannlib/vitro/webapp/search/SearchIndexer.java

@@ -202,15 +202,10 @@ public class SearchIndexer implements IndexerIface {
      * Returns true if there are documents in the index, false if there are none,
      * and returns false on failure to connect to server.
      */
-    public boolean isIndexEmpty() {
+    @Override
-    	SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
+	public boolean isIndexEmpty() {
-    	query.setQuery("*:*");
     	try {
-    		SearchResponse rsp = server.query(query);
+    		return server.documentCount() == 0;
-    		SearchResultDocumentList docs = rsp.getResults();
-    		if(docs==null || docs.size()==0){
-    			return true;
-    		}
     	} catch (SearchEngineException e) {
     		log.error("Could not connect to the search engine." ,e.getCause());
     	}

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java

@@ -48,7 +48,6 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
-import edu.cornell.mannlib.vitro.webapp.search.IndexConstants;
 import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
 import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
 import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
@@ -646,20 +645,6 @@ public class PagedSearchController extends FreemarkerHttpServlet {
         return rv;
     }
     
-    @SuppressWarnings({ "unchecked", "unused" })
-    private HashSet<String> getDataPropertyBlacklist(){
-        HashSet<String>dpBlacklist = (HashSet<String>)
-        getServletContext().getAttribute(IndexConstants.SEARCH_DATAPROPERTY_BLACKLIST);
-        return dpBlacklist;        
-    }
-    
-    @SuppressWarnings({ "unchecked", "unused" })
-    private HashSet<String> getObjectPropertyBlacklist(){
-        HashSet<String>opBlacklist = (HashSet<String>)
-        getServletContext().getAttribute(IndexConstants.SEARCH_OBJECTPROPERTY_BLACKLIST);
-        return opBlacklist;        
-    }
-    
     public static final int MAX_QUERY_LENGTH = 500;
 
     protected boolean isRequestedFormatXml(VitroRequest req){
@@ -707,8 +692,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
     }
     
     protected static Map<Format,Map<Result,String>> setupTemplateTable(){
-        Map<Format,Map<Result,String>> templateTable = 
+        Map<Format,Map<Result,String>> table = new HashMap<>();
-            new HashMap<Format,Map<Result,String>>();
         
         HashMap<Result,String> resultsToTemplates = new HashMap<Result,String>();
         
@@ -716,7 +700,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
         resultsToTemplates.put(Result.PAGED, "search-pagedResults.ftl");
         resultsToTemplates.put(Result.ERROR, "search-error.ftl");
         // resultsToTemplates.put(Result.BAD_QUERY, "search-badQuery.ftl");        
-        templateTable.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates));
+        table.put(Format.HTML, Collections.unmodifiableMap(resultsToTemplates));
         
         // set up XML format
         resultsToTemplates = new HashMap<Result,String>();
@@ -724,7 +708,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
         resultsToTemplates.put(Result.ERROR, "search-xmlError.ftl");
 
         // resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl");        
-        templateTable.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates));
+        table.put(Format.XML, Collections.unmodifiableMap(resultsToTemplates));
         
         
         // set up CSV format
@@ -733,9 +717,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
         resultsToTemplates.put(Result.ERROR, "search-csvError.ftl");
         
         // resultsToTemplates.put(Result.BAD_QUERY, "search-xmlBadQuery.ftl");        
-        templateTable.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates));
+        table.put(Format.CSV, Collections.unmodifiableMap(resultsToTemplates));
 
         
-        return Collections.unmodifiableMap(templateTable);
+        return Collections.unmodifiableMap(table);
     }
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java

@@ -463,7 +463,6 @@ public class IndexBuilder extends VitroBackgroundThread {
         }
     }               
     
-    /* maybe ObjectSourceIface should be replaced with just an iterator. */
     protected class UriToIndividualIterator implements Iterator<Individual>{        
         private final Iterator<String> uris;
         private final WebappDaoFactory wdf;

webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/InstrumentedSearchEngineWrapper.java

@@ -7,6 +7,10 @@ import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.Lif
 import static edu.cornell.mannlib.vitro.webapp.modules.Application.Component.LifecycleState.STOPPED;
 
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -15,9 +19,12 @@ import edu.cornell.mannlib.vitro.webapp.modules.Application;
 import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
+import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
+import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
+import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation;
 
@@ -206,4 +213,69 @@ public class InstrumentedSearchEngineWrapper implements SearchEngine {
 		}
 	}
 
+	@Override
+	public int documentCount() throws SearchEngineException {
+		try (SearchEngineLogger l = SearchEngineLogger.doCountQuery()) {
+			confirmActive();
+			int count = innerEngine.documentCount();
+			l.setSearchResponse(new SearchResponseForDocumentCount(count));
+			return count;
+		}
+	}
+	
+	// ----------------------------------------------------------------------
+	// Helper classes
+	// ----------------------------------------------------------------------
+
+	
+	private static class SearchResponseForDocumentCount implements SearchResponse {
+		private final int count;
+
+		public SearchResponseForDocumentCount(int count) {
+			this.count = count;
+		}
+
+		@Override
+		public SearchResultDocumentList getResults() {
+			return new EmptyDocumentListWithCount();
+		}
+
+		@Override
+		public Map<String, Map<String, List<String>>> getHighlighting() {
+			return Collections.emptyMap();
+		}
+
+		@Override
+		public SearchFacetField getFacetField(String name) {
+			return null;
+		}
+
+		@Override
+		public List<SearchFacetField> getFacetFields() {
+			return Collections.emptyList();
+		}
+		
+		private class EmptyDocumentListWithCount implements SearchResultDocumentList {
+				@Override
+				public Iterator<SearchResultDocument> iterator() {
+					return Collections.emptyIterator();
+				}
+				
+				@Override
+				public int size() {
+					return 0;
+				}
+				
+				@Override
+				public long getNumFound() {
+					return count;
+				}
+				
+				@Override
+				public SearchResultDocument get(int i) {
+					throw new ArrayIndexOutOfBoundsException(i);
+				}
+		}
+	}
+	
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/SearchEngineLogger.java

@@ -82,6 +82,14 @@ public abstract class SearchEngineLogger implements AutoCloseable {
 		}
 	}
 
+	public static SearchEngineLogger doCountQuery() {
+		if (isEnabled(SEARCH_ENGINE_ENABLE)) {
+			return new CountQueryLogger();
+		} else {
+			return new DisabledLogger();
+		}
+	}
+
 	private static boolean isEnabled(Key enableKey) {
 		return log.isInfoEnabled()
 				&& DeveloperSettings.getInstance().getBoolean(enableKey);
@@ -254,8 +262,8 @@ public abstract class SearchEngineLogger implements AutoCloseable {
 
 		QueryLogger(SearchQuery query) {
 			this.query = query;
-			this.stackTrace = new StackTraceUtility(InstrumentedSearchEngineWrapper.class,
+			this.stackTrace = new StackTraceUtility(
-					true);
+					InstrumentedSearchEngineWrapper.class, true);
 			this.passesRestrictions = passesQueryRestriction()
 					&& passesStackRestriction();
 			log.debug("QueryLogger: query=" + query + ", passes="
@@ -312,8 +320,56 @@ public abstract class SearchEngineLogger implements AutoCloseable {
 		}
 	}
 
+	public static class CountQueryLogger extends SearchEngineLogger {
+		private final StackTraceUtility stackTrace;
+		private final boolean passesRestrictions;
+
+		private long count;
+
+		CountQueryLogger() {
+			this.stackTrace = new StackTraceUtility(
+					InstrumentedSearchEngineWrapper.class, true);
+			this.passesRestrictions = passesQueryRestriction()
+					&& passesStackRestriction();
+			log.debug("CountQueryLogger: passes=" + passesRestrictions);
+		}
+
+		private boolean passesStackRestriction() {
+			return stackTrace.passesStackRestriction(DeveloperSettings
+					.getInstance().getString(
+							Key.SEARCH_ENGINE_STACK_RESTRICTION));
+		}
+
+		/** Only passes if there is no restriction. */
+		private boolean passesQueryRestriction() {
+			String restriction = DeveloperSettings.getInstance().getString(
+					Key.SEARCH_ENGINE_QUERY_RESTRICTION);
+			return StringUtils.isEmpty(restriction);
+		}
+
+		@Override
+		public void setSearchResponse(SearchResponse response) {
+			this.count = response.getResults().getNumFound();
+		}
+
+		@Override
+		public void writeToLog() {
+			if (!passesRestrictions) {
+				return;
+			}
+			String results = "Document count query found " + count + " documents.\n";
+			String trace = stackTrace.format(showStackTrace());
+			log.info(String.format("%8.3f %s%s", elapsedSeconds(), results,
+					trace));
+		}
+
+		private boolean showStackTrace() {
+			return DeveloperSettings.getInstance().getBoolean(
+					Key.SEARCH_ENGINE_ADD_STACK_TRACE);
+		}
+	}
+
 	private static class DisabledLogger extends SearchEngineLogger {
-		
 		@Override
 		public void setSearchResponse(SearchResponse response) {
 			// Does nothing.
@@ -323,6 +379,5 @@ public abstract class SearchEngineLogger implements AutoCloseable {
 		protected void writeToLog() {
 			// Does nothing.
 		}
-
 	}
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java

@@ -167,4 +167,9 @@ public class SolrSearchEngine implements SearchEngine {
 		}
 	}
 
+	@Override
+	public int documentCount() throws SearchEngineException {
+    	SearchResponse response = query(createQuery("*:*"));
+    	return (int) response.getResults().getNumFound();
+	}
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/DefaultTripleStoreQuirks.java

@@ -3,6 +3,7 @@
 package edu.cornell.mannlib.vitro.webapp.triplesource.impl;
 
 import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
 
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.TripleStoreQuirks;
 
@@ -12,8 +13,15 @@ import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.TripleStoreQuirks;
 public class DefaultTripleStoreQuirks implements TripleStoreQuirks {
 
 	@Override
-	public boolean hasFileGraphChanged(Model fromFile, Model previous, String graphURI) {
+	public boolean hasFileGraphChanged(Model fromFile, Model previous,
-		return !fromFile.isIsomorphicWith(previous);
+			String graphURI) {
+		/**
+		 * The test for isomorphism involves a large number of ASK queries. It
+		 * appears to be faster to read the previous graph data into memory than
+		 * to run those queries against the RDFService.
+		 */
+		return !fromFile.isIsomorphicWith(ModelFactory.createDefaultModel()
+				.add(previous));
 	}
 
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/tdb/TDBTripleStoreQuirks.java

@@ -47,7 +47,7 @@ public class TDBTripleStoreQuirks extends DefaultTripleStoreQuirks {
 			Model mangled = ModelFactory.createDefaultModel();
 			mangled.read(mangleStream, null, "N-TRIPLE");
 
-			return !mangled.isIsomorphicWith(previous);
+			return !isIsomorphic(previous, mangled);
 		} catch (Exception e) {
 			log.warn("Failed to test for changes in filegraph. "
 					+ "Change assumed.", e);
@@ -55,4 +55,17 @@ public class TDBTripleStoreQuirks extends DefaultTripleStoreQuirks {
 		}
 	}
 
+	/**
+	 * If we check isomorphism with a dbModel directly, we issue many ASK
+	 * queries against the underlying RDFService.
+	 * 
+	 * It's faster to read the entire model from the RDFService and then issue
+	 * all of those ASKs against a memory-resident model.
+	 */
+	private boolean isIsomorphic(Model previous, Model mangled) {
+		Model previousInMemory = ModelFactory.createDefaultModel()
+				.add(previous);
+		return mangled.isIsomorphicWith(previousInMemory);
+	}
+
 }

webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java

@@ -22,9 +22,9 @@ public class SearchEngineStub implements SearchEngine {
 	// ----------------------------------------------------------------------
 	// Stub infrastructure
 	// ----------------------------------------------------------------------
-	
+
 	Map<String, SearchResponseStub> queryResponses = new HashMap<>();
-	
+
 	public void setQueryResponse(String queryText, SearchResponseStub response) {
 		queryResponses.put(queryText, response);
 	}
@@ -58,77 +58,69 @@ public class SearchEngineStub implements SearchEngine {
 		return new BaseSearchInputDocument();
 	}
 
-
 	// ----------------------------------------------------------------------
 	// Un-implemented methods
 	// ----------------------------------------------------------------------
 
-	
-
 	@Override
 	public void startup(Application application, ComponentStartupStatus ss) {
-		// TODO Auto-generated method stub
 		throw new RuntimeException(
 				"SearchEngineStub.startup() not implemented.");
 	}
 
 	@Override
 	public void shutdown(Application application) {
-		// TODO Auto-generated method stub
 		throw new RuntimeException(
 				"SearchEngineStub.shutdown() not implemented.");
 	}
 
 	@Override
 	public void ping() throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException("SearchEngineStub.ping() not implemented.");
 	}
 
 	@Override
 	public void add(SearchInputDocument... docs) throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException("SearchEngineStub.add() not implemented.");
 	}
 
 	@Override
 	public void add(Collection<SearchInputDocument> docs)
 			throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException("SearchEngineStub.add() not implemented.");
 	}
 
 	@Override
 	public void commit() throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException("SearchEngineStub.commit() not implemented.");
 	}
 
 	@Override
 	public void commit(boolean wait) throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException("SearchEngineStub.commit() not implemented.");
 	}
 
 	@Override
 	public void deleteById(String... ids) throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException(
 				"SearchEngineStub.deleteById() not implemented.");
 	}
 
 	@Override
 	public void deleteById(Collection<String> ids) throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException(
 				"SearchEngineStub.deleteById() not implemented.");
 	}
 
 	@Override
 	public void deleteByQuery(String query) throws SearchEngineException {
-		// TODO Auto-generated method stub
 		throw new RuntimeException(
 				"SearchEngineStub.deleteByQuery() not implemented.");
 	}
 
+	@Override
+	public int documentCount() throws SearchEngineException {
+		throw new RuntimeException(
+				"SearchEngineStub.documentCount() not implemented.");
+	}
 }