VIVO-873 Adapt the client code to the new interface.

Create a bridge implementation of SearchIndexerImpl that just wraps around an old IndexBuilder.

Modify client code:
Application, BasicAuthenticator, SearchServiceController, SparqlUpdateApiController,
UpdateUrisInIndex and VClassGroupCache

Rewrite IndexController to use AJAX and to show the current status and history of the indexer events.

webapp/config/applicationSetup.n3

@@ -0,0 +1,45 @@
+@prefix : <http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationSetup#> .
+
+:application 
+    a   <java:edu.cornell.mannlib.vitro.webapp.application.ApplicationImpl> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.Application> ;
+    :hasSearchEngine              :instrumentedSearchEngineWrapper ;
+    :hasSearchIndexer             :basicSearchIndexer ;
+    :hasImageProcessor            :jaiImageProcessor ;
+    :hasFileStorage               :ptiFileStorage ;
+    :hasContentTripleSource       :sdbContentTripleSource ;
+    :hasConfigurationTripleSource :tdbConfigurationTripleSource ;
+    :hasTBoxReasonerModule        :jfactTBoxReasonerModule .
+
+:jaiImageProcessor 
+    a   <java:edu.cornell.mannlib.vitro.webapp.imageprocessor.jai.JaiImageProcessor> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor> .
+
+:ptiFileStorage 
+    a   <java:edu.cornell.mannlib.vitro.webapp.filestorage.impl.FileStorageImplWrapper> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage> .
+               
+:instrumentedSearchEngineWrapper 
+    a   <java:edu.cornell.mannlib.vitro.webapp.searchengine.InstrumentedSearchEngineWrapper> , 
+        <java:edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine> ;
+    :wraps :solrSearchEngine .
+
+:solrSearchEngine
+    a   <java:edu.cornell.mannlib.vitro.webapp.searchengine.solr.SolrSearchEngine> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine> .
+
+:basicSearchIndexer
+    a   <java:edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer> .
+    
+:sdbContentTripleSource
+    a   <java:edu.cornell.mannlib.vitro.webapp.triplesource.impl.sdb.ContentTripleSourceSDB> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource> .
+
+:tdbConfigurationTripleSource
+    a   <java:edu.cornell.mannlib.vitro.webapp.triplesource.impl.tdb.ConfigurationTripleSourceTDB> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource> .
+        
+:jfactTBoxReasonerModule
+    a   <java:edu.cornell.mannlib.vitro.webapp.tboxreasoner.impl.jfact.JFactTBoxReasonerModule> ,
+        <java:edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule> .

webapp/src/edu/cornell/mannlib/vitro/webapp/application/ApplicationImpl.java

@@ -16,13 +16,14 @@ import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus;
 import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage;
 import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
-import edu.cornell.mannlib.vitro.webapp.triplesource.impl.BasicCombinedTripleSource;
 import edu.cornell.mannlib.vitro.webapp.startup.ComponentStartupStatusImpl;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
+import edu.cornell.mannlib.vitro.webapp.triplesource.impl.BasicCombinedTripleSource;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation;
 
@@ -38,6 +39,7 @@ public class ApplicationImpl implements Application {
 	private VitroHomeDirectory homeDirectory;
 
 	private SearchEngine searchEngine;
+	private SearchIndexer searchIndexer;
 	private ImageProcessor imageProcessor;
 	private FileStorage fileStorage;
 	private ContentTripleSource contentTripleSource;
@@ -73,11 +75,27 @@ public class ApplicationImpl implements Application {
 			searchEngine = se;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple SearchEngine instancess: "
+					"Configuration includes multiple SearchEngine instances: "
 							+ searchEngine + ", and " + se);
 		}
 	}
 
+	@Override
+	public SearchIndexer getSearchIndexer() {
+		return searchIndexer;
+	}
+
+	@Property(uri = "http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationSetup#hasSearchIndexer")
+	public void setSearchIndexer(SearchIndexer si) {
+		if (searchIndexer == null) {
+			searchIndexer = si;
+		} else {
+			throw new IllegalStateException(
+					"Configuration includes multiple SearchIndexer instances: "
+							+ searchIndexer + ", and " + si);
+		}
+	}
+
 	@Override
 	public ImageProcessor getImageProcessor() {
 		return imageProcessor;
@@ -89,7 +107,7 @@ public class ApplicationImpl implements Application {
 			imageProcessor = ip;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple ImageProcessor instancess: "
+					"Configuration includes multiple ImageProcessor instances: "
 							+ imageProcessor + ", and " + ip);
 		}
 	}
@@ -105,7 +123,7 @@ public class ApplicationImpl implements Application {
 			fileStorage = fs;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple FileStorage intances: "
+					"Configuration includes multiple FileStorage instances: "
 							+ fileStorage + ", and " + fs);
 		}
 	}
@@ -121,7 +139,7 @@ public class ApplicationImpl implements Application {
 			contentTripleSource = source;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple intances of ContentTripleSource: "
+					"Configuration includes multiple instances of ContentTripleSource: "
 							+ contentTripleSource + ", and " + source);
 		}
 	}
@@ -137,7 +155,7 @@ public class ApplicationImpl implements Application {
 			configurationTripleSource = source;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple intances of ConfigurationTripleSource: "
+					"Configuration includes multiple instances of ConfigurationTripleSource: "
 							+ configurationTripleSource + ", and " + source);
 		}
 	}
@@ -153,7 +171,7 @@ public class ApplicationImpl implements Application {
 			tboxReasonerModule = module;
 		} else {
 			throw new IllegalStateException(
-					"Configuration includes multiple intances of TBoxReasonerModule: "
+					"Configuration includes multiple instances of TBoxReasonerModule: "
 							+ tboxReasonerModule + ", and " + module);
 		}
 	}
@@ -164,6 +182,10 @@ public class ApplicationImpl implements Application {
 			throw new IllegalStateException(
 					"Configuration did not include a SearchEngine.");
 		}
+		if (searchIndexer == null) {
+			throw new IllegalStateException(
+					"Configuration did not include a SearchIndexer.");
+		}
 		if (imageProcessor == null) {
 			throw new IllegalStateException(
 					"Configuration did not include an ImageProcessor.");

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/api/SparqlUpdateApiController.java

@@ -25,11 +25,12 @@ import com.hp.hpl.jena.update.UpdateAction;
 import com.hp.hpl.jena.update.UpdateFactory;
 import com.hp.hpl.jena.update.UpdateRequest;
 
+import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
 import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
 import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest;
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 
 /**
  * Process SPARQL Updates, as an API.
@@ -98,13 +99,14 @@ public class SparqlUpdateApiController extends VitroApiServlet {
 		ServletContext ctx = req.getSession().getServletContext();
 		VitroRequest vreq = new VitroRequest(req);
 
-		IndexBuilder.getBuilder(ctx).pause();
+		SearchIndexer indexer = ApplicationUtils.instance().getSearchIndexer();
+		indexer.pause();
 		try {
 			Dataset ds = new RDFServiceDataset(vreq.getUnfilteredRDFService());
 			GraphStore graphStore = GraphStoreFactory.create(ds);
 			UpdateAction.execute(parsed, graphStore);
 		} finally {
-			IndexBuilder.getBuilder(ctx).unpause();
+			indexer.unpause();
 		}
 	}
 

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/authenticate/BasicAuthenticator.java

@@ -15,6 +15,7 @@ import org.apache.commons.logging.LogFactory;
 
 import edu.cornell.mannlib.vedit.beans.LoginStatusBean;
 import edu.cornell.mannlib.vedit.beans.LoginStatusBean.AuthenticationSource;
+import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
 import edu.cornell.mannlib.vitro.webapp.auth.identifier.RequestIdentifiers;
 import edu.cornell.mannlib.vitro.webapp.auth.identifier.common.IsRootUser;
 import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel;
@@ -29,7 +30,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.LoginEvent;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.LogoutEvent;
 import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
+import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
+import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
 
 /**
  * The "standard" implementation of Authenticator.
@@ -162,7 +164,17 @@ public class BasicAuthenticator extends Authenticator {
 
 		if (IsRootUser.isRootUser(RequestIdentifiers
 				.getIdBundleForRequest(request))) {
-			IndexBuilder.checkIndexOnRootLogin(request);
+			try {
+				SearchEngine engine = ApplicationUtils.instance()
+						.getSearchEngine();
+				if (engine.documentCount() == 0) {
+					log.info("Search index is empty. Running a full index rebuild.");
+					ApplicationUtils.instance().getSearchIndexer()
+							.rebuildIndex();
+				}
+			} catch (SearchEngineException e) {
+				log.warn("Unable to check for search index", e);
+			}
 		}
 	}
 
@@ -262,7 +274,8 @@ public class BasicAuthenticator extends Authenticator {
 	 * Get a reference to the UserAccountsDao, or null.
 	 */
 	private UserAccountsDao getUserAccountsDao() {
-		UserAccountsDao userAccountsDao = getWebappDaoFactory().getUserAccountsDao();
+		UserAccountsDao userAccountsDao = getWebappDaoFactory()
+				.getUserAccountsDao();
 		if (userAccountsDao == null) {
 			log.error("getUserAccountsDao: no UserAccountsDao");
 		}

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java

@@ -40,9 +40,9 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event;
 import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
 import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
 
@@ -64,7 +64,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
  * search index is not built or if there were problems building the index,
  * the class counts from VClassGroupCache will be incorrect.
  */
-public class VClassGroupCache implements IndexingEventListener {
+public class VClassGroupCache implements SearchIndexer.Listener {
     private static final Log log = LogFactory.getLog(VClassGroupCache.class);
 
     private static final String ATTRIBUTE_NAME = "VClassGroupCache";
@@ -208,19 +208,17 @@ public class VClassGroupCache implements IndexingEventListener {
      * Handle notification of events from the IndexBuilder.
      */
     @Override
-    public void notifyOfIndexingEvent(EventTypes event) {
-        switch( event ){
-            case FINISH_FULL_REBUILD: 
-            case FINISHED_UPDATE:
-                log.debug("rebuilding because of IndexBuilder " + event.name());
-                requestCacheUpdate();
-                break;            
-            default: 
-                log.debug("ignoring event type " + event.name());
-                break;
-                    
-        }        
-    }
+	public void receiveSearchIndexerEvent(Event event) {
+    	switch (event.getType()) {
+    	case STOP_PROCESSING_URIS:
+            log.debug("rebuilding because of IndexBuilder " + event.getType());
+            requestCacheUpdate();
+            break;            
+        default: 
+            log.debug("ignoring event type " + event.getType());
+            break;
+    	}
+	}
     
     /* **************** static utility methods ***************** */
     
@@ -489,8 +487,8 @@ public class VClassGroupCache implements IndexingEventListener {
             context.setAttribute(ATTRIBUTE_NAME,vcgc);           
             log.info("VClassGroupCache added to context");   
             
-            IndexBuilder indexBuilder = IndexBuilder.getBuilder(context);
-            indexBuilder.addIndexBuilderListener(vcgc);
+            SearchIndexer searchIndexer = ApplicationUtils.instance().getSearchIndexer();
+            searchIndexer.addListener(vcgc);
             log.info("VClassGroupCache set to listen to events from IndexBuilder");
         }
 

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/Application.java

@@ -8,6 +8,7 @@ import edu.cornell.mannlib.vitro.webapp.application.VitroHomeDirectory;
 import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage;
 import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource;
@@ -22,6 +23,8 @@ public interface Application {
 
 	SearchEngine getSearchEngine();
 
+	SearchIndexer getSearchIndexer();
+
 	ImageProcessor getImageProcessor();
 
 	FileStorage getFileStorage();
@@ -39,8 +42,16 @@ public interface Application {
 			NEW, ACTIVE, STOPPED
 		}
 
+		/**
+		 * This should be called only once, and should be the first call on this
+		 * Component.
+		 */
 		void startup(Application application, ComponentStartupStatus ss);
 
+		/**
+		 * This should be called only once, and should be the last call on this
+		 * Component.
+		 */
 		void shutdown(Application application);
 	}
 

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java

@@ -0,0 +1,147 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.webapp.modules.searchIndexer;
+
+import java.util.Collection;
+
+import edu.cornell.mannlib.vitro.webapp.modules.Application;
+
+/**
+ * Interface for the code that controls the contents of the search index.
+ * 
+ * The only calls that are valid after shutdown are shutdown(), getStatus() and
+ * removeListener().
+ */
+public interface SearchIndexer extends Application.Module {
+	/**
+	 * Update the search documents for these URIs.
+	 * 
+	 * For each URI that belongs in the index, a new search document is built,
+	 * replacing any document that may already exist for that URI. For each URI
+	 * that does not belong in the index, any existing document is removed.
+	 * 
+	 * A URI belongs in the index if it refers to an existing individual in the
+	 * model, and is not excluded.
+	 * 
+	 * @param uris
+	 *            if null or empty, this call has no effect.
+	 * @throws IllegalStateException
+	 *             if called after shutdown()
+	 */
+	void scheduleUpdatesForUris(Collection<String> uris);
+
+	/**
+	 * Remove all of the existing documents in the index, and schedule updates
+	 * for all of the individuals in the model.
+	 * 
+	 * If a rebuild is already pending or in progress, this method has no
+	 * effect.
+	 * 
+	 * @throws IllegalStateException
+	 *             if called after shutdown()
+	 */
+	void rebuildIndex();
+
+	/**
+	 * Stop processing new tasks. Requests will be queued until a call to
+	 * unpause().
+	 * 
+	 * @throws IllegalStateException
+	 *             if called after shutdown()
+	 */
+	void pause();
+
+	/**
+	 * Resume processing new tasks. Any requests that were received since the
+	 * call to pause() will now be scheduled for processing.
+	 * 
+	 * Has no effect if called after shutdown().
+	 */
+	void unpause();
+
+	/**
+	 * What is the current status of the indexer?
+	 * 
+	 * Still valid after shutdown().
+	 */
+	SearchIndexerStatus getStatus();
+
+	/**
+	 * Add this listener, allowing it to receive events from the indexer. If
+	 * this listener has already been added, this method has no effect.
+	 * 
+	 * @param listener
+	 *            if null, this method has no effect.
+	 * @throws IllegalStateException
+	 *             if called after shutdown()
+	 */
+	void addListener(Listener listener);
+
+	/**
+	 * Remove this listener, meaning that it will no longer receive events from
+	 * the indexer. If this listener is not active, this method has no effect.
+	 * 
+	 * Has no effect if called after shutdown().
+	 * 
+	 * @param listener
+	 *            if null, this method has no effect.
+	 */
+	void removeListener(Listener listener);
+
+	/**
+	 * Stop processing and release resources. This call should block until the
+	 * dependent threads are stopped.
+	 * 
+	 * Repeated calls have no effect.
+	 */
+	@Override
+	void shutdown(Application app);
+
+	/**
+	 * A listener that will be notified of events from the SearchIndexer.
+	 */
+	public static interface Listener {
+		void receiveSearchIndexerEvent(Event event);
+	}
+
+	/**
+	 * An immutable event object. The event type describes just what happened.
+	 * The status object describes what the indexer is doing now.
+	 */
+	public static class Event {
+		public enum Type {
+			START_PROCESSING_URIS,
+
+			PROGRESS_PROCESSING_URIS,
+
+			STOP_PROCESSING_URIS,
+
+			START_PROCESSING_STATEMENTS,
+
+			PROGRESS_PROCESSING_STATEMENTS,
+
+			STOP_PROCESSING_STATEMENTS,
+
+			REBUILD_REQUESTED,
+
+			SHUTDOWN_REQUESTED
+		}
+
+		private final Type type;
+		private final SearchIndexerStatus status;
+
+		public Event(Type type, SearchIndexerStatus status) {
+			this.type = type;
+			this.status = status;
+		}
+
+		public Type getType() {
+			return type;
+		}
+
+		public SearchIndexerStatus getStatus() {
+			return status;
+		}
+	}
+
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java

@@ -0,0 +1,156 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.webapp.modules.searchIndexer;
+
+import java.util.Date;
+
+/**
+ * An immutable summary of the status of the SearchIndexer, at some point in
+ * time.Contains the current state, and some counts.
+ * 
+ * If the indexer is processing URIs, processing statements, or preparing a
+ * rebuild, the counts are URI_COUNTS, STATEMENT_COUNTS, or REBUILD_COUNTS.
+ * 
+ * When the indexer starts up, and when it is is shut down, the counts are
+ * NO_COUNTS.
+ * 
+ * If the indexer is idle, the counts are carried over from the previous
+ * operation.
+ */
+public class SearchIndexerStatus {
+	public enum State {
+		IDLE, PROCESSING_URIS, PROCESSING_STMTS, PREPARING_REBUILD, SHUTDOWN
+	}
+
+	private final State state;
+	private final Date since;
+	private final Counts counts;
+
+	public SearchIndexerStatus(State state, Date since, Counts counts) {
+		this.state = state;
+		this.since = since;
+		this.counts = counts;
+	}
+
+	public State getState() {
+		return state;
+	}
+
+	public Date getSince() {
+		return since;
+	}
+	
+	public Counts getCounts() {
+		return counts;
+	}
+
+	public abstract static class Counts {
+		public enum Type {
+			URI_COUNTS, STATEMENT_COUNTS, REBUILD_COUNTS, NO_COUNTS
+		}
+
+		private final Type type;
+
+		public Counts(Type type) {
+			this.type = type;
+		}
+
+		public Type getType() {
+			return this.type;
+		}
+
+		public UriCounts asUriCounts() {
+			return (UriCounts) this;
+		}
+
+		public StatementCounts asStatementCounts() {
+			return (StatementCounts) this;
+		}
+
+		public RebuildCounts asRebuildCounts() {
+			return (RebuildCounts) this;
+		}
+
+		public NoCounts asNoCounts() {
+			return (NoCounts) this;
+		}
+	}
+
+	public static class UriCounts extends Counts {
+		private final int deleted;
+		private final int updated;
+		private final int remaining;
+		private final int total;
+
+		public UriCounts(int deleted, int updated, int remaining, int total) {
+			super(Type.URI_COUNTS);
+			this.deleted = deleted;
+			this.updated = updated;
+			this.remaining = remaining;
+			this.total = total;
+		}
+
+		public int getDeleted() {
+			return deleted;
+		}
+
+		public int getUpdated() {
+			return updated;
+		}
+
+		public int getRemaining() {
+			return remaining;
+		}
+
+		public int getTotal() {
+			return total;
+		}
+
+	}
+
+	public static class StatementCounts extends Counts {
+		private final int processed;
+		private final int remaining;
+		private final int total;
+
+		public StatementCounts(int processed, int remaining, int total) {
+			super(Type.STATEMENT_COUNTS);
+			this.processed = processed;
+			this.remaining = remaining;
+			this.total = total;
+		}
+
+		public int getProcessed() {
+			return processed;
+		}
+
+		public int getRemaining() {
+			return remaining;
+		}
+
+		public int getTotal() {
+			return total;
+		}
+
+	}
+
+	public static class RebuildCounts extends Counts {
+		private final int numberOfIndividuals;
+
+		public RebuildCounts(int numberOfIndividuals) {
+			super(Type.REBUILD_COUNTS);
+			this.numberOfIndividuals = numberOfIndividuals;
+		}
+
+		public int getNumberOfIndividuals() {
+			return numberOfIndividuals;
+		}
+
+	}
+
+	public static class NoCounts extends Counts {
+		public NoCounts() {
+			super(Type.NO_COUNTS);
+		}
+	}
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java

@@ -3,12 +3,10 @@
 package edu.cornell.mannlib.vitro.webapp.search.controller;
 
 import java.io.IOException;
-import java.util.Collection;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.Map;
 
-import javax.servlet.ServletContext;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
@@ -18,31 +16,30 @@ import org.apache.commons.logging.LogFactory;
 
 import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
 import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
-import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest;
 import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.RequestedAction;
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
-import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
-import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.RedirectResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
-import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel;
-import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.StatementCounts;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.UriCounts;
+import edu.cornell.mannlib.vitro.webapp.services.freemarker.FreemarkerProcessingServiceSetup;
 
 /**
- * Accepts requests to rebuild or update the search index. It uses an
- * IndexBuilder and finds that IndexBuilder from the servletContext using the
- * key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder"
+ * Accepts requests to display the current status of the search index, or to
+ * initiate a rebuild.
  * 
- * That IndexBuilder will be associated with a object that implements the
- * IndexerIface.
+ * A DISPLAY or REBUILD request is handled like any other FreemarkerHttpServlet.
+ * A STATUS is an AJAX request, we override doGet() so we can format the
+ * template without enclosing it in a body template.
  * 
- * An example of the IndexerIface is SearchIndexer. An example of the IndexBuilder
- * and SearchIndexer setup is in SearchIndexerSetup.
- * 
- * @author bdc34
+ * When initialized, this servlet adds a listener to the SearchIndexer, so it
+ * can maintain a history of activity. This will provide the contents of the
+ * display.
  */
 public class IndexController extends FreemarkerHttpServlet {
 	private static final Log log = LogFactory.getLog(IndexController.class);
@@ -50,27 +47,22 @@ public class IndexController extends FreemarkerHttpServlet {
 	/**
 	 * <pre>
 	 * This request might be:
-	 * SETUP -- Index is not building and nothing is requested. Solicit requests.
-	 * REFRESH  -- Index is building, nothing is requested. Show continuing status.
-	 * REBUILD -- Rebuild is requested. Set the rebuild flag and show continuing status.
+	 * DISPLAY (default) -- Send the template that will contain the status display. 
+	 * STATUS  -- Send the current status and history.
+	 * REBUILD -- Initiate a rebuild. Then act like DISPLAY.
 	 * </pre>
 	 */
 	private enum RequestType {
-		SETUP, REFRESH, REBUILD;
+		DISPLAY, STATUS, REBUILD;
 
 		/** What type of request is this? */
 		static RequestType fromRequest(HttpServletRequest req) {
 			if (hasParameter(req, "rebuild")) {
 				return REBUILD;
+			} else if (hasParameter(req, "status")) {
+				return STATUS;
 			} else {
-				ServletContext ctx = req.getSession().getServletContext();
-				IndexBuilder builder = IndexBuilder.getBuilder(ctx);
-				WorkLevelStamp workLevel = builder.getWorkLevel();
-				if (workLevel.getLevel() == WorkLevel.WORKING) {
-					return REFRESH;
-				} else {
-					return SETUP;
-				}
+				return DISPLAY;
 			}
 		}
 
@@ -81,13 +73,43 @@ public class IndexController extends FreemarkerHttpServlet {
 	}
 
 	private static final String PAGE_URL = "/SearchIndex";
-	private static final String TEMPLATE_NAME = "searchIndex.ftl";
+	private static final String PAGE_TEMPLATE_NAME = "searchIndex.ftl";
+	private static final String STATUS_TEMPLATE_NAME = "searchIndexStatus.ftl";
 
 	public static final RequestedAction REQUIRED_ACTIONS = SimplePermission.MANAGE_SEARCH_INDEX.ACTION;
 
+	private SearchIndexer indexer;
+	private IndexHistory history;
+
 	@Override
-	protected AuthorizationRequest requiredActions(VitroRequest vreq) {
-		return REQUIRED_ACTIONS;
+	public void init() throws ServletException {
+		super.init();
+		this.indexer = ApplicationUtils.instance().getSearchIndexer();
+		this.history = new IndexHistory();
+		this.indexer.addListener(this.history);
+	}
+
+	@Override
+	public void destroy() {
+		this.indexer.removeListener(this.history);
+		super.destroy();
+	}
+
+	@Override
+	public void doGet(HttpServletRequest req, HttpServletResponse resp)
+			throws IOException, ServletException {
+		if (!isAuthorizedToDisplayPage(req, resp, REQUIRED_ACTIONS)) {
+			return;
+		}
+
+		switch (RequestType.fromRequest(req)) {
+		case STATUS:
+			showStatus(req, resp);
+			break;
+		default:
+			super.doGet(req, resp);
+			break;
+		}
 	}
 
 	@Override
@@ -95,78 +117,82 @@ public class IndexController extends FreemarkerHttpServlet {
 		return "Rebuild Search Index";
 	}
 
-	@Override
-	public void doGet(HttpServletRequest req, HttpServletResponse resp)
-			throws IOException, ServletException {
-		if (RequestType.fromRequest(req) == RequestType.REFRESH) {
-			resp.addHeader("Refresh", "5; " + UrlBuilder.getUrl(PAGE_URL));
-		}
-		super.doGet(req, resp);
-	}
-
 	@Override
 	protected ResponseValues processRequest(VitroRequest vreq) {
-		Map<String, Object> body = new HashMap<String, Object>();
-		body.put("actionUrl", UrlBuilder.getUrl(PAGE_URL));
-
-		try {
-			IndexBuilder builder = IndexBuilder.getBuilder(getServletContext());
-
-			switch (RequestType.fromRequest(vreq)) {
-			case REBUILD:
-				builder.doIndexRebuild();
-				Thread.sleep(500);
-				return redirectToRefresh();
-			default:
-				return showCurrentStatus(builder, body);
-			}
-		} catch (Exception e) {
-			log.error("Error rebuilding search index", e);
-			body.put("errorMessage",
-					"There was an error while rebuilding the search index. "
-							+ e.getMessage());
-			return new ExceptionResponseValues(
-					Template.ERROR_MESSAGE.toString(), body, e);
+		switch (RequestType.fromRequest(vreq)) {
+		case REBUILD:
+			requestRebuild();
+			return showDisplay();
+		default:
+			return showDisplay();
 		}
 	}
 
-	private ResponseValues redirectToRefresh() {
-		return new RedirectResponseValues(PAGE_URL);
+	private ResponseValues showDisplay() {
+		HashMap<String, Object> body = new HashMap<>();
+		body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true"));
+		body.put("rebuildUrl", UrlBuilder.getUrl(PAGE_URL, "rebuild", "true"));
+		return new TemplateResponseValues(PAGE_TEMPLATE_NAME, body);
 	}
 
-	private ResponseValues showCurrentStatus(IndexBuilder builder,
-			Map<String, Object> body) {
-		WorkLevelStamp stamp = builder.getWorkLevel();
-
-		WorkLevel workLevel = stamp.getLevel();
-		long completedCount = builder.getCompletedCount();
-		long totalToDo = builder.getTotalToDo();
-		Date since = stamp.getSince();
-		Date expectedCompletion = figureExpectedCompletion(since, totalToDo,
-				completedCount);
-
-		body.put("worklevel", workLevel.toString());
-		body.put("completedCount", completedCount);
-		body.put("totalToDo", totalToDo);
-		body.put("currentTask", figureCurrentTask(stamp.getFlags()));
-		body.put("since", since);
-		body.put("elapsed", formatElapsedTime(since, new Date()));
-		body.put("expected", formatElapsedTime(since, expectedCompletion));
-		body.put("hasPreviousBuild", since.getTime() > 0L);
-		body.put("indexIsConnected", testIndexConnection());
-		return new TemplateResponseValues(TEMPLATE_NAME, body);
-	}
-
-	private Boolean testIndexConnection() {
+	private void showStatus(HttpServletRequest req, HttpServletResponse resp)
+			throws IOException {
 		try {
-			ApplicationUtils.instance().getSearchEngine().ping();
-			return Boolean.TRUE;
+			Map<String, Object> body = new HashMap<>();
+			body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true"));
+			body.put("rebuildUrl",
+					UrlBuilder.getUrl(PAGE_URL, "rebuild", "true"));
+			body.put("status", buildStatusMap(indexer.getStatus()));
+			body.put("history", history.toMaps());
+
+			String rendered = FreemarkerProcessingServiceSetup.getService(
+					getServletContext()).renderTemplate(STATUS_TEMPLATE_NAME,
+					body, req);
+			resp.getWriter().write(rendered);
 		} catch (Exception e) {
-			log.error("Can't connect to the search engine.", e);
-			return Boolean.FALSE;
+			resp.setStatus(500);
+			resp.getWriter().write(e.toString());
+			log.error(e, e);
 		}
 	}
 
+	private void requestRebuild() {
+		indexer.rebuildIndex();
+	}
+
+	private Map<String, Object> buildStatusMap(SearchIndexerStatus status) {
+		Map<String, Object> map = new HashMap<>();
+		State state = status.getState();
+		map.put("statusType", state);
+		map.put("since", status.getSince());
+
+		if (state == State.PROCESSING_URIS) {
+			UriCounts counts = status.getCounts().asUriCounts();
+			map.put("updated", counts.getUpdated());
+			map.put("deleted", counts.getDeleted());
+			map.put("remaining", counts.getRemaining());
+			map.put("total", counts.getTotal());
+			map.put("elapsed", breakDownElapsedTime(status.getSince()));
+			map.put("expectedCompletion",
+					figureExpectedCompletion(status.getSince(),
+							counts.getTotal(),
+							counts.getTotal() - counts.getRemaining()));
+		} else if (state == State.PROCESSING_STMTS) {
+			StatementCounts counts = status.getCounts().asStatementCounts();
+			map.put("processed", counts.getProcessed());
+			map.put("remaining", counts.getRemaining());
+			map.put("total", counts.getTotal());
+			map.put("elapsed", breakDownElapsedTime(status.getSince()));
+			map.put("expectedCompletion",
+					figureExpectedCompletion(status.getSince(),
+							counts.getTotal(), counts.getProcessed()));
+		} else {
+			// nothing for IDLE or SHUTDOWN, except what's already there.
+		}
+
+		return map;
+	}
+
 	private Date figureExpectedCompletion(Date startTime, long totalToDo,
 			long completedCount) {
 		Date now = new Date();
@@ -186,22 +212,12 @@ public class IndexController extends FreemarkerHttpServlet {
 		return new Date(expectedDuration + startTime.getTime());
 	}
 
-	private String formatElapsedTime(Date since, Date until) {
-		long elapsedMillis = until.getTime() - since.getTime();
+	private int[] breakDownElapsedTime(Date since) {
+		long elapsedMillis = new Date().getTime() - since.getTime();
 		long seconds = (elapsedMillis / 1000L) % 60L;
 		long minutes = (elapsedMillis / 60000L) % 60L;
 		long hours = elapsedMillis / 3600000L;
-		return String.format("%02d:%02d:%02d", hours, minutes, seconds);
-	}
-
-	private String figureCurrentTask(Collection<String> flags) {
-		if (flags.contains(IndexBuilder.FLAG_REBUILDING)) {
-			return "Rebuilding";
-		} else if (flags.contains(IndexBuilder.FLAG_UPDATING)) {
-			return "Updating";
-		} else {
-			return "Not working on";
-		}
+		return new int[] {(int) hours, (int) minutes, (int) seconds};
 	}
 
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java

@@ -0,0 +1,95 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.webapp.search.controller;
+
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.Counts;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.Counts.Type;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.RebuildCounts;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.StatementCounts;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.UriCounts;
+
+/**
+ * This listener keeps a list of the last several SearchIndexer events, and will
+ * format them for display in a Freemarker template.
+ */
+public class IndexHistory implements SearchIndexer.Listener {
+	private final static int MAX_EVENTS = 10;
+
+	private final Deque<Event> events = new LinkedList<>();
+
+	@Override
+	public void receiveSearchIndexerEvent(Event event) {
+		synchronized (events) {
+			events.addFirst(event);
+			while (events.size() > MAX_EVENTS) {
+				events.removeLast();
+			}
+		}
+	}
+
+	public List<Map<String, Object>> toMaps() {
+		synchronized (events) {
+			List<Map<String, Object>> list = new ArrayList<>();
+			for (Event event : events) {
+				list.add(toMap(event));
+			}
+			return list;
+		}
+	}
+
+	private Map<String, Object> toMap(Event event) {
+		SearchIndexerStatus status = event.getStatus();
+		Counts counts = status.getCounts();
+		Type countsType = counts.getType();
+
+		Map<String, Object> map = new HashMap<>();
+		map.put("event", event.getType());
+		map.put("statusType", status.getState());
+		map.put("since", status.getSince());
+		map.put("countsType", countsType);
+
+		switch (countsType) {
+		case URI_COUNTS:
+			addCounts(counts.asUriCounts(), map);
+			break;
+		case STATEMENT_COUNTS:
+			addCounts(counts.asStatementCounts(), map);
+			break;
+		case REBUILD_COUNTS:
+			addCounts(counts.asRebuildCounts(), map);
+			break;
+		default: // NO_COUNTS
+			break;
+		}
+
+		return map;
+	}
+
+	private void addCounts(UriCounts counts, Map<String, Object> map) {
+		map.put("updated", counts.getUpdated());
+		map.put("deleted", counts.getDeleted());
+		map.put("remaining", counts.getRemaining());
+		map.put("total", counts.getTotal());
+	}
+
+	private void addCounts(StatementCounts counts, Map<String, Object> map) {
+		map.put("processed", counts.getProcessed());
+		map.put("remaining", counts.getRemaining());
+		map.put("total", counts.getTotal());
+	}
+
+	private void addCounts(RebuildCounts counts, Map<String, Object> map) {
+		map.put("numberOfIndividuals", counts.getNumberOfIndividuals());
+	}
+
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java

@@ -15,6 +15,7 @@ import javax.servlet.http.HttpServletRequest;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
 import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
 import edu.cornell.mannlib.vitro.webapp.auth.policy.PolicyHelper;
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@@ -23,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServ
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 
 /**
  * Accepts requests to update a set of URIs in the search index.
@@ -73,8 +74,8 @@ public class SearchServiceController extends FreemarkerHttpServlet {
 	 */
 	private ResponseValues doUpdateUrisInSearch(HttpServletRequest req)
 			throws IOException, ServletException {
-		IndexBuilder builder = IndexBuilder.getBuilder(getServletContext());
-		int uriCount = new UpdateUrisInIndex().doUpdateUris(req, builder);
+		SearchIndexer indexer = ApplicationUtils.instance().getSearchIndexer();
+		int uriCount = new UpdateUrisInIndex().doUpdateUris(req, indexer);
 
 		Map<String, Object> body = new HashMap<>();
 		body.put("msg", "Received " + uriCount + " URIs.");

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java

@@ -6,6 +6,7 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.Scanner;
@@ -20,7 +21,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
-import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 
 /**
  * Class that performs the update of the uris in the search index for the
@@ -38,7 +39,7 @@ public class UpdateUrisInIndex {
 	 * 
 	 * @throws IOException
 	 */
-	protected int doUpdateUris(HttpServletRequest req, IndexBuilder builder)
+	protected int doUpdateUris(HttpServletRequest req, SearchIndexer indexer)
 			throws ServletException, IOException {
 		Map<String, List<FileItem>> map = new VitroRequest(req).getFiles();
 		if (map == null) {
@@ -51,25 +52,25 @@ public class UpdateUrisInIndex {
 		for (String name : map.keySet()) {
 			for (FileItem item : map.get(name)) {
 				log.debug("Found " + item.getSize() + " byte file for '" + name + "'");
-				uriCount += processFileItem(builder, item, enc);
+				uriCount += processFileItem(indexer, item, enc);
 			}
 		}
 		return uriCount;
 	}
 
-	private int processFileItem(IndexBuilder builder, 
+	private int processFileItem(SearchIndexer indexer, 
 			FileItem item, Charset enc) throws IOException {
-		int count = 0;
+		List<String> uris = new ArrayList<>();
 		Reader reader = new InputStreamReader(item.getInputStream(), enc.name());
 		try (Scanner scanner = createScanner(reader)) {
 			while (scanner.hasNext()) {
 				String uri = scanner.next();
 				log.debug("Request to index uri '" + uri + "'");
-				builder.addToChanged(uri);
-				count++;
+				uris.add(uri);
 			}
 		}
-		return count;
+		indexer.scheduleUpdatesForUris(uris);
+		return uris.size();
 	}
 
 	@SuppressWarnings("resource")

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerImpl.java

@@ -0,0 +1,220 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+package edu.cornell.mannlib.vitro.webapp.searchindex;
+
+import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.REBUILD_REQUESTED;
+import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_STATEMENTS;
+import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_STATEMENTS;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+
+import javax.servlet.ServletContext;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import edu.cornell.mannlib.vitro.webapp.modules.Application;
+import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State;
+import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
+import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener;
+import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel;
+import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp;
+
+/**
+ * TODO
+ * A silly implementation that just wraps the old IndexBuilder.
+ */
+public class SearchIndexerImpl implements SearchIndexer {
+	private static final Log log = LogFactory.getLog(SearchIndexerImpl.class);
+
+	private final ListenerList listeners = new ListenerList();
+
+	private ServletContext ctx;
+	// private Set<SearchIndexExcluder> excluders;
+	// private Set<DocumentModifier> modifiers;
+	// private Set<IndexingUriFinder> uriFinders;
+
+	// TODO
+	private IndexBuilder indexBuilder;
+
+	@Override
+	public void startup(Application application, ComponentStartupStatus ss) {
+		try {
+			this.ctx = application.getServletContext();
+			// loadConfiguration();
+			// ss.info("Configured SearchIndexer: excluders=" + excluders
+			// + ", modifiers=" + modifiers + ", uriFinders=" + uriFinders);
+
+			// TODO
+			this.indexBuilder = (IndexBuilder) ctx
+					.getAttribute(IndexBuilder.class.getName());
+
+			this.indexBuilder.addIndexBuilderListener(new BridgeListener());
+		} catch (Exception e) {
+			ss.fatal("Failed to configure the SearchIndexer", e);
+		}
+	}
+
+	private void createAndFire(Event.Type type) {
+		listeners.fireEvent(new Event(type, getStatus()));
+	}
+
+	// private void loadConfiguration() throws ConfigurationBeanLoaderException
+	// {
+	// ConfigurationBeanLoader beanLoader = new ConfigurationBeanLoader(
+	// ModelAccess.on(ctx).getOntModel(DISPLAY), ctx);
+	// excluders = beanLoader.loadAll(SearchIndexExcluder.class);
+	// modifiers = beanLoader.loadAll(DocumentModifier.class);
+	// uriFinders = beanLoader.loadAll(IndexingUriFinder.class);
+	// }
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.Application.Component#shutdown
+	 * (edu.cornell.mannlib.vitro.webapp.modules.Application)
+	 */
+	@Override
+	public void shutdown(Application application) {
+		// TODO
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer#
+	 * scheduleUpdatesForUris(java.util.Collection)
+	 */
+	@Override
+	public void scheduleUpdatesForUris(Collection<String> uris) {
+		// TODO
+		for (String uri : uris) {
+			indexBuilder.addToChanged(uri);
+		}
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer#
+	 * rebuildIndex()
+	 */
+	@Override
+	public void rebuildIndex() {
+		// TODO
+		indexBuilder.doIndexRebuild();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer#
+	 * pause()
+	 */
+	@Override
+	public void pause() {
+		// TODO
+		indexBuilder.pause();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer#
+	 * unpause()
+	 */
+	@Override
+	public void unpause() {
+		// TODO
+		indexBuilder.unpause();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer#
+	 * getStatus()
+	 */
+	@Override
+	public SearchIndexerStatus getStatus() {
+		// TODO
+		WorkLevelStamp workLevel = indexBuilder.getWorkLevel();
+		WorkLevel level = workLevel.getLevel();
+		Date since = workLevel.getSince();
+		if (level == WorkLevel.IDLE) {
+			return new SearchIndexerStatus(State.IDLE, since,
+					new SearchIndexerStatus.NoCounts());
+		} else {
+			return new SearchIndexerStatus(State.PROCESSING_URIS, since,
+					new SearchIndexerStatus.UriCounts(1, 2, 3, 6));
+		}
+	}
+
+	@Override
+	public void addListener(Listener listener) {
+		listeners.add(listener);
+	}
+
+	@Override
+	public void removeListener(Listener listener) {
+		listeners.remove(listener);
+	}
+
+	private static class ListenerList {
+		private final List<Listener> list;
+
+		public ListenerList() {
+			list = Collections.synchronizedList(new ArrayList<Listener>());
+		}
+
+		public void add(Listener l) {
+			list.add(l);
+		}
+
+		public void remove(Listener l) {
+			list.remove(l);
+		}
+
+		public void fireEvent(Event e) {
+			synchronized (list) {
+				for (Listener l : list) {
+					l.receiveSearchIndexerEvent(e);
+				}
+			}
+		}
+	}
+
+	private class BridgeListener implements IndexingEventListener {
+		@Override
+		public void notifyOfIndexingEvent(EventTypes ie) {
+			switch (ie) {
+			case START_UPDATE:
+				createAndFire(START_PROCESSING_STATEMENTS);
+				break;
+			case FINISHED_UPDATE:
+				createAndFire(STOP_PROCESSING_STATEMENTS);
+				break;
+			case START_FULL_REBUILD:
+				createAndFire(REBUILD_REQUESTED);
+				createAndFire(START_PROCESSING_STATEMENTS);
+				break;
+			default: // FINISH_FULL_REBUILD
+				createAndFire(STOP_PROCESSING_STATEMENTS);
+				break;
+			}
+		}
+	}
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java

@@ -35,6 +35,7 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding.DocumentMod
 import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExcluder;
 import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.AdditionalUriFinders;
 import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.StatementToURIsToUpdate;
+import edu.cornell.mannlib.vitro.webapp.startup.ComponentStartupStatusImpl;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.ConfigurationBeanLoader;
 import edu.cornell.mannlib.vitro.webapp.utils.configuration.ConfigurationBeanLoaderException;
@@ -43,6 +44,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.developer.listeners.DeveloperDisab
 
 /**
  * TODO
+ * A silly implementation that just wraps the old IndexBuilder with a new SearchIndexerImpl.
  */
 public class SearchIndexerSetup implements ServletContextListener {
 	private static final Log log = LogFactory.getLog(SearchIndexerSetup.class);
@@ -96,6 +98,8 @@ public class SearchIndexerSetup implements ServletContextListener {
 							Key.SEARCH_INDEX_SUPPRESS_MODEL_CHANGE_LISTENER));
 
 			ss.info(this, "Setup of search indexer completed.");
+			
+			ApplicationUtils.instance().getSearchIndexer().startup(ApplicationUtils.instance(), new ComponentStartupStatusImpl(this, ss));
 		} catch (Throwable e) {
 			ss.fatal(this, "could not setup search engine", e);
 		}

webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modelaccess/ContextModelAccessStub.java

@@ -4,6 +4,7 @@ package stubs.edu.cornell.mannlib.vitro.webapp.modelaccess;
 
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.ReasoningOption.ASSERTIONS_AND_INFERENCES;
 import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService.CONTENT;
+import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames.FULL_UNION;
 
 import java.util.EnumMap;
 import java.util.HashMap;
@@ -18,6 +19,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
 import edu.cornell.mannlib.vitro.webapp.modelaccess.ContextModelAccess;
 import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.ReasoningOption;
 import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService;
+import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 
 /**
@@ -29,10 +31,14 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 public class ContextModelAccessStub implements ContextModelAccess {
 	// ----------------------------------------------------------------------
 	// Stub infrastructure
+	//
+	// Warning: ontModelMap and rdfServiceMap are not connected, so it's up to
+	// the user to insure that they are consistent with each other.
 	// ----------------------------------------------------------------------
 
 	private final Map<ReasoningOption, WebappDaoFactory> wadfMap = new HashMap<>();
 	private final Map<WhichService, RDFService> rdfServiceMap = new EnumMap<>(WhichService.class);
+	private final Map<String, OntModel> ontModelMap = new HashMap<>();
 
 	public void setWebappDaoFactory(WebappDaoFactory wadf) {
 		setWebappDaoFactory(wadf, ASSERTIONS_AND_INFERENCES);
@@ -47,6 +53,10 @@ public class ContextModelAccessStub implements ContextModelAccess {
 		rdfServiceMap.put(which, rdfService);
 	}
 	
+	public void setOntModel(String name, OntModel model) {
+		ontModelMap.put(name, model);
+	}
+
 	// ----------------------------------------------------------------------
 	// Stub methods
 	// ----------------------------------------------------------------------
@@ -66,6 +76,16 @@ public class ContextModelAccessStub implements ContextModelAccess {
 		return rdfServiceMap.get(which);
 	}
 	
+	@Override
+	public OntModel getOntModel() {
+		return getOntModel(FULL_UNION);
+	}
+
+	@Override
+	public OntModel getOntModel(String name) {
+		return ontModelMap.get(name);
+	}
+
 	// ----------------------------------------------------------------------
 	// Un-implemented methods
 	// ----------------------------------------------------------------------
@@ -94,18 +114,6 @@ public class ContextModelAccessStub implements ContextModelAccess {
 				"ContextModelAccessStub.getModelMaker() not implemented.");
 	}
 
-	@Override
-	public OntModel getOntModel() {
-		throw new RuntimeException(
-				"ContextModelAccessStub.getOntModel() not implemented.");
-	}
-
-	@Override
-	public OntModel getOntModel(String name) {
-		throw new RuntimeException(
-				"ContextModelAccessStub.getOntModel() not implemented.");
-	}
-
 	@Override
 	public OntModelSelector getOntModelSelector() {
 		throw new RuntimeException(

webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/ApplicationStub.java

@@ -12,6 +12,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.Application;
 import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage;
 import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor;
 import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
+import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
 import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource;
 import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource;
@@ -104,9 +105,14 @@ public class ApplicationStub implements Application {
 
 	@Override
 	public TBoxReasonerModule getTBoxReasonerModule() {
-		// TODO Auto-generated method stub
-		throw new RuntimeException("ApplicationStub.getTBoxReasonerModule() not implemented.");
+		throw new RuntimeException(
+				"ApplicationStub.getTBoxReasonerModule() not implemented.");
+	}
 
+	@Override
+	public SearchIndexer getSearchIndexer() {
+		throw new RuntimeException(
+				"Application.getSearchIndexer() not implemented.");
 	}
 
 }

webapp/web/css/search/searchIndex.css

@@ -0,0 +1,17 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+/* Styles for the search index controller. */
+
+section#indexer table.history {
+    font-size: smaller;
+    border: 1px solid gray;
+    width: 100%;
+}
+
+section#indexer table.history th{
+    font-weight: bolder;
+}
+
+section#indexer table.history td{
+    padding: 2px 5px 2px 5px;
+}

webapp/web/js/search/searchIndex.js

@@ -0,0 +1,22 @@
+/* $This file is distributed under the terms of the license in /doc/license.txt$ */
+
+/*
+    Functions for use by searchIndex.ftl
+*/
+
+function updateSearchIndexerStatus() {
+    $.ajax({
+        url: searchIndexerStatusUrl,
+        dataType: "html",
+        complete: function(xhr, status) {
+        	updatePanelContents(xhr.responseText);
+        	setTimeout(updateSearchIndexerStatus,5000);
+        }
+    });
+}
+
+function updatePanelContents(contents) {
+	document.getElementById("searchIndexerStatus").innerHTML = contents;
+}
+
+$(document).ready(updateSearchIndexerStatus());

webapp/web/templates/freemarker/body/admin/searchIndex.ftl

@@ -1,43 +1,21 @@
 <#-- $This file is distributed under the terms of the license in /doc/license.txt$ -->
 
 <#-- 
-    Template for the page that controls the updating or rebuilding of the Search Index. 
+    Template for the page that displays the status of the Search Indexer.
+    Most of it is provided by the AJAX call. 
 -->
 
 <h2>${i18n().search_index_status}</h2>
 
-<#if !indexIsConnected>
-    <!-- Can't contact the search engine. Indexing would be impossible. Show an error message. -->
-    <section id="error-alert" role="alert">
-        <img src="${urls.images}/iconAlert.png" width="24" height="24" alt="Error alert icon" />
-        <p>${i18n().search_index_not_connected}</p>
-        <p><tt>SearchEngine.ping()</tt> ${i18n().failed}.
-        <p>${i18n().check_startup_status}</p>
-    </section>
+<div id="searchIndexerStatus">
+	Search Indexer Status
+</div>
 
-<#elseif worklevel == "IDLE">
-    <!-- Search indexer is idle. Show the button that rebuilds the index. -->
-    <h3>${i18n().search_indexer_idle}</h3>
-    <#if hasPreviousBuild??>
-        <p>${i18n().most_recent_update} ${since?string("hh:mm:ss a, MMMM dd, yyyy")}</p>
-    </#if>
+<script>
+    searchIndexerStatusUrl = '${statusUrl}'
+</script>
 
-    <form action="${actionUrl}" method="POST">
-        <p>
-            <input class="submit" type="submit" name="rebuild" value="${i18n().rebuild_button}" role="button" />
-            ${i18n().reset_search_index}
-        </p>
-    </form>
+${stylesheets.add('<link rel="stylesheet" href="${urls.base}/css/search/searchIndex.css" />')}
 
-<#elseif totalToDo == 0>
-    <!-- Search indexer is preparing the list of records. Show elapsed time since request. -->
-    <h3>${i18n().preparing_to_rebuild_index}</h3>
-    <p>${i18n().since_elapsed_time(since?string("hh:mm:ss a, MMMM dd, yyyy"),elapsed)}</p>
-    
-<#else>
-    <!-- Search indexer is re-building the index. Show the progress. -->
-    <h3>${i18n().current_task(currentTask)}</h3>
-    <p>${i18n().since_elapsed_time_est_total(since?string("hh:mm:ss a, MMMM dd, yyyy"),elapsed,expected)}</p>
-    <p>${i18n().index_recs_completed(completedCount,totalToDo)}</p>
-    
-</#if>
+${scripts.add('<script type="text/javascript" src="${urls.base}/js/search/searchIndex.js"></script>')}   
+${scripts.add('<script type="text/javascript" src="${urls.base}/js/jquery-ui/js/jquery-ui-1.8.9.custom.min.js"></script>')}   

webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl

@@ -0,0 +1,72 @@
+<#-- $This file is distributed under the terms of the license in /doc/license.txt$ -->
+
+<#-- 
+    Template for the page that controls the updating or rebuilding of the Search Index. 
+-->
+
+<section id="indexer" role="region">
+    <#if status.statusType == "IDLE">
+        <h3>The search indexer has been idle since ${status.since?datetime}</h3>
+
+    <#elseif status.statusType = "PROCESSING_URIS">
+        <h3>The search indexer has been processing URIs since ${status.since?datetime}</h3>
+        <p><@showIndexerCounts "URI_COUNTS", status /></p>
+        <p><@showElapsedTime status.elapsed /> Expected completion ${status.expectedCompletion?datetime}.</p>
+
+    <#elseif status.statusType = "PROCESSING_STMTS">
+        <h3>The search indexer has been processing changed statements since ${status.since?datetime}</h3>
+        <p><@showIndexerCounts "STATEMENT_COUNTS", status /></p>
+        <p><@showElapsedTime status.elapsed /> Expected completion ${status.expectedCompletion?datetime}.</p>
+
+    <#elseif status.statusType = "PREPARING_REBUILD">
+        <h3>The search indexer has been preparing to rebuild the index since ${status.since?datetime}</h3>
+
+    <#else>
+        <h3>The search indexer status is: ${status.statusType}
+    </#if>
+    
+    
+    <form action="${rebuildUrl}" method="POST">
+        <p>
+            <#if status.statusType == "IDLE">
+                <input class="submit" type="submit" name="rebuild" value="${i18n().rebuild_button}" role="button" />
+                ${i18n().reset_search_index}
+            </#if>
+        </p>
+    </form>
+
+
+    <h3>History</h3>
+    <table class="history">
+        <tr> <th>Event</th> <th>Status</th> <th>Since</th> <th>Counts</th> </tr>
+        <#list history as ie>
+           <@showIndexerEvent ie />
+        </#list>
+    </table>
+</section>
+
+
+<#macro showElapsedTime elapsed>
+    Elapsed time ${elapsed[0]}:${elapsed[1]}:${elapsed[2]}.
+</#macro>
+
+
+<#macro showIndexerEvent event>
+    <tr>
+        <td>${event.event}</td>
+        <td>${event.statusType}</td>
+        <td>${event.since?datetime}</td>
+        <td><@showIndexerCounts event.countsType, event /></td>
+    </tr>
+</#macro>
+
+
+<#macro showIndexerCounts countsType, counts>
+   <#if countsType == "URI_COUNTS">
+       Updated: ${counts.updated}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total}
+   <#elseif countsType == "STATEMENT_COUNTS">
+       Processed: ${counts.processed}, remaining: ${counts.remaining}, total: ${counts.total}
+   <#elseif countsType == "REBUILD_COUNTS">
+       Number of individuals before rebuild: ${counts.numberOfIndividuals}
+   </#if>
+</#macro>