diff --git a/webapp/config/applicationSetup.n3 b/webapp/config/applicationSetup.n3 new file mode 100644 index 000000000..cabf41a08 --- /dev/null +++ b/webapp/config/applicationSetup.n3 @@ -0,0 +1,45 @@ +@prefix : . + +:application + a , + ; + :hasSearchEngine :instrumentedSearchEngineWrapper ; + :hasSearchIndexer :basicSearchIndexer ; + :hasImageProcessor :jaiImageProcessor ; + :hasFileStorage :ptiFileStorage ; + :hasContentTripleSource :sdbContentTripleSource ; + :hasConfigurationTripleSource :tdbConfigurationTripleSource ; + :hasTBoxReasonerModule :jfactTBoxReasonerModule . + +:jaiImageProcessor + a , + . + +:ptiFileStorage + a , + . + +:instrumentedSearchEngineWrapper + a , + ; + :wraps :solrSearchEngine . + +:solrSearchEngine + a , + . + +:basicSearchIndexer + a , + . + +:sdbContentTripleSource + a , + . + +:tdbConfigurationTripleSource + a , + . + +:jfactTBoxReasonerModule + a , + . \ No newline at end of file diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/application/ApplicationImpl.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/application/ApplicationImpl.java index 45842830f..c8e7b7085 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/application/ApplicationImpl.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/application/ApplicationImpl.java @@ -16,13 +16,14 @@ import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus; import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage; import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; -import edu.cornell.mannlib.vitro.webapp.triplesource.impl.BasicCombinedTripleSource; import edu.cornell.mannlib.vitro.webapp.startup.ComponentStartupStatusImpl; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; +import edu.cornell.mannlib.vitro.webapp.triplesource.impl.BasicCombinedTripleSource; import edu.cornell.mannlib.vitro.webapp.utils.configuration.Property; import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation; @@ -38,6 +39,7 @@ public class ApplicationImpl implements Application { private VitroHomeDirectory homeDirectory; private SearchEngine searchEngine; + private SearchIndexer searchIndexer; private ImageProcessor imageProcessor; private FileStorage fileStorage; private ContentTripleSource contentTripleSource; @@ -73,11 +75,27 @@ public class ApplicationImpl implements Application { searchEngine = se; } else { throw new IllegalStateException( - "Configuration includes multiple SearchEngine instancess: " + "Configuration includes multiple SearchEngine instances: " + searchEngine + ", and " + se); } } + @Override + public SearchIndexer getSearchIndexer() { + return searchIndexer; + } + + @Property(uri = "http://vitro.mannlib.cornell.edu/ns/vitro/ApplicationSetup#hasSearchIndexer") + public void setSearchIndexer(SearchIndexer si) { + if (searchIndexer == null) { + searchIndexer = si; + } else { + throw new IllegalStateException( + "Configuration includes multiple SearchIndexer instances: " + + searchIndexer + ", and " + si); + } + } + @Override public ImageProcessor getImageProcessor() { return imageProcessor; @@ -89,7 +107,7 @@ public class ApplicationImpl implements Application { imageProcessor = ip; } else { throw new IllegalStateException( - "Configuration includes multiple ImageProcessor instancess: " + "Configuration includes multiple ImageProcessor instances: " + imageProcessor + ", and " + ip); } } @@ -105,7 +123,7 @@ public class ApplicationImpl implements Application { fileStorage = fs; } else { throw new IllegalStateException( - "Configuration includes multiple FileStorage intances: " + "Configuration includes multiple FileStorage instances: " + fileStorage + ", and " + fs); } } @@ -121,7 +139,7 @@ public class ApplicationImpl implements Application { contentTripleSource = source; } else { throw new IllegalStateException( - "Configuration includes multiple intances of ContentTripleSource: " + "Configuration includes multiple instances of ContentTripleSource: " + contentTripleSource + ", and " + source); } } @@ -137,7 +155,7 @@ public class ApplicationImpl implements Application { configurationTripleSource = source; } else { throw new IllegalStateException( - "Configuration includes multiple intances of ConfigurationTripleSource: " + "Configuration includes multiple instances of ConfigurationTripleSource: " + configurationTripleSource + ", and " + source); } } @@ -153,7 +171,7 @@ public class ApplicationImpl implements Application { tboxReasonerModule = module; } else { throw new IllegalStateException( - "Configuration includes multiple intances of TBoxReasonerModule: " + "Configuration includes multiple instances of TBoxReasonerModule: " + tboxReasonerModule + ", and " + module); } } @@ -164,6 +182,10 @@ public class ApplicationImpl implements Application { throw new IllegalStateException( "Configuration did not include a SearchEngine."); } + if (searchIndexer == null) { + throw new IllegalStateException( + "Configuration did not include a SearchIndexer."); + } if (imageProcessor == null) { throw new IllegalStateException( "Configuration did not include an ImageProcessor."); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/api/SparqlUpdateApiController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/api/SparqlUpdateApiController.java index cbff4774d..21ba21aea 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/api/SparqlUpdateApiController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/api/SparqlUpdateApiController.java @@ -25,11 +25,12 @@ import com.hp.hpl.jena.update.UpdateAction; import com.hp.hpl.jena.update.UpdateFactory; import com.hp.hpl.jena.update.UpdateRequest; +import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; /** * Process SPARQL Updates, as an API. @@ -98,13 +99,14 @@ public class SparqlUpdateApiController extends VitroApiServlet { ServletContext ctx = req.getSession().getServletContext(); VitroRequest vreq = new VitroRequest(req); - IndexBuilder.getBuilder(ctx).pause(); + SearchIndexer indexer = ApplicationUtils.instance().getSearchIndexer(); + indexer.pause(); try { Dataset ds = new RDFServiceDataset(vreq.getUnfilteredRDFService()); GraphStore graphStore = GraphStoreFactory.create(ds); UpdateAction.execute(parsed, graphStore); } finally { - IndexBuilder.getBuilder(ctx).unpause(); + indexer.unpause(); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/authenticate/BasicAuthenticator.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/authenticate/BasicAuthenticator.java index 99a3fbcfd..cd5063b70 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/authenticate/BasicAuthenticator.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/authenticate/BasicAuthenticator.java @@ -15,6 +15,7 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vedit.beans.LoginStatusBean; import edu.cornell.mannlib.vedit.beans.LoginStatusBean.AuthenticationSource; +import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.auth.identifier.RequestIdentifiers; import edu.cornell.mannlib.vitro.webapp.auth.identifier.common.IsRootUser; import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel; @@ -29,7 +30,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.dao.jena.LoginEvent; import edu.cornell.mannlib.vitro.webapp.dao.jena.LogoutEvent; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; /** * The "standard" implementation of Authenticator. @@ -162,7 +164,17 @@ public class BasicAuthenticator extends Authenticator { if (IsRootUser.isRootUser(RequestIdentifiers .getIdBundleForRequest(request))) { - IndexBuilder.checkIndexOnRootLogin(request); + try { + SearchEngine engine = ApplicationUtils.instance() + .getSearchEngine(); + if (engine.documentCount() == 0) { + log.info("Search index is empty. Running a full index rebuild."); + ApplicationUtils.instance().getSearchIndexer() + .rebuildIndex(); + } + } catch (SearchEngineException e) { + log.warn("Unable to check for search index", e); + } } } @@ -262,7 +274,8 @@ public class BasicAuthenticator extends Authenticator { * Get a reference to the UserAccountsDao, or null. */ private UserAccountsDao getUserAccountsDao() { - UserAccountsDao userAccountsDao = getWebappDaoFactory().getUserAccountsDao(); + UserAccountsDao userAccountsDao = getWebappDaoFactory() + .getUserAccountsDao(); if (userAccountsDao == null) { log.error("getUserAccountsDao: no UserAccountsDao"); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java index 5dd11602c..ce2c220df 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java @@ -40,9 +40,9 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; @@ -64,7 +64,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; * search index is not built or if there were problems building the index, * the class counts from VClassGroupCache will be incorrect. */ -public class VClassGroupCache implements IndexingEventListener { +public class VClassGroupCache implements SearchIndexer.Listener { private static final Log log = LogFactory.getLog(VClassGroupCache.class); private static final String ATTRIBUTE_NAME = "VClassGroupCache"; @@ -208,19 +208,17 @@ public class VClassGroupCache implements IndexingEventListener { * Handle notification of events from the IndexBuilder. */ @Override - public void notifyOfIndexingEvent(EventTypes event) { - switch( event ){ - case FINISH_FULL_REBUILD: - case FINISHED_UPDATE: - log.debug("rebuilding because of IndexBuilder " + event.name()); - requestCacheUpdate(); - break; - default: - log.debug("ignoring event type " + event.name()); - break; - - } - } + public void receiveSearchIndexerEvent(Event event) { + switch (event.getType()) { + case STOP_PROCESSING_URIS: + log.debug("rebuilding because of IndexBuilder " + event.getType()); + requestCacheUpdate(); + break; + default: + log.debug("ignoring event type " + event.getType()); + break; + } + } /* **************** static utility methods ***************** */ @@ -489,8 +487,8 @@ public class VClassGroupCache implements IndexingEventListener { context.setAttribute(ATTRIBUTE_NAME,vcgc); log.info("VClassGroupCache added to context"); - IndexBuilder indexBuilder = IndexBuilder.getBuilder(context); - indexBuilder.addIndexBuilderListener(vcgc); + SearchIndexer searchIndexer = ApplicationUtils.instance().getSearchIndexer(); + searchIndexer.addListener(vcgc); log.info("VClassGroupCache set to listen to events from IndexBuilder"); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/Application.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/Application.java index 7d875f707..c25ef7c06 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/Application.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/Application.java @@ -8,6 +8,7 @@ import edu.cornell.mannlib.vitro.webapp.application.VitroHomeDirectory; import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage; import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource; @@ -17,21 +18,23 @@ import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource */ public interface Application { ServletContext getServletContext(); - + VitroHomeDirectory getHomeDirectory(); SearchEngine getSearchEngine(); + SearchIndexer getSearchIndexer(); + ImageProcessor getImageProcessor(); - + FileStorage getFileStorage(); - + ContentTripleSource getContentTripleSource(); - + ConfigurationTripleSource getConfigurationTripleSource(); - + TBoxReasonerModule getTBoxReasonerModule(); - + void shutdown(); public interface Component { @@ -39,8 +42,16 @@ public interface Application { NEW, ACTIVE, STOPPED } + /** + * This should be called only once, and should be the first call on this + * Component. + */ void startup(Application application, ComponentStartupStatus ss); + /** + * This should be called only once, and should be the last call on this + * Component. + */ void shutdown(Application application); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java new file mode 100644 index 000000000..ef21c88e3 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java @@ -0,0 +1,147 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.modules.searchIndexer; + +import java.util.Collection; + +import edu.cornell.mannlib.vitro.webapp.modules.Application; + +/** + * Interface for the code that controls the contents of the search index. + * + * The only calls that are valid after shutdown are shutdown(), getStatus() and + * removeListener(). + */ +public interface SearchIndexer extends Application.Module { + /** + * Update the search documents for these URIs. + * + * For each URI that belongs in the index, a new search document is built, + * replacing any document that may already exist for that URI. For each URI + * that does not belong in the index, any existing document is removed. + * + * A URI belongs in the index if it refers to an existing individual in the + * model, and is not excluded. + * + * @param uris + * if null or empty, this call has no effect. + * @throws IllegalStateException + * if called after shutdown() + */ + void scheduleUpdatesForUris(Collection uris); + + /** + * Remove all of the existing documents in the index, and schedule updates + * for all of the individuals in the model. + * + * If a rebuild is already pending or in progress, this method has no + * effect. + * + * @throws IllegalStateException + * if called after shutdown() + */ + void rebuildIndex(); + + /** + * Stop processing new tasks. Requests will be queued until a call to + * unpause(). + * + * @throws IllegalStateException + * if called after shutdown() + */ + void pause(); + + /** + * Resume processing new tasks. Any requests that were received since the + * call to pause() will now be scheduled for processing. + * + * Has no effect if called after shutdown(). + */ + void unpause(); + + /** + * What is the current status of the indexer? + * + * Still valid after shutdown(). + */ + SearchIndexerStatus getStatus(); + + /** + * Add this listener, allowing it to receive events from the indexer. If + * this listener has already been added, this method has no effect. + * + * @param listener + * if null, this method has no effect. + * @throws IllegalStateException + * if called after shutdown() + */ + void addListener(Listener listener); + + /** + * Remove this listener, meaning that it will no longer receive events from + * the indexer. If this listener is not active, this method has no effect. + * + * Has no effect if called after shutdown(). + * + * @param listener + * if null, this method has no effect. + */ + void removeListener(Listener listener); + + /** + * Stop processing and release resources. This call should block until the + * dependent threads are stopped. + * + * Repeated calls have no effect. + */ + @Override + void shutdown(Application app); + + /** + * A listener that will be notified of events from the SearchIndexer. + */ + public static interface Listener { + void receiveSearchIndexerEvent(Event event); + } + + /** + * An immutable event object. The event type describes just what happened. + * The status object describes what the indexer is doing now. + */ + public static class Event { + public enum Type { + START_PROCESSING_URIS, + + PROGRESS_PROCESSING_URIS, + + STOP_PROCESSING_URIS, + + START_PROCESSING_STATEMENTS, + + PROGRESS_PROCESSING_STATEMENTS, + + STOP_PROCESSING_STATEMENTS, + + REBUILD_REQUESTED, + + SHUTDOWN_REQUESTED + } + + private final Type type; + private final SearchIndexerStatus status; + + public Event(Type type, SearchIndexerStatus status) { + this.type = type; + this.status = status; + } + + public Type getType() { + return type; + } + + public SearchIndexerStatus getStatus() { + return status; + } + } + +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java new file mode 100644 index 000000000..afc5e1c16 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java @@ -0,0 +1,156 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.modules.searchIndexer; + +import java.util.Date; + +/** + * An immutable summary of the status of the SearchIndexer, at some point in + * time.Contains the current state, and some counts. + * + * If the indexer is processing URIs, processing statements, or preparing a + * rebuild, the counts are URI_COUNTS, STATEMENT_COUNTS, or REBUILD_COUNTS. + * + * When the indexer starts up, and when it is is shut down, the counts are + * NO_COUNTS. + * + * If the indexer is idle, the counts are carried over from the previous + * operation. + */ +public class SearchIndexerStatus { + public enum State { + IDLE, PROCESSING_URIS, PROCESSING_STMTS, PREPARING_REBUILD, SHUTDOWN + } + + private final State state; + private final Date since; + private final Counts counts; + + public SearchIndexerStatus(State state, Date since, Counts counts) { + this.state = state; + this.since = since; + this.counts = counts; + } + + public State getState() { + return state; + } + + public Date getSince() { + return since; + } + + public Counts getCounts() { + return counts; + } + + public abstract static class Counts { + public enum Type { + URI_COUNTS, STATEMENT_COUNTS, REBUILD_COUNTS, NO_COUNTS + } + + private final Type type; + + public Counts(Type type) { + this.type = type; + } + + public Type getType() { + return this.type; + } + + public UriCounts asUriCounts() { + return (UriCounts) this; + } + + public StatementCounts asStatementCounts() { + return (StatementCounts) this; + } + + public RebuildCounts asRebuildCounts() { + return (RebuildCounts) this; + } + + public NoCounts asNoCounts() { + return (NoCounts) this; + } + } + + public static class UriCounts extends Counts { + private final int deleted; + private final int updated; + private final int remaining; + private final int total; + + public UriCounts(int deleted, int updated, int remaining, int total) { + super(Type.URI_COUNTS); + this.deleted = deleted; + this.updated = updated; + this.remaining = remaining; + this.total = total; + } + + public int getDeleted() { + return deleted; + } + + public int getUpdated() { + return updated; + } + + public int getRemaining() { + return remaining; + } + + public int getTotal() { + return total; + } + + } + + public static class StatementCounts extends Counts { + private final int processed; + private final int remaining; + private final int total; + + public StatementCounts(int processed, int remaining, int total) { + super(Type.STATEMENT_COUNTS); + this.processed = processed; + this.remaining = remaining; + this.total = total; + } + + public int getProcessed() { + return processed; + } + + public int getRemaining() { + return remaining; + } + + public int getTotal() { + return total; + } + + } + + public static class RebuildCounts extends Counts { + private final int numberOfIndividuals; + + public RebuildCounts(int numberOfIndividuals) { + super(Type.REBUILD_COUNTS); + this.numberOfIndividuals = numberOfIndividuals; + } + + public int getNumberOfIndividuals() { + return numberOfIndividuals; + } + + } + + public static class NoCounts extends Counts { + public NoCounts() { + super(Type.NO_COUNTS); + } + } +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java index 0d939f982..62e18f45d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java @@ -3,12 +3,10 @@ package edu.cornell.mannlib.vitro.webapp.search.controller; import java.io.IOException; -import java.util.Collection; import java.util.Date; import java.util.HashMap; import java.util.Map; -import javax.servlet.ServletContext; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -18,31 +16,30 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; -import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.RequestedAction; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder; -import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues; -import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.RedirectResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; -import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel; -import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.StatementCounts; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.UriCounts; +import edu.cornell.mannlib.vitro.webapp.services.freemarker.FreemarkerProcessingServiceSetup; /** - * Accepts requests to rebuild or update the search index. It uses an - * IndexBuilder and finds that IndexBuilder from the servletContext using the - * key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder" + * Accepts requests to display the current status of the search index, or to + * initiate a rebuild. * - * That IndexBuilder will be associated with a object that implements the - * IndexerIface. + * A DISPLAY or REBUILD request is handled like any other FreemarkerHttpServlet. + * A STATUS is an AJAX request, we override doGet() so we can format the + * template without enclosing it in a body template. * - * An example of the IndexerIface is SearchIndexer. An example of the IndexBuilder - * and SearchIndexer setup is in SearchIndexerSetup. - * - * @author bdc34 + * When initialized, this servlet adds a listener to the SearchIndexer, so it + * can maintain a history of activity. This will provide the contents of the + * display. */ public class IndexController extends FreemarkerHttpServlet { private static final Log log = LogFactory.getLog(IndexController.class); @@ -50,27 +47,22 @@ public class IndexController extends FreemarkerHttpServlet { /** * 
 	 * This request might be:
-	 * SETUP -- Index is not building and nothing is requested. Solicit requests.
-	 * REFRESH  -- Index is building, nothing is requested. Show continuing status.
-	 * REBUILD -- Rebuild is requested. Set the rebuild flag and show continuing status.
+	 * DISPLAY (default) -- Send the template that will contain the status display. 
+	 * STATUS  -- Send the current status and history.
+	 * REBUILD -- Initiate a rebuild. Then act like DISPLAY.
 	 *
*/ private enum RequestType { - SETUP, REFRESH, REBUILD; + DISPLAY, STATUS, REBUILD; /** What type of request is this? */ static RequestType fromRequest(HttpServletRequest req) { if (hasParameter(req, "rebuild")) { return REBUILD; + } else if (hasParameter(req, "status")) { + return STATUS; } else { - ServletContext ctx = req.getSession().getServletContext(); - IndexBuilder builder = IndexBuilder.getBuilder(ctx); - WorkLevelStamp workLevel = builder.getWorkLevel(); - if (workLevel.getLevel() == WorkLevel.WORKING) { - return REFRESH; - } else { - return SETUP; - } + return DISPLAY; } } @@ -81,13 +73,43 @@ public class IndexController extends FreemarkerHttpServlet { } private static final String PAGE_URL = "/SearchIndex"; - private static final String TEMPLATE_NAME = "searchIndex.ftl"; + private static final String PAGE_TEMPLATE_NAME = "searchIndex.ftl"; + private static final String STATUS_TEMPLATE_NAME = "searchIndexStatus.ftl"; public static final RequestedAction REQUIRED_ACTIONS = SimplePermission.MANAGE_SEARCH_INDEX.ACTION; + private SearchIndexer indexer; + private IndexHistory history; + @Override - protected AuthorizationRequest requiredActions(VitroRequest vreq) { - return REQUIRED_ACTIONS; + public void init() throws ServletException { + super.init(); + this.indexer = ApplicationUtils.instance().getSearchIndexer(); + this.history = new IndexHistory(); + this.indexer.addListener(this.history); + } + + @Override + public void destroy() { + this.indexer.removeListener(this.history); + super.destroy(); + } + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException, ServletException { + if (!isAuthorizedToDisplayPage(req, resp, REQUIRED_ACTIONS)) { + return; + } + + switch (RequestType.fromRequest(req)) { + case STATUS: + showStatus(req, resp); + break; + default: + super.doGet(req, resp); + break; + } } @Override @@ -95,78 +117,82 @@ public class IndexController extends FreemarkerHttpServlet { return "Rebuild Search Index"; } - @Override - public void doGet(HttpServletRequest req, HttpServletResponse resp) - throws IOException, ServletException { - if (RequestType.fromRequest(req) == RequestType.REFRESH) { - resp.addHeader("Refresh", "5; " + UrlBuilder.getUrl(PAGE_URL)); - } - super.doGet(req, resp); - } - @Override protected ResponseValues processRequest(VitroRequest vreq) { - Map body = new HashMap(); - body.put("actionUrl", UrlBuilder.getUrl(PAGE_URL)); - - try { - IndexBuilder builder = IndexBuilder.getBuilder(getServletContext()); - - switch (RequestType.fromRequest(vreq)) { - case REBUILD: - builder.doIndexRebuild(); - Thread.sleep(500); - return redirectToRefresh(); - default: - return showCurrentStatus(builder, body); - } - } catch (Exception e) { - log.error("Error rebuilding search index", e); - body.put("errorMessage", - "There was an error while rebuilding the search index. " - + e.getMessage()); - return new ExceptionResponseValues( - Template.ERROR_MESSAGE.toString(), body, e); + switch (RequestType.fromRequest(vreq)) { + case REBUILD: + requestRebuild(); + return showDisplay(); + default: + return showDisplay(); } } - private ResponseValues redirectToRefresh() { - return new RedirectResponseValues(PAGE_URL); + private ResponseValues showDisplay() { + HashMap body = new HashMap<>(); + body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true")); + body.put("rebuildUrl", UrlBuilder.getUrl(PAGE_URL, "rebuild", "true")); + return new TemplateResponseValues(PAGE_TEMPLATE_NAME, body); } - private ResponseValues showCurrentStatus(IndexBuilder builder, - Map body) { - WorkLevelStamp stamp = builder.getWorkLevel(); - - WorkLevel workLevel = stamp.getLevel(); - long completedCount = builder.getCompletedCount(); - long totalToDo = builder.getTotalToDo(); - Date since = stamp.getSince(); - Date expectedCompletion = figureExpectedCompletion(since, totalToDo, - completedCount); - - body.put("worklevel", workLevel.toString()); - body.put("completedCount", completedCount); - body.put("totalToDo", totalToDo); - body.put("currentTask", figureCurrentTask(stamp.getFlags())); - body.put("since", since); - body.put("elapsed", formatElapsedTime(since, new Date())); - body.put("expected", formatElapsedTime(since, expectedCompletion)); - body.put("hasPreviousBuild", since.getTime() > 0L); - body.put("indexIsConnected", testIndexConnection()); - return new TemplateResponseValues(TEMPLATE_NAME, body); - } - - private Boolean testIndexConnection() { + private void showStatus(HttpServletRequest req, HttpServletResponse resp) + throws IOException { try { - ApplicationUtils.instance().getSearchEngine().ping(); - return Boolean.TRUE; + Map body = new HashMap<>(); + body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true")); + body.put("rebuildUrl", + UrlBuilder.getUrl(PAGE_URL, "rebuild", "true")); + body.put("status", buildStatusMap(indexer.getStatus())); + body.put("history", history.toMaps()); + + String rendered = FreemarkerProcessingServiceSetup.getService( + getServletContext()).renderTemplate(STATUS_TEMPLATE_NAME, + body, req); + resp.getWriter().write(rendered); } catch (Exception e) { - log.error("Can't connect to the search engine.", e); - return Boolean.FALSE; + resp.setStatus(500); + resp.getWriter().write(e.toString()); + log.error(e, e); } } + private void requestRebuild() { + indexer.rebuildIndex(); + } + + private Map buildStatusMap(SearchIndexerStatus status) { + Map map = new HashMap<>(); + State state = status.getState(); + map.put("statusType", state); + map.put("since", status.getSince()); + + if (state == State.PROCESSING_URIS) { + UriCounts counts = status.getCounts().asUriCounts(); + map.put("updated", counts.getUpdated()); + map.put("deleted", counts.getDeleted()); + map.put("remaining", counts.getRemaining()); + map.put("total", counts.getTotal()); + map.put("elapsed", breakDownElapsedTime(status.getSince())); + map.put("expectedCompletion", + figureExpectedCompletion(status.getSince(), + counts.getTotal(), + counts.getTotal() - counts.getRemaining())); + } else if (state == State.PROCESSING_STMTS) { + StatementCounts counts = status.getCounts().asStatementCounts(); + map.put("processed", counts.getProcessed()); + map.put("remaining", counts.getRemaining()); + map.put("total", counts.getTotal()); + map.put("elapsed", breakDownElapsedTime(status.getSince())); + map.put("expectedCompletion", + figureExpectedCompletion(status.getSince(), + counts.getTotal(), counts.getProcessed())); + } else { + // nothing for IDLE or SHUTDOWN, except what's already there. + } + + return map; + } + private Date figureExpectedCompletion(Date startTime, long totalToDo, long completedCount) { Date now = new Date(); @@ -186,22 +212,12 @@ public class IndexController extends FreemarkerHttpServlet { return new Date(expectedDuration + startTime.getTime()); } - private String formatElapsedTime(Date since, Date until) { - long elapsedMillis = until.getTime() - since.getTime(); + private int[] breakDownElapsedTime(Date since) { + long elapsedMillis = new Date().getTime() - since.getTime(); long seconds = (elapsedMillis / 1000L) % 60L; long minutes = (elapsedMillis / 60000L) % 60L; long hours = elapsedMillis / 3600000L; - return String.format("%02d:%02d:%02d", hours, minutes, seconds); - } - - private String figureCurrentTask(Collection flags) { - if (flags.contains(IndexBuilder.FLAG_REBUILDING)) { - return "Rebuilding"; - } else if (flags.contains(IndexBuilder.FLAG_UPDATING)) { - return "Updating"; - } else { - return "Not working on"; - } + return new int[] {(int) hours, (int) minutes, (int) seconds}; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java new file mode 100644 index 000000000..663e620a8 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java @@ -0,0 +1,95 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.search.controller; + +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.Counts; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.Counts.Type; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.RebuildCounts; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.StatementCounts; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.UriCounts; + +/** + * This listener keeps a list of the last several SearchIndexer events, and will + * format them for display in a Freemarker template. + */ +public class IndexHistory implements SearchIndexer.Listener { + private final static int MAX_EVENTS = 10; + + private final Deque events = new LinkedList<>(); + + @Override + public void receiveSearchIndexerEvent(Event event) { + synchronized (events) { + events.addFirst(event); + while (events.size() > MAX_EVENTS) { + events.removeLast(); + } + } + } + + public List> toMaps() { + synchronized (events) { + List> list = new ArrayList<>(); + for (Event event : events) { + list.add(toMap(event)); + } + return list; + } + } + + private Map toMap(Event event) { + SearchIndexerStatus status = event.getStatus(); + Counts counts = status.getCounts(); + Type countsType = counts.getType(); + + Map map = new HashMap<>(); + map.put("event", event.getType()); + map.put("statusType", status.getState()); + map.put("since", status.getSince()); + map.put("countsType", countsType); + + switch (countsType) { + case URI_COUNTS: + addCounts(counts.asUriCounts(), map); + break; + case STATEMENT_COUNTS: + addCounts(counts.asStatementCounts(), map); + break; + case REBUILD_COUNTS: + addCounts(counts.asRebuildCounts(), map); + break; + default: // NO_COUNTS + break; + } + + return map; + } + + private void addCounts(UriCounts counts, Map map) { + map.put("updated", counts.getUpdated()); + map.put("deleted", counts.getDeleted()); + map.put("remaining", counts.getRemaining()); + map.put("total", counts.getTotal()); + } + + private void addCounts(StatementCounts counts, Map map) { + map.put("processed", counts.getProcessed()); + map.put("remaining", counts.getRemaining()); + map.put("total", counts.getTotal()); + } + + private void addCounts(RebuildCounts counts, Map map) { + map.put("numberOfIndividuals", counts.getNumberOfIndividuals()); + } + +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java index a649b8b22..65777ee6e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java @@ -15,6 +15,7 @@ import javax.servlet.http.HttpServletRequest; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; import edu.cornell.mannlib.vitro.webapp.auth.policy.PolicyHelper; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; @@ -23,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; /** * Accepts requests to update a set of URIs in the search index. @@ -73,8 +74,8 @@ public class SearchServiceController extends FreemarkerHttpServlet { */ private ResponseValues doUpdateUrisInSearch(HttpServletRequest req) throws IOException, ServletException { - IndexBuilder builder = IndexBuilder.getBuilder(getServletContext()); - int uriCount = new UpdateUrisInIndex().doUpdateUris(req, builder); + SearchIndexer indexer = ApplicationUtils.instance().getSearchIndexer(); + int uriCount = new UpdateUrisInIndex().doUpdateUris(req, indexer); Map body = new HashMap<>(); body.put("msg", "Received " + uriCount + " URIs."); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java index f723613ac..c8cf04478 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java @@ -6,6 +6,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.Charset; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Scanner; @@ -20,7 +21,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; /** * Class that performs the update of the uris in the search index for the @@ -38,7 +39,7 @@ public class UpdateUrisInIndex { * * @throws IOException */ - protected int doUpdateUris(HttpServletRequest req, IndexBuilder builder) + protected int doUpdateUris(HttpServletRequest req, SearchIndexer indexer) throws ServletException, IOException { Map> map = new VitroRequest(req).getFiles(); if (map == null) { @@ -51,25 +52,25 @@ public class UpdateUrisInIndex { for (String name : map.keySet()) { for (FileItem item : map.get(name)) { log.debug("Found " + item.getSize() + " byte file for '" + name + "'"); - uriCount += processFileItem(builder, item, enc); + uriCount += processFileItem(indexer, item, enc); } } return uriCount; } - private int processFileItem(IndexBuilder builder, + private int processFileItem(SearchIndexer indexer, FileItem item, Charset enc) throws IOException { - int count = 0; + List uris = new ArrayList<>(); Reader reader = new InputStreamReader(item.getInputStream(), enc.name()); try (Scanner scanner = createScanner(reader)) { while (scanner.hasNext()) { String uri = scanner.next(); log.debug("Request to index uri '" + uri + "'"); - builder.addToChanged(uri); - count++; + uris.add(uri); } } - return count; + indexer.scheduleUpdatesForUris(uris); + return uris.size(); } @SuppressWarnings("resource") diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerImpl.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerImpl.java new file mode 100644 index 000000000..f28893ebb --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerImpl.java @@ -0,0 +1,220 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.searchindex; + +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.REBUILD_REQUESTED; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_STATEMENTS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_STATEMENTS; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import javax.servlet.ServletContext; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import edu.cornell.mannlib.vitro.webapp.modules.Application; +import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State; +import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp; + +/** + * TODO + * A silly implementation that just wraps the old IndexBuilder. + */ +public class SearchIndexerImpl implements SearchIndexer { + private static final Log log = LogFactory.getLog(SearchIndexerImpl.class); + + private final ListenerList listeners = new ListenerList(); + + private ServletContext ctx; + // private Set excluders; + // private Set modifiers; + // private Set uriFinders; + + // TODO + private IndexBuilder indexBuilder; + + @Override + public void startup(Application application, ComponentStartupStatus ss) { + try { + this.ctx = application.getServletContext(); + // loadConfiguration(); + // ss.info("Configured SearchIndexer: excluders=" + excluders + // + ", modifiers=" + modifiers + ", uriFinders=" + uriFinders); + + // TODO + this.indexBuilder = (IndexBuilder) ctx + .getAttribute(IndexBuilder.class.getName()); + + this.indexBuilder.addIndexBuilderListener(new BridgeListener()); + } catch (Exception e) { + ss.fatal("Failed to configure the SearchIndexer", e); + } + } + + private void createAndFire(Event.Type type) { + listeners.fireEvent(new Event(type, getStatus())); + } + + // private void loadConfiguration() throws ConfigurationBeanLoaderException + // { + // ConfigurationBeanLoader beanLoader = new ConfigurationBeanLoader( + // ModelAccess.on(ctx).getOntModel(DISPLAY), ctx); + // excluders = beanLoader.loadAll(SearchIndexExcluder.class); + // modifiers = beanLoader.loadAll(DocumentModifier.class); + // uriFinders = beanLoader.loadAll(IndexingUriFinder.class); + // } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.Application.Component#shutdown + * (edu.cornell.mannlib.vitro.webapp.modules.Application) + */ + @Override + public void shutdown(Application application) { + // TODO + } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer# + * scheduleUpdatesForUris(java.util.Collection) + */ + @Override + public void scheduleUpdatesForUris(Collection uris) { + // TODO + for (String uri : uris) { + indexBuilder.addToChanged(uri); + } + } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer# + * rebuildIndex() + */ + @Override + public void rebuildIndex() { + // TODO + indexBuilder.doIndexRebuild(); + } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer# + * pause() + */ + @Override + public void pause() { + // TODO + indexBuilder.pause(); + } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer# + * unpause() + */ + @Override + public void unpause() { + // TODO + indexBuilder.unpause(); + } + + /* + * (non-Javadoc) + * + * @see + * edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer# + * getStatus() + */ + @Override + public SearchIndexerStatus getStatus() { + // TODO + WorkLevelStamp workLevel = indexBuilder.getWorkLevel(); + WorkLevel level = workLevel.getLevel(); + Date since = workLevel.getSince(); + if (level == WorkLevel.IDLE) { + return new SearchIndexerStatus(State.IDLE, since, + new SearchIndexerStatus.NoCounts()); + } else { + return new SearchIndexerStatus(State.PROCESSING_URIS, since, + new SearchIndexerStatus.UriCounts(1, 2, 3, 6)); + } + } + + @Override + public void addListener(Listener listener) { + listeners.add(listener); + } + + @Override + public void removeListener(Listener listener) { + listeners.remove(listener); + } + + private static class ListenerList { + private final List list; + + public ListenerList() { + list = Collections.synchronizedList(new ArrayList()); + } + + public void add(Listener l) { + list.add(l); + } + + public void remove(Listener l) { + list.remove(l); + } + + public void fireEvent(Event e) { + synchronized (list) { + for (Listener l : list) { + l.receiveSearchIndexerEvent(e); + } + } + } + } + + private class BridgeListener implements IndexingEventListener { + @Override + public void notifyOfIndexingEvent(EventTypes ie) { + switch (ie) { + case START_UPDATE: + createAndFire(START_PROCESSING_STATEMENTS); + break; + case FINISHED_UPDATE: + createAndFire(STOP_PROCESSING_STATEMENTS); + break; + case START_FULL_REBUILD: + createAndFire(REBUILD_REQUESTED); + createAndFire(START_PROCESSING_STATEMENTS); + break; + default: // FINISH_FULL_REBUILD + createAndFire(STOP_PROCESSING_STATEMENTS); + break; + } + } + } +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java index 9d71b083c..cec9b9aa6 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java @@ -35,6 +35,7 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding.DocumentMod import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExcluder; import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.AdditionalUriFinders; import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.StatementToURIsToUpdate; +import edu.cornell.mannlib.vitro.webapp.startup.ComponentStartupStatusImpl; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; import edu.cornell.mannlib.vitro.webapp.utils.configuration.ConfigurationBeanLoader; import edu.cornell.mannlib.vitro.webapp.utils.configuration.ConfigurationBeanLoaderException; @@ -43,6 +44,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.developer.listeners.DeveloperDisab /** * TODO + * A silly implementation that just wraps the old IndexBuilder with a new SearchIndexerImpl. */ public class SearchIndexerSetup implements ServletContextListener { private static final Log log = LogFactory.getLog(SearchIndexerSetup.class); @@ -96,6 +98,8 @@ public class SearchIndexerSetup implements ServletContextListener { Key.SEARCH_INDEX_SUPPRESS_MODEL_CHANGE_LISTENER)); ss.info(this, "Setup of search indexer completed."); + + ApplicationUtils.instance().getSearchIndexer().startup(ApplicationUtils.instance(), new ComponentStartupStatusImpl(this, ss)); } catch (Throwable e) { ss.fatal(this, "could not setup search engine", e); } diff --git a/webapp/test/edu/cornell/mannlib/vitro/webapp/search/documentBuilding/testPerson.n3 b/webapp/test/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/testPerson.n3 similarity index 100% rename from webapp/test/edu/cornell/mannlib/vitro/webapp/search/documentBuilding/testPerson.n3 rename to webapp/test/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/testPerson.n3 diff --git a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modelaccess/ContextModelAccessStub.java b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modelaccess/ContextModelAccessStub.java index 566b6cb02..5cd0d77c9 100644 --- a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modelaccess/ContextModelAccessStub.java +++ b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modelaccess/ContextModelAccessStub.java @@ -4,6 +4,7 @@ package stubs.edu.cornell.mannlib.vitro.webapp.modelaccess; import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.ReasoningOption.ASSERTIONS_AND_INFERENCES; import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService.CONTENT; +import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames.FULL_UNION; import java.util.EnumMap; import java.util.HashMap; @@ -18,6 +19,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector; import edu.cornell.mannlib.vitro.webapp.modelaccess.ContextModelAccess; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.ReasoningOption; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService; +import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; /** @@ -29,10 +31,14 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; public class ContextModelAccessStub implements ContextModelAccess { // ---------------------------------------------------------------------- // Stub infrastructure + // + // Warning: ontModelMap and rdfServiceMap are not connected, so it's up to + // the user to insure that they are consistent with each other. // ---------------------------------------------------------------------- private final Map wadfMap = new HashMap<>(); private final Map rdfServiceMap = new EnumMap<>(WhichService.class); + private final Map ontModelMap = new HashMap<>(); public void setWebappDaoFactory(WebappDaoFactory wadf) { setWebappDaoFactory(wadf, ASSERTIONS_AND_INFERENCES); @@ -46,6 +52,10 @@ public class ContextModelAccessStub implements ContextModelAccess { public void setRDFService(WhichService which, RDFService rdfService) { rdfServiceMap.put(which, rdfService); } + + public void setOntModel(String name, OntModel model) { + ontModelMap.put(name, model); + } // ---------------------------------------------------------------------- // Stub methods @@ -66,6 +76,16 @@ public class ContextModelAccessStub implements ContextModelAccess { return rdfServiceMap.get(which); } + @Override + public OntModel getOntModel() { + return getOntModel(FULL_UNION); + } + + @Override + public OntModel getOntModel(String name) { + return ontModelMap.get(name); + } + // ---------------------------------------------------------------------- // Un-implemented methods // ---------------------------------------------------------------------- @@ -94,18 +114,6 @@ public class ContextModelAccessStub implements ContextModelAccess { "ContextModelAccessStub.getModelMaker() not implemented."); } - @Override - public OntModel getOntModel() { - throw new RuntimeException( - "ContextModelAccessStub.getOntModel() not implemented."); - } - - @Override - public OntModel getOntModel(String name) { - throw new RuntimeException( - "ContextModelAccessStub.getOntModel() not implemented."); - } - @Override public OntModelSelector getOntModelSelector() { throw new RuntimeException( diff --git a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/ApplicationStub.java b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/ApplicationStub.java index b8534649c..92bd80107 100644 --- a/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/ApplicationStub.java +++ b/webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/ApplicationStub.java @@ -12,6 +12,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.Application; import edu.cornell.mannlib.vitro.webapp.modules.fileStorage.FileStorage; import edu.cornell.mannlib.vitro.webapp.modules.imageProcessor.ImageProcessor; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; import edu.cornell.mannlib.vitro.webapp.modules.tboxreasoner.TBoxReasonerModule; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ConfigurationTripleSource; import edu.cornell.mannlib.vitro.webapp.modules.tripleSource.ContentTripleSource; @@ -104,9 +105,14 @@ public class ApplicationStub implements Application { @Override public TBoxReasonerModule getTBoxReasonerModule() { - // TODO Auto-generated method stub - throw new RuntimeException("ApplicationStub.getTBoxReasonerModule() not implemented."); - + throw new RuntimeException( + "ApplicationStub.getTBoxReasonerModule() not implemented."); + } + + @Override + public SearchIndexer getSearchIndexer() { + throw new RuntimeException( + "Application.getSearchIndexer() not implemented."); } } diff --git a/webapp/web/css/search/searchIndex.css b/webapp/web/css/search/searchIndex.css new file mode 100644 index 000000000..b4c21f8fd --- /dev/null +++ b/webapp/web/css/search/searchIndex.css @@ -0,0 +1,17 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +/* Styles for the search index controller. */ + +section#indexer table.history { + font-size: smaller; + border: 1px solid gray; + width: 100%; +} + +section#indexer table.history th{ + font-weight: bolder; +} + +section#indexer table.history td{ + padding: 2px 5px 2px 5px; +} diff --git a/webapp/web/js/search/searchIndex.js b/webapp/web/js/search/searchIndex.js new file mode 100644 index 000000000..560136fa6 --- /dev/null +++ b/webapp/web/js/search/searchIndex.js @@ -0,0 +1,22 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +/* + Functions for use by searchIndex.ftl +*/ + +function updateSearchIndexerStatus() { + $.ajax({ + url: searchIndexerStatusUrl, + dataType: "html", + complete: function(xhr, status) { + updatePanelContents(xhr.responseText); + setTimeout(updateSearchIndexerStatus,5000); + } + }); +} + +function updatePanelContents(contents) { + document.getElementById("searchIndexerStatus").innerHTML = contents; +} + +$(document).ready(updateSearchIndexerStatus()); diff --git a/webapp/web/templates/freemarker/body/admin/searchIndex.ftl b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl index 06b609b24..76fae4191 100644 --- a/webapp/web/templates/freemarker/body/admin/searchIndex.ftl +++ b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl @@ -1,43 +1,21 @@ <#-- $This file is distributed under the terms of the license in /doc/license.txt$ --> <#-- - Template for the page that controls the updating or rebuilding of the Search Index. + Template for the page that displays the status of the Search Indexer. + Most of it is provided by the AJAX call. -->

${i18n().search_index_status}

-<#if !indexIsConnected> - - - -<#elseif worklevel == "IDLE"> - -

${i18n().search_indexer_idle}

- <#if hasPreviousBuild??> -

${i18n().most_recent_update} ${since?string("hh:mm:ss a, MMMM dd, yyyy")}

- - -
-

- - ${i18n().reset_search_index} -

-
- -<#elseif totalToDo == 0> - -

${i18n().preparing_to_rebuild_index}

-

${i18n().since_elapsed_time(since?string("hh:mm:ss a, MMMM dd, yyyy"),elapsed)}

- -<#else> - -

${i18n().current_task(currentTask)}

-

${i18n().since_elapsed_time_est_total(since?string("hh:mm:ss a, MMMM dd, yyyy"),elapsed,expected)}

-

${i18n().index_recs_completed(completedCount,totalToDo)}

- - +
+ Search Indexer Status +
+ + + +${stylesheets.add('')} + +${scripts.add('')} +${scripts.add('')} diff --git a/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl b/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl new file mode 100644 index 000000000..c9bc8abf0 --- /dev/null +++ b/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl @@ -0,0 +1,72 @@ +<#-- $This file is distributed under the terms of the license in /doc/license.txt$ --> + +<#-- + Template for the page that controls the updating or rebuilding of the Search Index. +--> + +
+ <#if status.statusType == "IDLE"> +

The search indexer has been idle since ${status.since?datetime}

+ + <#elseif status.statusType = "PROCESSING_URIS"> +

The search indexer has been processing URIs since ${status.since?datetime}

+

<@showIndexerCounts "URI_COUNTS", status />

+

<@showElapsedTime status.elapsed /> Expected completion ${status.expectedCompletion?datetime}.

+ + <#elseif status.statusType = "PROCESSING_STMTS"> +

The search indexer has been processing changed statements since ${status.since?datetime}

+

<@showIndexerCounts "STATEMENT_COUNTS", status />

+

<@showElapsedTime status.elapsed /> Expected completion ${status.expectedCompletion?datetime}.

+ + <#elseif status.statusType = "PREPARING_REBUILD"> +

The search indexer has been preparing to rebuild the index since ${status.since?datetime}

+ + <#else> +

The search indexer status is: ${status.statusType} + + + +
+

+ <#if status.statusType == "IDLE"> + + ${i18n().reset_search_index} + +

+
+ + +

History

+ + + <#list history as ie> + <@showIndexerEvent ie /> + +
Event Status Since Counts
+
+ + +<#macro showElapsedTime elapsed> + Elapsed time ${elapsed[0]}:${elapsed[1]}:${elapsed[2]}. + + + +<#macro showIndexerEvent event> + + ${event.event} + ${event.statusType} + ${event.since?datetime} + <@showIndexerCounts event.countsType, event /> + + + + +<#macro showIndexerCounts countsType, counts> + <#if countsType == "URI_COUNTS"> + Updated: ${counts.updated}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total} + <#elseif countsType == "STATEMENT_COUNTS"> + Processed: ${counts.processed}, remaining: ${counts.remaining}, total: ${counts.total} + <#elseif countsType == "REBUILD_COUNTS"> + Number of individuals before rebuild: ${counts.numberOfIndividuals} + +