From e8d8c494f7a7d9b71f0e874c9f15058233d8ad83 Mon Sep 17 00:00:00 2001 From: j2blake Date: Tue, 11 Oct 2011 19:45:30 +0000 Subject: [PATCH] NIHVIVO-3209 Create VitroBackgroundThread to provide an answer to the question "are all background threads idle?". Make IndexBuilder and VClassGroupCache.RebuildGroupCacheThread subclasses of VitroBackgroundThread, so they will be included in that question. --- .../webapp/dao/jena/VClassGroupCache.java | 5 +- .../search/controller/IndexController.java | 159 +++++++++++++----- .../webapp/search/indexing/IndexBuilder.java | 21 ++- .../utils/threads/VitroBackgroundThread.java | 80 +++++++++ .../freemarker/body/admin/searchIndex.ftl | 23 +++ 5 files changed, 242 insertions(+), 46 deletions(-) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/utils/threads/VitroBackgroundThread.java create mode 100644 webapp/web/templates/freemarker/body/admin/searchIndex.ftl diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java index 2a50f82dd..eaae87cee 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java @@ -34,6 +34,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; public class VClassGroupCache { private static final Log log = LogFactory.getLog(VClassGroupCache.class); @@ -230,7 +231,7 @@ public class VClassGroupCache { /* ******************** RebuildGroupCacheThread **************** */ - protected class RebuildGroupCacheThread extends Thread { + protected class RebuildGroupCacheThread extends VitroBackgroundThread { private final VClassGroupCache cache; private long queueChangeMillis = 0L; private long timeToBuildLastCache = 100L; //in msec @@ -256,8 +257,10 @@ public class VClassGroupCache { log.debug("rebuildGroupCacheThread.run() -- starting rebuildCache()"); long start = System.currentTimeMillis(); + setWorkLevel(WorkLevel.WORKING); rebuildRequested = false; rebuildCache( cache ); + setWorkLevel(WorkLevel.IDLE); timeToBuildLastCache = System.currentTimeMillis() - start; log.debug("rebuildGroupCacheThread.run() -- rebuilt cache in " diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java index b84eefef8..73b79feec 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java @@ -1,10 +1,16 @@ -/* $This file is distributed under the terms of the license in /doc/license.txt$ */ +/* $This file is dactionistrietupbuted under the terms of theare license in /doc/license.txt$ */ package edu.cornell.mannlib.vitro.webapp.search.controller; +import java.io.IOException; import java.util.HashMap; import java.util.Map; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -12,58 +18,131 @@ import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.usepages.ManageSearchIndex; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.RedirectResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp; /** - * Accepts requests to rebuild or update the search index. It uses - * an IndexBuilder and finds that IndexBuilder from the servletContext using - * the key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder" - * - * That IndexBuilder will be associated with a object that implements the IndexerIface. - * - * An example of the IndexerIface is SolrIndexer. - * An example of the IndexBuilder and SolrIndexer setup is in SolrSetup. - * + * Accepts requests to rebuild or update the search index. It uses an + * IndexBuilder and finds that IndexBuilder from the servletContext using the + * key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder" + * + * That IndexBuilder will be associated with a object that implements the + * IndexerIface. + * + * An example of the IndexerIface is SolrIndexer. An example of the IndexBuilder + * and SolrIndexer setup is in SolrSetup. + * * @author bdc34 */ public class IndexController extends FreemarkerHttpServlet { - private static final Log log = LogFactory.getLog(IndexController.class); - public static final Actions REQUIRED_ACTIONS = new Actions(new ManageSearchIndex()); - + /** + * 
+	 * This request might be:
+	 * SETUP -- Index is not building and nothing is requested. Solicit requests.
+	 * REFRESH  -- Index is building, nothing is requested. Show continuing status.
+	 * REBUILD -- Rebuild is requested. Set the rebuild flag and show continuing status.
+	 * UPDATE -- Update is requested. Set the update flag and show continuing status.
+	 *
+ */ + private enum RequestType { + SETUP, REFRESH, REBUILD, UPDATE; + + /** What type of request is this? */ + static RequestType fromRequest(HttpServletRequest req) { + if (hasParameter(req, "rebuild")) { + return REBUILD; + } else if (hasParameter(req, "update")) { + return UPDATE; + } else { + ServletContext ctx = req.getSession().getServletContext(); + IndexBuilder builder = IndexBuilder.getBuilder(ctx); + WorkLevelStamp workLevel = builder.getWorkLevel(); + if (workLevel.getLevel() == WorkLevel.WORKING) { + return REFRESH; + } else { + return SETUP; + } + } + } + + private static boolean hasParameter(HttpServletRequest req, String key) { + String value = req.getParameter(key); + return (value != null) && (!value.isEmpty()); + } + } + + private static final String PAGE_URL = "/SearchIndex"; + private static final String TEMPLATE_NAME = "searchIndex.ftl"; + + public static final Actions REQUIRED_ACTIONS = new Actions( + new ManageSearchIndex()); + @Override protected Actions requiredActions(VitroRequest vreq) { return REQUIRED_ACTIONS; } - - @Override - protected String getTitle(String siteName, VitroRequest vreq) { - return "Full Search Index Rebuild"; - } - - @Override - protected ResponseValues processRequest(VitroRequest vreq) { - Map body = new HashMap(); - - try { - IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName()); - if( vreq.getParameter("update") != null ){ - builder.doUpdateIndex(); - }else{ - builder.doIndexRebuild(); - } - - } catch (Exception e) { - log.error("Error rebuilding search index",e); - body.put("errorMessage", "There was an error while rebuilding the search index. " + e.getMessage()); - return new ExceptionResponseValues(Template.ERROR_MESSAGE.toString(), body, e); - } - - body.put("message","Rebuild of search index started. A message will be written to the vivo log when indexing is complete."); - return new TemplateResponseValues(Template.MESSAGE.toString(), body); - } + + @Override + protected String getTitle(String siteName, VitroRequest vreq) { + return "Search Index Update or Rebuild"; + } + + @Override + public void doGet(HttpServletRequest req, HttpServletResponse resp) + throws IOException, ServletException { + if (RequestType.fromRequest(req) == RequestType.REFRESH) { + resp.addHeader("Refresh", "5; " + UrlBuilder.getUrl(PAGE_URL)); + } + super.doGet(req, resp); + } + + @Override + protected ResponseValues processRequest(VitroRequest vreq) { + Map body = new HashMap(); + body.put("actionUrl", UrlBuilder.getUrl(PAGE_URL)); + + try { + IndexBuilder builder = IndexBuilder.getBuilder(getServletContext()); + + switch (RequestType.fromRequest(vreq)) { + case REBUILD: + builder.doIndexRebuild(); + return redirectToRefresh(body); + case UPDATE: + builder.doUpdateIndex(); + return redirectToRefresh(body); + default: + return showCurrentStatus(builder, body); + } + } catch (Exception e) { + log.error("Error rebuilding search index", e); + body.put("errorMessage", + "There was an error while rebuilding the search index. " + + e.getMessage()); + return new ExceptionResponseValues( + Template.ERROR_MESSAGE.toString(), body, e); + } + } + + private ResponseValues redirectToRefresh(Map body) { + return new RedirectResponseValues(PAGE_URL); + } + + private ResponseValues showCurrentStatus(IndexBuilder builder, + Map body) { + WorkLevelStamp stamp = builder.getWorkLevel(); + body.put("worklevel", stamp.getLevel().toString()); + body.put("since", stamp.getSince()); + body.put("hasPreviousBuild", stamp.getSince().getTime() > 0L); + return new TemplateResponseValues(TEMPLATE_NAME, body); + } + } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index 4f3c52cb6..a7b8de8d6 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -25,6 +25,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.beans.StatementToURIsToUpdate; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; /** @@ -37,7 +38,7 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.StatementToURIsToUpdate; * listener can use an IndexBuilder to keep the full text index in sncy with * updates to a model. It calls IndexBuilder.addToChangedUris(). */ -public class IndexBuilder extends Thread { +public class IndexBuilder extends VitroBackgroundThread { private WebappDaoFactory wdf; private final IndexerIface indexer; @@ -71,6 +72,16 @@ public class IndexBuilder extends Thread { public static final int MAX_THREADS = Math.max( MAX_UPDATE_THREADS, MAX_REINDEX_THREADS); private static final Log log = LogFactory.getLog(IndexBuilder.class); + + public static IndexBuilder getBuilder(ServletContext ctx) { + Object o = ctx.getAttribute(IndexBuilder.class.getName()); + if (o instanceof IndexBuilder) { + return (IndexBuilder) o; + } else { + log.error("IndexBuilder has not bee initialized."); + return null; + } + } public IndexBuilder(IndexerIface indexer, WebappDaoFactory wdf, @@ -127,10 +138,6 @@ public class IndexBuilder extends Thread { this.notifyAll(); } - public boolean isIndexing(){ - return indexer.isIndexing(); - } - /** * This is called when the system shuts down. */ @@ -145,12 +152,16 @@ public class IndexBuilder extends Thread { while(! stopRequested ){ try{ if( reindexRequested ){ + setWorkLevel(WorkLevel.WORKING); log.debug("full re-index requested"); indexRebuild(); + setWorkLevel(WorkLevel.IDLE); }else if( !changedStmtQueue.isEmpty() ){ + setWorkLevel(WorkLevel.WORKING); Thread.sleep(WAIT_AFTER_NEW_WORK_INTERVAL); //wait a bit to let a bit more work to come into the queue log.debug("work found for IndexBuilder, starting update"); updatedIndex(); + setWorkLevel(WorkLevel.IDLE); } else { log.debug("there is no indexing working to do, waiting for work"); synchronized (this) { this.wait(MAX_IDLE_INTERVAL); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/threads/VitroBackgroundThread.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/threads/VitroBackgroundThread.java new file mode 100644 index 000000000..eb69092fc --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/threads/VitroBackgroundThread.java @@ -0,0 +1,80 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.utils.threads; + +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * A simple base class that will allow us to find the background threads and + * check their current status. + */ +public class VitroBackgroundThread extends Thread { + Log log = LogFactory.getLog(VitroBackgroundThread.class); + + private static final ConcurrentLinkedQueue> allThreads = new ConcurrentLinkedQueue>(); + + public static List getThreads() { + List list = new ArrayList(); + for (WeakReference ref : allThreads) { + VitroBackgroundThread t = ref.get(); + if (t != null) { + list.add(t); + } + } + return list; + } + + public enum WorkLevel { + IDLE, WORKING + } + + private volatile WorkLevelStamp stamp = new WorkLevelStamp(WorkLevel.IDLE); + + public VitroBackgroundThread(String name) { + super(name); + allThreads.add(new WeakReference(this)); + } + + public VitroBackgroundThread(Runnable target, String name) { + super(target, name); + allThreads.add(new WeakReference(this)); + } + + protected void setWorkLevel(WorkLevel level) { + log.debug("Set work level on '" + this.getName() + "' to " + level); + stamp = new WorkLevelStamp(level); + } + + public WorkLevelStamp getWorkLevel() { + return stamp; + } + + /** + * An immutable object that holds both the current work level and the time + * that it was set. + */ + public static class WorkLevelStamp { + private final WorkLevel level; + private final long since; + + public WorkLevelStamp(WorkLevel level) { + this.level = level; + this.since = System.currentTimeMillis(); + } + + public WorkLevel getLevel() { + return level; + } + + public Date getSince() { + return new Date(since); + } + } +} diff --git a/webapp/web/templates/freemarker/body/admin/searchIndex.ftl b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl new file mode 100644 index 000000000..6dbac6070 --- /dev/null +++ b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl @@ -0,0 +1,23 @@ +<#-- $This file is distributed under the terms of the license in /doc/license.txt$ --> + +<#-- + Template for the page that controls the updating or rebuilding of the Search Index. +--> + +

Search Index Status

+ +<#if worklevel == "IDLE"> + <#if hasPreviousBuild??> +

Previous activity completed at ${since?string("hh:mm:ss a, MMMM dd, yyyy")}

+ + +
+ + Add the latest changes to the index. +
+ + Start with an empty index and build it completely. +
+<#else> +

Active since ${since?string("hh:mm:ss a, MMMM dd, yyyy")}

+