From b3564e31bda01d14ad55b6b0ca344daae9464ed3 Mon Sep 17 00:00:00 2001 From: Jim Blake Date: Mon, 26 Jan 2015 10:18:02 -0500 Subject: [PATCH] VIVO-869 Cleanup: Committer, shorter event type, ALLTEXTUNSTEMMED, et. al. Get rid of the SearchIndexerSetup.Committer listener class. This functionality becomes internal to UpdateUrisTask. Use shorter names for Event.Type values, so the display will be more compact. ALLTEXTUNSTEMMED is not automatically populated from ALLTEXT, so change the default target field on SimpleSparqlQueryDocumentModifier to include both of them. Trivial cleanup in UpdateDocumentWorkUnit. Create StackTraceLayout, a simple Log4J layout that will print the stack trace for each call. Used for diagnosing the source of uninformative log messages. --- .../webapp/dao/jena/VClassGroupCache.java | 2 +- .../modules/searchIndexer/SearchIndexer.java | 4 +- .../searchindex/SearchIndexerSetup.java | 40 ------------- .../SimpleSparqlQueryDocumentModifier.java | 7 ++- .../tasks/UpdateDocumentWorkUnit.java | 38 +++++------- .../tasks/UpdateStatementsTask.java | 10 ++-- .../searchindex/tasks/UpdateUrisTask.java | 42 ++++++++++---- .../utils/logging/StackTraceLayout.java | 58 +++++++++++++++++++ 8 files changed, 117 insertions(+), 84 deletions(-) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java index ce2c220df..9d2c36bad 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java @@ -210,7 +210,7 @@ public class VClassGroupCache implements SearchIndexer.Listener { @Override public void receiveSearchIndexerEvent(Event event) { switch (event.getType()) { - case STOP_PROCESSING_URIS: + case STOP_URIS: log.debug("rebuilding because of IndexBuilder " + event.getType()); requestCacheUpdate(); break; diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java index 8780e892a..a1ffb6330 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java @@ -152,9 +152,9 @@ public interface SearchIndexer extends Application.Module { PAUSE, UNPAUSE, - START_PROCESSING_URIS, STOP_PROCESSING_URIS, + START_URIS, STOP_URIS, - START_PROCESSING_STATEMENTS, STOP_PROCESSING_STATEMENTS, + START_STATEMENTS, STOP_STATEMENTS, START_REBUILD, STOP_REBUILD, diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java index 394ba688a..01d407b7f 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java @@ -2,9 +2,6 @@ package edu.cornell.mannlib.vitro.webapp.searchindex; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS; - import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -14,12 +11,7 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.modules.Application; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Listener; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.search.controller.IndexController; @@ -48,7 +40,6 @@ public class SearchIndexerSetup implements ServletContextListener { private IndexingChangeListener listener; private DeveloperDisabledChangeListener listenerWrapper; private IndexHistory history; - private Committer committer; @Override public void contextInitialized(ServletContextEvent sce) { @@ -73,9 +64,6 @@ public class SearchIndexerSetup implements ServletContextListener { searchIndexer.addListener(this.history); IndexController.setHistory(this.history); - this.committer = new Committer(); - searchIndexer.addListener(this.committer); - searchIndexer .startup(app, new ComponentStartupStatusImpl(this, ss)); searchIndexer.unpause(); @@ -90,7 +78,6 @@ public class SearchIndexerSetup implements ServletContextListener { public void contextDestroyed(ServletContextEvent sce) { searchIndexer.shutdown(app); - searchIndexer.removeListener(this.committer); searchIndexer.removeListener(this.history); try { @@ -102,31 +89,4 @@ public class SearchIndexerSetup implements ServletContextListener { listener.shutdown(); } - // ---------------------------------------------------------------------- - // Helper classes - // ---------------------------------------------------------------------- - - private static class Committer implements Listener { - private final SearchEngine searchEngine = ApplicationUtils.instance() - .getSearchEngine(); - - @Override - public void receiveSearchIndexerEvent(Event event) { - Type type = event.getType(); - if (type == PROGRESS || type == STOP_PROCESSING_URIS) { - commitChanges(); - } - } - - private void commitChanges() { - try { - searchEngine.commit(); - } catch (SearchEngineNotRespondingException e) { - log.error("Failed to commit the changes: " - + "the search engine is not responding."); - } catch (Exception e) { - log.error("Failed to commit the changes.", e); - } - } - } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java index 849eeb9ca..6804a8e55 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java @@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding; import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService.CONTENT; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXT; +import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXTUNSTEMMED; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +34,8 @@ import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation; * If the individual qualifies, execute the SPARQL queries and add the results * to the specified search fields. * - * If there are no specified search fields, ALLTEXT is assumed. + * If there are no specified search fields, ALLTEXT and ALLTEXTUNSTEMMED are + * assumed. * * An individual qualifies if it satisfies any of the type restrictions, or if * there are no type restrictions. @@ -63,7 +65,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier, /** * The names of the fields where the results of the queries will be stored. - * If empty, it is assumed to be ALLTEXT. + * If empty, it is assumed to be ALLTEXT and ALLTEXTUNSTEMMED. */ private List fieldNames = new ArrayList<>(); @@ -105,6 +107,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier, } if (fieldNames.isEmpty()) { fieldNames.add(ALLTEXT); + fieldNames.add(ALLTEXTUNSTEMMED); } if (queries.isEmpty()) { throw new IllegalStateException( diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java index c0840f05b..9b147385f 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java @@ -145,7 +145,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: IdUriLabel"; + return "Internal: IdUriLabel"; } } @@ -184,7 +184,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddClasses"; + return "Internal: AddClasses"; } } @@ -203,7 +203,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddMostSpecificTypes"; + return "Internal: AddMostSpecificTypes"; } } @@ -212,21 +212,18 @@ public class UpdateDocumentWorkUnit implements Runnable { public void modifyDocument(Individual ind, SearchInputDocument doc) { List stmts = ind .getObjectPropertyStatements(); - if (stmts == null) { - return; - } - - for (ObjectPropertyStatement stmt : stmts) { - if (URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) { - continue; + if (stmts != null) { + for (ObjectPropertyStatement stmt : stmts) { + if (!URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) { + addToAlltext(doc, stmt.getObject().getRdfsLabel()); + } } - addToAlltext(doc, stmt.getObject().getRdfsLabel()); } } @Override public String toString() { - return "REQUIRED: AddObjectPropertyText"; + return "Internal: AddObjectPropertyText"; } } @@ -234,21 +231,18 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public void modifyDocument(Individual ind, SearchInputDocument doc) { List stmts = ind.getDataPropertyStatements(); - if (stmts == null) { - return; - } - - for (DataPropertyStatement stmt : stmts) { - if (stmt.getDatapropURI().equals(URI_RDFS_LABEL)) { - continue; + if (stmts != null) { + for (DataPropertyStatement stmt : stmts) { + if (!stmt.getDatapropURI().equals(URI_RDFS_LABEL)) { + addToAlltext(doc, stmt.getData()); + } } - addToAlltext(doc, stmt.getData()); } } @Override public String toString() { - return "REQUIRED: AddDataPropertyText"; + return "Internal: AddDataPropertyText"; } } @@ -263,7 +257,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddEntityBoost"; + return "Internal: AddEntityBoost"; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java index dcf489fa6..b6aa98894 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java @@ -3,8 +3,8 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.tasks; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_STATEMENTS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_STATEMENTS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_STATEMENTS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_STATEMENTS; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_STMTS; import java.util.ArrayList; @@ -82,14 +82,12 @@ public class UpdateStatementsTask implements Task { @Override public void run() { - listeners - .fireEvent(new Event(START_PROCESSING_STATEMENTS, getStatus())); + listeners.fireEvent(new Event(START_STATEMENTS, getStatus())); findAffectedUris(); - listeners.fireEvent(new Event(PROGRESS, getStatus())); updateTheUris(); - listeners.fireEvent(new Event(STOP_PROCESSING_STATEMENTS, getStatus())); + listeners.fireEvent(new Event(STOP_STATEMENTS, getStatus())); } private void findAffectedUris() { diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java index 9634616f5..3b61e29b0 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java @@ -3,8 +3,8 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.tasks; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_URIS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_URIS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_URIS; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_URIS; import java.util.Collection; @@ -21,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus; @@ -43,6 +44,9 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExclud * * Deletions are done synchronously, but updates are scheduled to run on the * thread pool. + * + * Commit requests are issued to the SearchEngine at each progress event and + * again at the end of the task. */ public class UpdateUrisTask implements Task { private static final Log log = LogFactory.getLog(UpdateUrisTask.class); @@ -67,7 +71,7 @@ public class UpdateUrisTask implements Task { this.listeners = listeners; this.pool = pool; - this.status = new Status(uris.size(), 500, listeners); + this.status = new Status(this, uris.size(), 500); this.searchEngine = ApplicationUtils.instance().getSearchEngine(); @@ -75,7 +79,7 @@ public class UpdateUrisTask implements Task { @Override public void run() { - listeners.fireEvent(new Event(START_PROCESSING_URIS, status + listeners.fireEvent(new Event(START_URIS, status .getSearchIndexerStatus())); excluders.startIndexing(); modifiers.startIndexing(); @@ -99,9 +103,11 @@ public class UpdateUrisTask implements Task { } pool.waitUntilIdle(); + commitChanges(); + excluders.stopIndexing(); modifiers.stopIndexing(); - listeners.fireEvent(new Event(STOP_PROCESSING_URIS, status + listeners.fireEvent(new Event(STOP_URIS, status .getSearchIndexerStatus())); } @@ -160,6 +166,21 @@ public class UpdateUrisTask implements Task { log.debug("scheduled update to " + ind); } + private void fireEvent(Event event) { + listeners.fireEvent(event); + if (event.getType() == PROGRESS || event.getType() == STOP_URIS) { + commitChanges(); + } + } + + private void commitChanges() { + try { + searchEngine.commit(); + } catch (SearchEngineException e) { + log.warn("Failed to commit changes.", e); + } + } + @Override public void notifyWorkUnitCompletion(Runnable workUnit) { log.debug("completed update to " @@ -181,18 +202,18 @@ public class UpdateUrisTask implements Task { * synchronized. */ private static class Status { + private final UpdateUrisTask parent; private final int total; private final int progressInterval; - private final ListenerList listeners; private int updated = 0; private int deleted = 0; private int excluded = 0; private Date since = new Date(); - public Status(int total, int progressInterval, ListenerList listeners) { + public Status(UpdateUrisTask parent, int total, int progressInterval) { + this.parent = parent; this.total = total; this.progressInterval = progressInterval; - this.listeners = listeners; } public synchronized void incrementUpdates() { @@ -213,8 +234,7 @@ public class UpdateUrisTask implements Task { private void maybeFireProgressEvent() { if (updated > 0 && updated % progressInterval == 0) { - listeners.fireEvent(new Event(PROGRESS, - getSearchIndexerStatus())); + parent.fireEvent(new Event(PROGRESS, getSearchIndexerStatus())); } } @@ -241,7 +261,7 @@ public class UpdateUrisTask implements Task { @Override public String toString() { - return "ExcludeIfNoVClasses"; + return "Internal: ExcludeIfNoVClasses"; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java new file mode 100644 index 000000000..383d509d8 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java @@ -0,0 +1,58 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.utils.logging; + +import org.apache.log4j.Layout; +import org.apache.log4j.spi.LoggingEvent; + +/** + * This is a nasty Layout that prints a full stack trace for every logging + * event. + * + * Here is an example of how to use this to show full stack traces every time we + * get a log message from org.apache.jena.riot. + * + * 
+ * log4j.appender.RiotAppender=org.apache.log4j.RollingFileAppender 
+ * log4j.appender.RiotAppender.File= $${catalina.home}/logs/${webapp.name}.riot.log
+ * log4j.appender.RiotAppender.MaxFileSize=10MB 
+ * log4j.appender.RiotAppender.MaxBackupIndex=10 
+ * log4j.appender.RiotAppender.layout=edu.cornell.mannlib.vitro.webapp.utils.logging.StackTraceLayout 
+ * log4j.logger.org.apache.jena.riot=INFO, RiotAppender
+ *
+ */ +public class StackTraceLayout extends Layout { + + public StackTraceLayout() { + } + + @Override + public void activateOptions() { + } + + /** + * Print the level, the message, and the full stack trace. + */ + @Override + public String format(LoggingEvent event) { + StringBuilder buffer = new StringBuilder(); + buffer.append(event.getLevel().toString()).append(" - ") + .append(event.getRenderedMessage()).append(LINE_SEP); + for (StackTraceElement traceElement : Thread.currentThread() + .getStackTrace()) { + buffer.append(" ").append(traceElement.toString()) + .append(LINE_SEP); + } + return buffer.toString(); + } + + /** + * The StackTraceLayout does not handle the throwable contained within + * LoggingEvents. Thus, it returns true. + */ + @Override + public boolean ignoresThrowable() { + return true; + } + +}