diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java index ce2c220df..9d2c36bad 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java @@ -210,7 +210,7 @@ public class VClassGroupCache implements SearchIndexer.Listener { @Override public void receiveSearchIndexerEvent(Event event) { switch (event.getType()) { - case STOP_PROCESSING_URIS: + case STOP_URIS: log.debug("rebuilding because of IndexBuilder " + event.getType()); requestCacheUpdate(); break; diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java index 8780e892a..a1ffb6330 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java @@ -152,9 +152,9 @@ public interface SearchIndexer extends Application.Module { PAUSE, UNPAUSE, - START_PROCESSING_URIS, STOP_PROCESSING_URIS, + START_URIS, STOP_URIS, - START_PROCESSING_STATEMENTS, STOP_PROCESSING_STATEMENTS, + START_STATEMENTS, STOP_STATEMENTS, START_REBUILD, STOP_REBUILD, diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java index 394ba688a..01d407b7f 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java @@ -2,9 +2,6 @@ package edu.cornell.mannlib.vitro.webapp.searchindex; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS; - import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -14,12 +11,7 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.modules.Application; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; -import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type; -import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Listener; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.search.controller.IndexController; @@ -48,7 +40,6 @@ public class SearchIndexerSetup implements ServletContextListener { private IndexingChangeListener listener; private DeveloperDisabledChangeListener listenerWrapper; private IndexHistory history; - private Committer committer; @Override public void contextInitialized(ServletContextEvent sce) { @@ -73,9 +64,6 @@ public class SearchIndexerSetup implements ServletContextListener { searchIndexer.addListener(this.history); IndexController.setHistory(this.history); - this.committer = new Committer(); - searchIndexer.addListener(this.committer); - searchIndexer .startup(app, new ComponentStartupStatusImpl(this, ss)); searchIndexer.unpause(); @@ -90,7 +78,6 @@ public class SearchIndexerSetup implements ServletContextListener { public void contextDestroyed(ServletContextEvent sce) { searchIndexer.shutdown(app); - searchIndexer.removeListener(this.committer); searchIndexer.removeListener(this.history); try { @@ -102,31 +89,4 @@ public class SearchIndexerSetup implements ServletContextListener { listener.shutdown(); } - // ---------------------------------------------------------------------- - // Helper classes - // ---------------------------------------------------------------------- - - private static class Committer implements Listener { - private final SearchEngine searchEngine = ApplicationUtils.instance() - .getSearchEngine(); - - @Override - public void receiveSearchIndexerEvent(Event event) { - Type type = event.getType(); - if (type == PROGRESS || type == STOP_PROCESSING_URIS) { - commitChanges(); - } - } - - private void commitChanges() { - try { - searchEngine.commit(); - } catch (SearchEngineNotRespondingException e) { - log.error("Failed to commit the changes: " - + "the search engine is not responding."); - } catch (Exception e) { - log.error("Failed to commit the changes.", e); - } - } - } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java index 849eeb9ca..6804a8e55 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java @@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding; import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService.CONTENT; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXT; +import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXTUNSTEMMED; import java.util.ArrayList; import java.util.HashSet; @@ -33,7 +34,8 @@ import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation; * If the individual qualifies, execute the SPARQL queries and add the results * to the specified search fields. * - * If there are no specified search fields, ALLTEXT is assumed. + * If there are no specified search fields, ALLTEXT and ALLTEXTUNSTEMMED are + * assumed. * * An individual qualifies if it satisfies any of the type restrictions, or if * there are no type restrictions. @@ -63,7 +65,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier, /** * The names of the fields where the results of the queries will be stored. - * If empty, it is assumed to be ALLTEXT. + * If empty, it is assumed to be ALLTEXT and ALLTEXTUNSTEMMED. */ private List fieldNames = new ArrayList<>(); @@ -105,6 +107,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier, } if (fieldNames.isEmpty()) { fieldNames.add(ALLTEXT); + fieldNames.add(ALLTEXTUNSTEMMED); } if (queries.isEmpty()) { throw new IllegalStateException( diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java index c0840f05b..9b147385f 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java @@ -145,7 +145,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: IdUriLabel"; + return "Internal: IdUriLabel"; } } @@ -184,7 +184,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddClasses"; + return "Internal: AddClasses"; } } @@ -203,7 +203,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddMostSpecificTypes"; + return "Internal: AddMostSpecificTypes"; } } @@ -212,21 +212,18 @@ public class UpdateDocumentWorkUnit implements Runnable { public void modifyDocument(Individual ind, SearchInputDocument doc) { List stmts = ind .getObjectPropertyStatements(); - if (stmts == null) { - return; - } - - for (ObjectPropertyStatement stmt : stmts) { - if (URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) { - continue; + if (stmts != null) { + for (ObjectPropertyStatement stmt : stmts) { + if (!URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) { + addToAlltext(doc, stmt.getObject().getRdfsLabel()); + } } - addToAlltext(doc, stmt.getObject().getRdfsLabel()); } } @Override public String toString() { - return "REQUIRED: AddObjectPropertyText"; + return "Internal: AddObjectPropertyText"; } } @@ -234,21 +231,18 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public void modifyDocument(Individual ind, SearchInputDocument doc) { List stmts = ind.getDataPropertyStatements(); - if (stmts == null) { - return; - } - - for (DataPropertyStatement stmt : stmts) { - if (stmt.getDatapropURI().equals(URI_RDFS_LABEL)) { - continue; + if (stmts != null) { + for (DataPropertyStatement stmt : stmts) { + if (!stmt.getDatapropURI().equals(URI_RDFS_LABEL)) { + addToAlltext(doc, stmt.getData()); + } } - addToAlltext(doc, stmt.getData()); } } @Override public String toString() { - return "REQUIRED: AddDataPropertyText"; + return "Internal: AddDataPropertyText"; } } @@ -263,7 +257,7 @@ public class UpdateDocumentWorkUnit implements Runnable { @Override public String toString() { - return "REQUIRED: AddEntityBoost"; + return "Internal: AddEntityBoost"; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java index dcf489fa6..b6aa98894 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java @@ -3,8 +3,8 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.tasks; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_STATEMENTS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_STATEMENTS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_STATEMENTS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_STATEMENTS; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_STMTS; import java.util.ArrayList; @@ -82,14 +82,12 @@ public class UpdateStatementsTask implements Task { @Override public void run() { - listeners - .fireEvent(new Event(START_PROCESSING_STATEMENTS, getStatus())); + listeners.fireEvent(new Event(START_STATEMENTS, getStatus())); findAffectedUris(); - listeners.fireEvent(new Event(PROGRESS, getStatus())); updateTheUris(); - listeners.fireEvent(new Event(STOP_PROCESSING_STATEMENTS, getStatus())); + listeners.fireEvent(new Event(STOP_STATEMENTS, getStatus())); } private void findAffectedUris() { diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java index 9634616f5..3b61e29b0 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java @@ -3,8 +3,8 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.tasks; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_URIS; -import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_URIS; +import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_URIS; import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_URIS; import java.util.Collection; @@ -21,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine; +import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus; @@ -43,6 +44,9 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExclud * * Deletions are done synchronously, but updates are scheduled to run on the * thread pool. + * + * Commit requests are issued to the SearchEngine at each progress event and + * again at the end of the task. */ public class UpdateUrisTask implements Task { private static final Log log = LogFactory.getLog(UpdateUrisTask.class); @@ -67,7 +71,7 @@ public class UpdateUrisTask implements Task { this.listeners = listeners; this.pool = pool; - this.status = new Status(uris.size(), 500, listeners); + this.status = new Status(this, uris.size(), 500); this.searchEngine = ApplicationUtils.instance().getSearchEngine(); @@ -75,7 +79,7 @@ public class UpdateUrisTask implements Task { @Override public void run() { - listeners.fireEvent(new Event(START_PROCESSING_URIS, status + listeners.fireEvent(new Event(START_URIS, status .getSearchIndexerStatus())); excluders.startIndexing(); modifiers.startIndexing(); @@ -99,9 +103,11 @@ public class UpdateUrisTask implements Task { } pool.waitUntilIdle(); + commitChanges(); + excluders.stopIndexing(); modifiers.stopIndexing(); - listeners.fireEvent(new Event(STOP_PROCESSING_URIS, status + listeners.fireEvent(new Event(STOP_URIS, status .getSearchIndexerStatus())); } @@ -160,6 +166,21 @@ public class UpdateUrisTask implements Task { log.debug("scheduled update to " + ind); } + private void fireEvent(Event event) { + listeners.fireEvent(event); + if (event.getType() == PROGRESS || event.getType() == STOP_URIS) { + commitChanges(); + } + } + + private void commitChanges() { + try { + searchEngine.commit(); + } catch (SearchEngineException e) { + log.warn("Failed to commit changes.", e); + } + } + @Override public void notifyWorkUnitCompletion(Runnable workUnit) { log.debug("completed update to " @@ -181,18 +202,18 @@ public class UpdateUrisTask implements Task { * synchronized. */ private static class Status { + private final UpdateUrisTask parent; private final int total; private final int progressInterval; - private final ListenerList listeners; private int updated = 0; private int deleted = 0; private int excluded = 0; private Date since = new Date(); - public Status(int total, int progressInterval, ListenerList listeners) { + public Status(UpdateUrisTask parent, int total, int progressInterval) { + this.parent = parent; this.total = total; this.progressInterval = progressInterval; - this.listeners = listeners; } public synchronized void incrementUpdates() { @@ -213,8 +234,7 @@ public class UpdateUrisTask implements Task { private void maybeFireProgressEvent() { if (updated > 0 && updated % progressInterval == 0) { - listeners.fireEvent(new Event(PROGRESS, - getSearchIndexerStatus())); + parent.fireEvent(new Event(PROGRESS, getSearchIndexerStatus())); } } @@ -241,7 +261,7 @@ public class UpdateUrisTask implements Task { @Override public String toString() { - return "ExcludeIfNoVClasses"; + return "Internal: ExcludeIfNoVClasses"; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java new file mode 100644 index 000000000..383d509d8 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java @@ -0,0 +1,58 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.utils.logging; + +import org.apache.log4j.Layout; +import org.apache.log4j.spi.LoggingEvent; + +/** + * This is a nasty Layout that prints a full stack trace for every logging + * event. + * + * Here is an example of how to use this to show full stack traces every time we + * get a log message from org.apache.jena.riot. + * + * 
+ * log4j.appender.RiotAppender=org.apache.log4j.RollingFileAppender 
+ * log4j.appender.RiotAppender.File= $${catalina.home}/logs/${webapp.name}.riot.log
+ * log4j.appender.RiotAppender.MaxFileSize=10MB 
+ * log4j.appender.RiotAppender.MaxBackupIndex=10 
+ * log4j.appender.RiotAppender.layout=edu.cornell.mannlib.vitro.webapp.utils.logging.StackTraceLayout 
+ * log4j.logger.org.apache.jena.riot=INFO, RiotAppender
+ *
+ */ +public class StackTraceLayout extends Layout { + + public StackTraceLayout() { + } + + @Override + public void activateOptions() { + } + + /** + * Print the level, the message, and the full stack trace. + */ + @Override + public String format(LoggingEvent event) { + StringBuilder buffer = new StringBuilder(); + buffer.append(event.getLevel().toString()).append(" - ") + .append(event.getRenderedMessage()).append(LINE_SEP); + for (StackTraceElement traceElement : Thread.currentThread() + .getStackTrace()) { + buffer.append(" ").append(traceElement.toString()) + .append(LINE_SEP); + } + return buffer.toString(); + } + + /** + * The StackTraceLayout does not handle the throwable contained within + * LoggingEvents. Thus, it returns true. + */ + @Override + public boolean ignoresThrowable() { + return true; + } + +}