litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-869 Cleanup: Committer, shorter event type, ALLTEXTUNSTEMMED, et. al.

Browse source 
Get rid of the SearchIndexerSetup.Committer listener class. This functionality becomes internal to UpdateUrisTask.
Use shorter names for Event.Type values, so the display will be more compact.
ALLTEXTUNSTEMMED is not automatically populated from ALLTEXT, so change the default target field on
    SimpleSparqlQueryDocumentModifier to include both of them.
Trivial cleanup in UpdateDocumentWorkUnit.
Create StackTraceLayout, a simple Log4J layout that will print the stack trace for each call.
    Used for diagnosing the source of uninformative log messages.
This commit is contained in:
Jim Blake 2015-01-26 10:18:02 -05:00
parent de5b80bf75
commit b3564e31bd
8 changed files with 117 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java
View file

@ -210,7 +210,7 @@ public class VClassGroupCache implements SearchIndexer.Listener {
@Override
public void receiveSearchIndexerEvent(Event event) {
switch (event.getType()) {
case STOP_PROCESSING_URIS:
case STOP_URIS:
log.debug("rebuilding because of IndexBuilder " + event.getType());
requestCacheUpdate();
break;

4
webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexer.java
View file

@ -152,9 +152,9 @@ public interface SearchIndexer extends Application.Module {
PAUSE, UNPAUSE,
START_PROCESSING_URIS, STOP_PROCESSING_URIS,
START_URIS, STOP_URIS,
START_PROCESSING_STATEMENTS, STOP_PROCESSING_STATEMENTS,
START_STATEMENTS, STOP_STATEMENTS,
START_REBUILD, STOP_REBUILD,

40
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java
View file

@ -2,9 +2,6 @@
package edu.cornell.mannlib.vitro.webapp.searchindex;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
@ -14,12 +11,7 @@ import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.modules.Application;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Listener;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.search.controller.IndexController;
@ -48,7 +40,6 @@ public class SearchIndexerSetup implements ServletContextListener {
private IndexingChangeListener listener;
private DeveloperDisabledChangeListener listenerWrapper;
private IndexHistory history;
private Committer committer;
@Override
public void contextInitialized(ServletContextEvent sce) {
@ -73,9 +64,6 @@ public class SearchIndexerSetup implements ServletContextListener {
searchIndexer.addListener(this.history);
IndexController.setHistory(this.history);
this.committer = new Committer();
searchIndexer.addListener(this.committer);
searchIndexer
.startup(app, new ComponentStartupStatusImpl(this, ss));
searchIndexer.unpause();
@ -90,7 +78,6 @@ public class SearchIndexerSetup implements ServletContextListener {
public void contextDestroyed(ServletContextEvent sce) {
searchIndexer.shutdown(app);
searchIndexer.removeListener(this.committer);
searchIndexer.removeListener(this.history);
try {
@ -102,31 +89,4 @@ public class SearchIndexerSetup implements ServletContextListener {
listener.shutdown();
}
// ----------------------------------------------------------------------
// Helper classes
// ----------------------------------------------------------------------
private static class Committer implements Listener {
private final SearchEngine searchEngine = ApplicationUtils.instance()
.getSearchEngine();
@Override
public void receiveSearchIndexerEvent(Event event) {
Type type = event.getType();
if (type == PROGRESS || type == STOP_PROCESSING_URIS) {
commitChanges();
}
}
private void commitChanges() {
try {
searchEngine.commit();
} catch (SearchEngineNotRespondingException e) {
log.error("Failed to commit the changes: "
+ "the search engine is not responding.");
} catch (Exception e) {
log.error("Failed to commit the changes.", e);
}
}
}
}

7
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/documentBuilding/SimpleSparqlQueryDocumentModifier.java
View file

@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.searchindex.documentBuilding;
import static edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess.WhichService.CONTENT;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXT;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.ALLTEXTUNSTEMMED;
import java.util.ArrayList;
import java.util.HashSet;
@ -33,7 +34,8 @@ import edu.cornell.mannlib.vitro.webapp.utils.configuration.Validation;
* If the individual qualifies, execute the SPARQL queries and add the results
* to the specified search fields.
*
* If there are no specified search fields, ALLTEXT is assumed.
* If there are no specified search fields, ALLTEXT and ALLTEXTUNSTEMMED are
* assumed.
*
* An individual qualifies if it satisfies any of the type restrictions, or if
* there are no type restrictions.
@ -63,7 +65,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier,
/**
* The names of the fields where the results of the queries will be stored.
* If empty, it is assumed to be ALLTEXT.
* If empty, it is assumed to be ALLTEXT and ALLTEXTUNSTEMMED.
*/
private List<String> fieldNames = new ArrayList<>();
@ -105,6 +107,7 @@ public class SimpleSparqlQueryDocumentModifier implements DocumentModifier,
}
if (fieldNames.isEmpty()) {
fieldNames.add(ALLTEXT);
fieldNames.add(ALLTEXTUNSTEMMED);
}
if (queries.isEmpty()) {
throw new IllegalStateException(

38
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateDocumentWorkUnit.java
View file

@ -145,7 +145,7 @@ public class UpdateDocumentWorkUnit implements Runnable {
@Override
public String toString() {
return "REQUIRED: IdUriLabel";
return "Internal: IdUriLabel";
}
}
@ -184,7 +184,7 @@ public class UpdateDocumentWorkUnit implements Runnable {
@Override
public String toString() {
return "REQUIRED: AddClasses";
return "Internal: AddClasses";
}
}
@ -203,7 +203,7 @@ public class UpdateDocumentWorkUnit implements Runnable {
@Override
public String toString() {
return "REQUIRED: AddMostSpecificTypes";
return "Internal: AddMostSpecificTypes";
}
}
@ -212,21 +212,18 @@ public class UpdateDocumentWorkUnit implements Runnable {
public void modifyDocument(Individual ind, SearchInputDocument doc) {
List<ObjectPropertyStatement> stmts = ind
.getObjectPropertyStatements();
if (stmts == null) {
return;
}
for (ObjectPropertyStatement stmt : stmts) {
if (URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) {
continue;
if (stmts != null) {
for (ObjectPropertyStatement stmt : stmts) {
if (!URI_DIFFERENT_FROM.equals(stmt.getPropertyURI())) {
addToAlltext(doc, stmt.getObject().getRdfsLabel());
}
}
addToAlltext(doc, stmt.getObject().getRdfsLabel());
}
}
@Override
public String toString() {
return "REQUIRED: AddObjectPropertyText";
return "Internal: AddObjectPropertyText";
}
}
@ -234,21 +231,18 @@ public class UpdateDocumentWorkUnit implements Runnable {
@Override
public void modifyDocument(Individual ind, SearchInputDocument doc) {
List<DataPropertyStatement> stmts = ind.getDataPropertyStatements();
if (stmts == null) {
return;
}
for (DataPropertyStatement stmt : stmts) {
if (stmt.getDatapropURI().equals(URI_RDFS_LABEL)) {
continue;
if (stmts != null) {
for (DataPropertyStatement stmt : stmts) {
if (!stmt.getDatapropURI().equals(URI_RDFS_LABEL)) {
addToAlltext(doc, stmt.getData());
}
}
addToAlltext(doc, stmt.getData());
}
}
@Override
public String toString() {
return "REQUIRED: AddDataPropertyText";
return "Internal: AddDataPropertyText";
}
}
@ -263,7 +257,7 @@ public class UpdateDocumentWorkUnit implements Runnable {
@Override
public String toString() {
return "REQUIRED: AddEntityBoost";
return "Internal: AddEntityBoost";
}
}

10
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java
View file

@ -3,8 +3,8 @@
package edu.cornell.mannlib.vitro.webapp.searchindex.tasks;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_STATEMENTS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_STATEMENTS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_STATEMENTS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_STATEMENTS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_STMTS;
import java.util.ArrayList;
@ -82,14 +82,12 @@ public class UpdateStatementsTask implements Task {
@Override
public void run() {
listeners
.fireEvent(new Event(START_PROCESSING_STATEMENTS, getStatus()));
listeners.fireEvent(new Event(START_STATEMENTS, getStatus()));
findAffectedUris();
listeners.fireEvent(new Event(PROGRESS, getStatus()));
updateTheUris();
listeners.fireEvent(new Event(STOP_PROCESSING_STATEMENTS, getStatus()));
listeners.fireEvent(new Event(STOP_STATEMENTS, getStatus()));
}
private void findAffectedUris() {

42
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java
View file

@ -3,8 +3,8 @@
package edu.cornell.mannlib.vitro.webapp.searchindex.tasks;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.PROGRESS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_PROCESSING_URIS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_PROCESSING_URIS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.START_URIS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_URIS;
import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_URIS;
import java.util.Collection;
@ -21,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineNotRespondingException;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event;
import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
@ -43,6 +44,9 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExclud
*
* Deletions are done synchronously, but updates are scheduled to run on the
* thread pool.
*
* Commit requests are issued to the SearchEngine at each progress event and
* again at the end of the task.
*/
public class UpdateUrisTask implements Task {
private static final Log log = LogFactory.getLog(UpdateUrisTask.class);
@ -67,7 +71,7 @@ public class UpdateUrisTask implements Task {
this.listeners = listeners;
this.pool = pool;
this.status = new Status(uris.size(), 500, listeners);
this.status = new Status(this, uris.size(), 500);
this.searchEngine = ApplicationUtils.instance().getSearchEngine();
@ -75,7 +79,7 @@ public class UpdateUrisTask implements Task {
@Override
public void run() {
listeners.fireEvent(new Event(START_PROCESSING_URIS, status
listeners.fireEvent(new Event(START_URIS, status
.getSearchIndexerStatus()));
excluders.startIndexing();
modifiers.startIndexing();
@ -99,9 +103,11 @@ public class UpdateUrisTask implements Task {
}
pool.waitUntilIdle();
commitChanges();
excluders.stopIndexing();
modifiers.stopIndexing();
listeners.fireEvent(new Event(STOP_PROCESSING_URIS, status
listeners.fireEvent(new Event(STOP_URIS, status
.getSearchIndexerStatus()));
}
@ -160,6 +166,21 @@ public class UpdateUrisTask implements Task {
log.debug("scheduled update to " + ind);
}
private void fireEvent(Event event) {
listeners.fireEvent(event);
if (event.getType() == PROGRESS || event.getType() == STOP_URIS) {
commitChanges();
}
}
private void commitChanges() {
try {
searchEngine.commit();
} catch (SearchEngineException e) {
log.warn("Failed to commit changes.", e);
}
}
@Override
public void notifyWorkUnitCompletion(Runnable workUnit) {
log.debug("completed update to "
@ -181,18 +202,18 @@ public class UpdateUrisTask implements Task {
* synchronized.
*/
private static class Status {
private final UpdateUrisTask parent;
private final int total;
private final int progressInterval;
private final ListenerList listeners;
private int updated = 0;
private int deleted = 0;
private int excluded = 0;
private Date since = new Date();
public Status(int total, int progressInterval, ListenerList listeners) {
public Status(UpdateUrisTask parent, int total, int progressInterval) {
this.parent = parent;
this.total = total;
this.progressInterval = progressInterval;
this.listeners = listeners;
}
public synchronized void incrementUpdates() {
@ -213,8 +234,7 @@ public class UpdateUrisTask implements Task {
private void maybeFireProgressEvent() {
if (updated > 0 && updated % progressInterval == 0) {
listeners.fireEvent(new Event(PROGRESS,
getSearchIndexerStatus()));
parent.fireEvent(new Event(PROGRESS, getSearchIndexerStatus()));
}
}
@ -241,7 +261,7 @@ public class UpdateUrisTask implements Task {
@Override
public String toString() {
return "ExcludeIfNoVClasses";
return "Internal: ExcludeIfNoVClasses";
}
}

58
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/logging/StackTraceLayout.java Normal file
View file

@ -0,0 +1,58 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.utils.logging;
import org.apache.log4j.Layout;
import org.apache.log4j.spi.LoggingEvent;
/**
* This is a nasty Layout that prints a full stack trace for every logging
* event.
*
* Here is an example of how to use this to show full stack traces every time we
* get a log message from org.apache.jena.riot.
*
* <pre>
* log4j.appender.RiotAppender=org.apache.log4j.RollingFileAppender
* log4j.appender.RiotAppender.File= $${catalina.home}/logs/${webapp.name}.riot.log
* log4j.appender.RiotAppender.MaxFileSize=10MB
* log4j.appender.RiotAppender.MaxBackupIndex=10
* log4j.appender.RiotAppender.layout=edu.cornell.mannlib.vitro.webapp.utils.logging.StackTraceLayout
* log4j.logger.org.apache.jena.riot=INFO, RiotAppender
* </pre>
*/
public class StackTraceLayout extends Layout {
public StackTraceLayout() {
}
@Override
public void activateOptions() {
}
/**
* Print the level, the message, and the full stack trace.
*/
@Override
public String format(LoggingEvent event) {
StringBuilder buffer = new StringBuilder();
buffer.append(event.getLevel().toString()).append(" - ")
.append(event.getRenderedMessage()).append(LINE_SEP);
for (StackTraceElement traceElement : Thread.currentThread()
.getStackTrace()) {
buffer.append(" ").append(traceElement.toString())
.append(LINE_SEP);
}
return buffer.toString();
}
/**
* The StackTraceLayout does not handle the throwable contained within
* LoggingEvents. Thus, it returns <code>true</code>.
*/
@Override
public boolean ignoresThrowable() {
return true;
}
}