litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixing problems with too many threads for indexing.

Browse source
This commit is contained in:
bdc34 2010-07-08 23:43:51 +00:00
parent 499511d5ef
commit 8f4b448dcb
12 changed files with 367 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

100
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/edit/EntityRetryController.java
View file

@ -2,57 +2,55 @@
package edu.cornell.mannlib.vitro.webapp.controller.edit; package edu.cornell.mannlib.vitro.webapp.controller.edit;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder; import java.net.URLEncoder;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Calendar; import java.util.Calendar;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.TimeZone; import java.util.TimeZone;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.map.ListOrderedMap; import org.apache.commons.collections.map.ListOrderedMap;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vedit.beans.DynamicField; import edu.cornell.mannlib.vedit.beans.DynamicField;
import edu.cornell.mannlib.vedit.beans.DynamicFieldRow; import edu.cornell.mannlib.vedit.beans.DynamicFieldRow;
import edu.cornell.mannlib.vedit.beans.EditProcessObject; import edu.cornell.mannlib.vedit.beans.EditProcessObject;
import edu.cornell.mannlib.vedit.beans.FormObject; import edu.cornell.mannlib.vedit.beans.FormObject;
import edu.cornell.mannlib.vedit.beans.LoginFormBean; import edu.cornell.mannlib.vedit.beans.LoginFormBean;
import edu.cornell.mannlib.vedit.beans.Option; import edu.cornell.mannlib.vedit.beans.Option;
import edu.cornell.mannlib.vedit.controller.BaseEditController; import edu.cornell.mannlib.vedit.controller.BaseEditController;
import edu.cornell.mannlib.vedit.forwarder.PageForwarder; import edu.cornell.mannlib.vedit.forwarder.PageForwarder;
import edu.cornell.mannlib.vedit.forwarder.impl.UrlForwarder; import edu.cornell.mannlib.vedit.forwarder.impl.UrlForwarder;
import edu.cornell.mannlib.vedit.util.FormUtils; import edu.cornell.mannlib.vedit.util.FormUtils;
import edu.cornell.mannlib.vedit.validator.impl.RequiredFieldValidator; import edu.cornell.mannlib.vedit.validator.impl.RequiredFieldValidator;
import edu.cornell.mannlib.vitro.webapp.auth.policy.JenaNetidPolicy.ContextSetup; import edu.cornell.mannlib.vitro.webapp.beans.DataProperty;
import edu.cornell.mannlib.vitro.webapp.beans.DataProperty; import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl; import edu.cornell.mannlib.vitro.webapp.beans.Portal;
import edu.cornell.mannlib.vitro.webapp.beans.Portal; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; import edu.cornell.mannlib.vitro.webapp.controller.Controllers;
import edu.cornell.mannlib.vitro.webapp.controller.Controllers; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.edit.utils.RoleLevelOptionsSetup;
import edu.cornell.mannlib.vitro.webapp.controller.edit.utils.RoleLevelOptionsSetup; import edu.cornell.mannlib.vitro.webapp.dao.DataPropertyDao;
import edu.cornell.mannlib.vitro.webapp.dao.DataPropertyDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao; import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao; import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.edit.listener.impl.IndividualDataPropertyStatementProcessor;
import edu.cornell.mannlib.vitro.webapp.edit.listener.impl.IndividualDataPropertyStatementProcessor;
import edu.cornell.mannlib.vitro.webapp.edit.listener.impl.SearchReindexer;
public class EntityRetryController extends BaseEditController { public class EntityRetryController extends BaseEditController {

2
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/edit/UpdateEntityFlagServlet.java
View file

@ -133,7 +133,7 @@ public class UpdateEntityFlagServlet extends VitroHttpServlet {
private void updateSearchIndex(HttpServletRequest request){ private void updateSearchIndex(HttpServletRequest request){
IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName()); IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName());
if( builder != null ) if( builder != null )
(new Thread(builder)).start(); builder.doUpdateIndex();
} }
} }

24
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java
View file

@ -23,15 +23,13 @@ import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
* This class is thread safe. Notice that doAsyncIndexBuild() is frequently * This class is thread safe. Notice that doAsyncIndexBuild() is frequently
* called because the inference system does not seem to send notifyEvents. * called because the inference system does not seem to send notifyEvents.
*/ */
public class SearchReindexingListener implements ModelChangedListener { public class SearchReindexingListener implements ModelChangedListener {
private HashSet<String> changedUris;
private IndexBuilder indexBuilder; private IndexBuilder indexBuilder;
public SearchReindexingListener(IndexBuilder indexBuilder) { public SearchReindexingListener(IndexBuilder indexBuilder) {
if(indexBuilder == null ) if(indexBuilder == null )
throw new IllegalArgumentException("Constructor parameter indexBuilder must not be null"); throw new IllegalArgumentException("Constructor parameter indexBuilder must not be null");
this.indexBuilder = indexBuilder; this.indexBuilder = indexBuilder;
this.changedUris = new HashSet<String>();
} }
private synchronized void addChange(Statement stmt){ private synchronized void addChange(Statement stmt){
@ -49,8 +47,8 @@ public class SearchReindexingListener implements ModelChangedListener {
} }
} }
private void doAsyncIndexBuild(){ private void requestAsyncIndexUpdate(){
new Thread(indexBuilder).start(); indexBuilder.doUpdateIndex();
} }
@Override @Override
@ -59,7 +57,7 @@ public class SearchReindexingListener implements ModelChangedListener {
EditEvent editEvent = (EditEvent)arg1; EditEvent editEvent = (EditEvent)arg1;
if( !editEvent.getBegin() ){// editEvent is the end of an edit if( !editEvent.getBegin() ){// editEvent is the end of an edit
log.debug("Doing search index build at end of EditEvent"); log.debug("Doing search index build at end of EditEvent");
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
} else{ } else{
log.debug("ignoring event " + arg1.getClass().getName() + " "+ arg1 ); log.debug("ignoring event " + arg1.getClass().getName() + " "+ arg1 );
@ -69,13 +67,13 @@ public class SearchReindexingListener implements ModelChangedListener {
@Override @Override
public void addedStatement(Statement stmt) { public void addedStatement(Statement stmt) {
addChange(stmt); addChange(stmt);
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
@Override @Override
public void removedStatement(Statement stmt){ public void removedStatement(Statement stmt){
addChange(stmt); addChange(stmt);
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
private static final Log log = LogFactory.getLog(SearchReindexingListener.class.getName()); private static final Log log = LogFactory.getLog(SearchReindexingListener.class.getName());
@ -85,7 +83,7 @@ public class SearchReindexingListener implements ModelChangedListener {
for( Statement s: arg0){ for( Statement s: arg0){
addChange(s); addChange(s);
} }
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
@Override @Override
@ -93,7 +91,7 @@ public class SearchReindexingListener implements ModelChangedListener {
for( Statement s: arg0){ for( Statement s: arg0){
addChange(s); addChange(s);
} }
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
@Override @Override
@ -106,7 +104,7 @@ public class SearchReindexingListener implements ModelChangedListener {
}finally{ }finally{
arg0.close(); arg0.close();
} }
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
@Override @Override
@ -122,7 +120,7 @@ public class SearchReindexingListener implements ModelChangedListener {
if( it != null ) it.close(); if( it != null ) it.close();
m.leaveCriticalSection(); m.leaveCriticalSection();
} }
doAsyncIndexBuild(); requestAsyncIndexUpdate();
} }
@Override @Override

2
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/listener/impl/KeywordSearchReindexer.java
View file

@ -10,7 +10,7 @@ public class KeywordSearchReindexer implements ChangeListener {
public void doInserted(Object newObj, EditProcessObject epo){ public void doInserted(Object newObj, EditProcessObject epo){
IndexBuilder builder = (IndexBuilder)epo.getSession().getServletContext().getAttribute(IndexBuilder.class.getName()); IndexBuilder builder = (IndexBuilder)epo.getSession().getServletContext().getAttribute(IndexBuilder.class.getName());
(new Thread(builder)).start(); builder.doUpdateIndex();
} }
public void doUpdated(Object oldObj, Object newObj, EditProcessObject epo){ public void doUpdated(Object oldObj, Object newObj, EditProcessObject epo){

26
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/listener/impl/SearchReindexer.java
View file

@ -1,26 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.edit.listener.impl;
import edu.cornell.mannlib.vedit.beans.EditProcessObject;
import edu.cornell.mannlib.vedit.listener.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
public class SearchReindexer implements ChangeListener {
public void doInserted(Object newObj, EditProcessObject epo){
IndexBuilder builder = (IndexBuilder)epo.getSession().getServletContext().getAttribute(IndexBuilder.class.getName());
(new Thread(builder)).start();
}
public void doUpdated(Object oldObj, Object newObj, EditProcessObject epo){
doInserted(newObj, epo);
}
public void doDeleted(Object oldObj, EditProcessObject epo){
IndexBuilder builder = (IndexBuilder)epo.getSession().getServletContext().getAttribute(IndexBuilder.class.getName());
builder.entityDeleted(((Individual)oldObj).getURI());
}
}

8
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java
View file

@ -20,7 +20,7 @@ import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
/** /**
* Acepts requests to rebuild or update the search index. It uses * Accepts requests to rebuild or update the search index. It uses
* an IndexBuilder and finds that IndexBuilder from the servletContext using * an IndexBuilder and finds that IndexBuilder from the servletContext using
* the key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder" * the key "edu.cornel.mannlib.vitro.search.indexing.IndexBuilder"
* *
@ -61,12 +61,8 @@ public class IndexController extends HttpServlet {
IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName()); IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName());
if( request.getParameter("update") != null ){ if( request.getParameter("update") != null ){
builder.doUpdateIndex(); builder.doUpdateIndex();
}
if (request.getParameter("clear") != null ) {
builder.clearIndex();
}else{ }else{
builder.doIndexBuild(); builder.doIndexRebuild();
} }
} catch (IndexingException e) { } catch (IndexingException e) {

264
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -2,7 +2,9 @@
package edu.cornell.mannlib.vitro.webapp.search.indexing; package edu.cornell.mannlib.vitro.webapp.search.indexing;
import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
@ -28,8 +30,9 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
* It uses an implementation of a back-end through an object that * It uses an implementation of a back-end through an object that
* implements IndexerIface. An example of a back-end is LuceneIndexer. * implements IndexerIface. An example of a back-end is LuceneIndexer.
* *
* The IndexBuilder implements the EntityChangeListener so it can * See the class SearchReindexingListener for an example of how a model change
* be registered for Entity changes from the GenericDB classes. * listener can use an IndexBuilder to keep the full text index in sncy with
* updates to a model.
* *
* There should be an IndexBuilder in the servlet context, try: * There should be an IndexBuilder in the servlet context, try:
* *
@ -40,19 +43,28 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
* @author bdc34 * @author bdc34
* *
*/ */
public class IndexBuilder implements Runnable { public class IndexBuilder {
List<ObjectSourceIface> sourceList = new LinkedList<ObjectSourceIface>(); private List<ObjectSourceIface> sourceList = new LinkedList<ObjectSourceIface>();
IndexerIface indexer = null; private IndexerIface indexer = null;
ServletContext context = null; private ServletContext context = null;
ProhibitedFromSearch classesProhibitedFromSearch = null; private ProhibitedFromSearch classesProhibitedFromSearch = null;
long lastRun = 0; private long lastRun = 0;
Collection<String> changedUris = null;
private HashSet<String> changedUris = null;
private List<Individual> updatedInds = null;
private List<Individual> deletedInds = null;
private IndexBuilderThread indexingThread = null;
//shared with IndexBuilderThread
private boolean reindexRequested = false;
public static final boolean UPDATE_DOCS = false; public static final boolean UPDATE_DOCS = false;
public static final boolean NEW_DOCS = true; public static final boolean NEW_DOCS = true;
private static final Log log = LogFactory.getLog(IndexBuilder.class.getName()); private static final Log log = LogFactory.getLog(IndexBuilder.class);
public IndexBuilder(ServletContext context, public IndexBuilder(ServletContext context,
IndexerIface indexer, IndexerIface indexer,
@ -61,7 +73,9 @@ public class IndexBuilder implements Runnable {
this.sourceList = sources; this.sourceList = sources;
this.context = context; this.context = context;
changedUris = new HashSet<String>(); this.changedUris = new HashSet<String>();
this.indexingThread = new IndexBuilderThread(this);
this.indexingThread.start();
} }
public void addObjectSource(ObjectSourceIface osi) { public void addObjectSource(ObjectSourceIface osi) {
@ -77,9 +91,69 @@ public class IndexBuilder implements Runnable {
return sourceList; return sourceList;
} }
public void doIndexBuild() throws IndexingException { public void doIndexRebuild() throws IndexingException {
log.debug(this.getClass().getName() //set up full index rebuild
+ " performing doFullRebuildIndex()\n"); setReindexRequested( true );
//wake up indexing thread
synchronized (this.indexingThread) {
this.indexingThread.notifyAll();
}
}
/**
* This will re-index Individuals that changed because of modtime or because they
* were added with addChangedUris().
*/
public void doUpdateIndex() {
//wake up thread
synchronized (this.indexingThread) {
this.indexingThread.notifyAll();
}
}
public synchronized void addToChangedUris(String uri){
changedUris.add(uri);
}
public synchronized void addToChangedUris(Collection<String> uris){
changedUris.addAll(uris);
}
public synchronized boolean isReindexRequested() {
return reindexRequested;
}
public synchronized boolean isThereWorkToDo(){
return isReindexRequested() || ! changedUris.isEmpty() ;
}
public ProhibitedFromSearch getClassesProhibitedFromSearch() {
return classesProhibitedFromSearch;
}
public void setClassesProhibitedFromSearch(
ProhibitedFromSearch classesProhibitedFromSearch) {
this.classesProhibitedFromSearch = classesProhibitedFromSearch;
}
public void killIndexingThread() {
this.indexingThread.kill();
}
/* ******************** non-public methods ************************* */
private synchronized void setReindexRequested(boolean reindexRequested) {
this.reindexRequested = reindexRequested;
}
private synchronized Collection<String> getAndEmptyChangedUris(){
Collection<String> out = changedUris;
changedUris = new HashSet<String>();
return out;
}
protected void indexRebuild() throws IndexingException {
setReindexRequested(false);
log.debug("performing indexRebuild()");
Iterator<ObjectSourceIface> sources = sourceList.iterator(); Iterator<ObjectSourceIface> sources = sourceList.iterator();
List listOfIterators = new LinkedList(); List listOfIterators = new LinkedList();
@ -98,43 +172,68 @@ public class IndexBuilder implements Runnable {
getAndEmptyChangedUris(); getAndEmptyChangedUris();
if( listOfIterators.size() == 0){ log.debug("Warning: no ObjectSources found.");} if( listOfIterators.size() == 0){ log.debug("Warning: no ObjectSources found.");}
doBuild( listOfIterators, true, NEW_DOCS );
log.debug(this.getClass().getName() + ".doFullRebuildIndex() Done \n");
}
public void run() {
doUpdateIndex();
}
public void doUpdateIndex() {
long since = indexer.getModified() - 60000;
Iterator<ObjectSourceIface> sources = sourceList.iterator();
List<Iterator<Individual>> listOfIterators = doBuild( listOfIterators, Collections.EMPTY_LIST, true, NEW_DOCS );
new LinkedList<Iterator<Individual>>(); log.debug(this.getClass().getName() + ".doFullRebuildIndex() Done \n");
while (sources.hasNext()) {
Object obj = sources.next();
if (obj != null && obj instanceof ObjectSourceIface)
listOfIterators.add((((ObjectSourceIface) obj)
.getUpdatedSinceIterator(since)));
else
log.debug("\tskipping object of class "
+ obj.getClass().getName() + "\n"
+ "\tIt doesn not implement " + "ObjectSourceIface.\n");
}
List<Individual> changedInds = addDepResourceClasses(checkForDeletes(getAndEmptyChangedUris()));
listOfIterators.add( (new IndexBuilder.BuilderObjectSource(changedInds)).getUpdatedSinceIterator(0) );
doBuild( listOfIterators, false, UPDATE_DOCS );
} }
protected void updatedIndex() throws IndexingException{
log.debug("Starting updateIndex()");
long since = indexer.getModified() - 60000;
Iterator<ObjectSourceIface> sources = sourceList.iterator();
List<Iterator<Individual>> listOfIterators =
new LinkedList<Iterator<Individual>>();
while (sources.hasNext()) {
Object obj = sources.next();
if (obj != null && obj instanceof ObjectSourceIface)
listOfIterators.add((((ObjectSourceIface) obj)
.getUpdatedSinceIterator(since)));
else
log.debug("\tskipping object of class "
+ obj.getClass().getName() + "\n"
+ "\tIt doesn not implement " + "ObjectSourceIface.\n");
}
buildAddAndDeleteLists( getAndEmptyChangedUris());
listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) );
doBuild( listOfIterators, deletedInds, false, UPDATE_DOCS );
}
/**
* Sets updatedUris and deletedUris.
* @param changedUris
*/
private void buildAddAndDeleteLists( Collection<String> uris){
/* clear updateInds and deletedUris. This is the only method that should set these. */
this.updatedInds = new ArrayList<Individual>();
this.deletedInds = new ArrayList<Individual>();
WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory");
for( String uri: uris){
if( uri != null ){
Individual ind = wdf.getIndividualDao().getIndividualByURI(uri);
if( ind != null)
this.updatedInds.add(ind);
else{
log.debug("found delete in changed uris");
this.deletedInds.add(ind);
}
}
}
this.updatedInds = addDepResourceClasses(updatedInds);
}
private List<Individual> addDepResourceClasses(List<Individual> inds) { private List<Individual> addDepResourceClasses(List<Individual> inds) {
WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory"); WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory");
VClassDao vClassDao = wdf.getVClassDao(); VClassDao vClassDao = wdf.getVClassDao();
java.util.ListIterator<Individual> it = inds.listIterator(); Iterator<Individual> it = inds.iterator();
VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE); VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE);
while(it.hasNext()){ while(it.hasNext()){
Individual ind = it.next(); Individual ind = it.next();
@ -166,14 +265,6 @@ public class IndexBuilder implements Runnable {
} }
return inds; return inds;
} }
public void clearIndex(){
try {
indexer.clearIndex();
} catch (IndexingException e) {
log.error("error while clearing index", e);
}
}
/** /**
* For each sourceIterator, get all of the objects and attempt to * For each sourceIterator, get all of the objects and attempt to
@ -189,12 +280,17 @@ public class IndexBuilder implements Runnable {
* to false, and a check is made before adding, it will work fine; but * to false, and a check is made before adding, it will work fine; but
* checking if an object is on the index is slow. * checking if an object is on the index is slow.
*/ */
private void doBuild(List sourceIterators, boolean wipeIndexFirst, boolean newDocs ){ private void doBuild(List sourceIterators, Collection<Individual> deletes, boolean wipeIndexFirst, boolean newDocs ){
try { try {
indexer.startIndexing(); indexer.startIndexing();
if( wipeIndexFirst ) if( wipeIndexFirst )
indexer.clearIndex(); indexer.clearIndex();
else{
for(Individual deleteMe : deletes ){
indexer.removeFromIndex(deleteMe);
}
}
//get an iterator for all of the sources of indexable objects //get an iterator for all of the sources of indexable objects
Iterator sourceIters = sourceIterators.iterator(); Iterator sourceIters = sourceIterators.iterator();
@ -224,36 +320,19 @@ public class IndexBuilder implements Runnable {
* @param items * @param items
* @return * @return
*/ */
protected void indexForSource(Iterator<Individual> individuals , boolean newDocs){ private void indexForSource(Iterator<Individual> individuals , boolean newDocs){
if( individuals == null ) return; if( individuals == null ) return;
while(individuals.hasNext()){ while(individuals.hasNext()){
indexItem(individuals.next(), newDocs); indexItem(individuals.next(), newDocs);
} }
} }
private List<Individual> checkForDeletes(List<String> uris){
WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory");
List<Individual> nonDeletes = new LinkedList<Individual>();
for( String uri: uris){
if( uri != null ){
Individual ind = wdf.getIndividualDao().getIndividualByURI(uri);
if( ind != null)
nonDeletes.add(ind);
else{
log.debug("found delete in changed uris");
entityDeleted(uri);
}
}
}
return nonDeletes;
}
/** /**
* Use the backend indexer to index a single item. * Use the backend indexer to index a single item.
* @param item * @param item
* @return * @return
*/ */
protected void indexItem( Individual ind, boolean newDoc){ private void indexItem( Individual ind, boolean newDoc){
try{ try{
if( ind == null ) if( ind == null )
return; return;
@ -275,33 +354,7 @@ public class IndexBuilder implements Runnable {
+ ind + "\n" +ex); + ind + "\n" +ex);
} }
return ; return ;
} }
public void entityDeleted(String entityURI) {
if( log.isDebugEnabled())
log.debug("IndexBuilder.entityDeleted() " + entityURI);
Individual ent = new IndividualImpl(entityURI);
try {
indexer.removeFromIndex(ent);
} catch (IndexingException e) {
log.debug("IndexBuilder.entityDeleted failed: " + e);
}
}
public synchronized void addToChangedUris(String uri){
changedUris.add(uri);
}
public synchronized void addToChangedUris(Collection<String> uris){
changedUris.addAll(uris);
}
private synchronized List<String> getAndEmptyChangedUris(){
LinkedList<String> out = new LinkedList<String>();
out.addAll( changedUris );
changedUris = new HashSet<String>();
return out;
}
private class BuilderObjectSource implements ObjectSourceIface { private class BuilderObjectSource implements ObjectSourceIface {
private final List<Individual> individuals; private final List<Individual> individuals;
@ -330,12 +383,7 @@ public class IndexBuilder implements Runnable {
} }
} }
public ProhibitedFromSearch getClassesProhibitedFromSearch() {
return classesProhibitedFromSearch;
}
public void setClassesProhibitedFromSearch(
ProhibitedFromSearch classesProhibitedFromSearch) {
this.classesProhibitedFromSearch = classesProhibitedFromSearch;
}
} }

65
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThread.java Normal file
View file

@ -0,0 +1,65 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.indexing;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Thread that executes the methods in IndexBuilder.
*
* @author bdc34
*
*/
public class IndexBuilderThread extends Thread{
private IndexBuilder indexBuilder;
protected boolean stopRequested = false;
protected long reindexInterval = 1000 * 60 /* msec */ ;
private static final Log log = LogFactory.getLog(IndexBuilderThread.class.getName());
public IndexBuilderThread(IndexBuilder ib){
super("IndexBuilderThread");
this.indexBuilder = ib;
}
@Override
public void run() {
while(true){
if( stopRequested ){
log.info("Stopping IndexBuilderThread ");
return;
}
try{
if( indexBuilder.isReindexRequested() ){
log.debug("full re-index requested");
indexBuilder.indexRebuild();
}else{
log.debug("updated requested");
Thread.sleep(250);
indexBuilder.updatedIndex();
}
}catch (Throwable e) {
log.error(e,e);
}
if( indexBuilder != null && ! indexBuilder.isThereWorkToDo() ){
log.debug("there is no indexing working to do, going to sleep");
try {
synchronized (this) {
this.wait(reindexInterval);
}
} catch (InterruptedException e) {
log.debug(" woken up",e);
}
}
}
}
public synchronized void kill(){
log.debug("Attempting to kill IndexBuilderThread ");
stopRequested = true;
this.notifyAll();
}
}

68
webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneIndexer.java
View file

@ -2,29 +2,30 @@
package edu.cornell.mannlib.vitro.webapp.search.lucene; package edu.cornell.mannlib.vitro.webapp.search.lucene;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Iterator; import java.util.HashSet;
import java.util.LinkedList; import java.util.Iterator;
import java.util.List; import java.util.LinkedList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.Log;
import org.apache.lucene.analysis.Analyzer; import org.apache.commons.logging.LogFactory;
import org.apache.lucene.document.Document; import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.IndexReader; import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory; import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException; import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher; import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher;
import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface; import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexerIface;
/** /**
* *
@ -39,7 +40,8 @@ public class LuceneIndexer implements IndexerIface {
Analyzer analyzer = null; Analyzer analyzer = null;
List<Searcher> searchers = null; List<Searcher> searchers = null;
IndexWriter writer = null; IndexWriter writer = null;
boolean indexing = false; boolean indexing = false;
HashSet<String> urisIndexed;
//JODA timedate library can use java date format strings. //JODA timedate library can use java date format strings.
//http://java.sun.com/j2se/1.3/docs/api/java/text/SimpleDateFormat.html //http://java.sun.com/j2se/1.3/docs/api/java/text/SimpleDateFormat.html
@ -135,7 +137,8 @@ public class LuceneIndexer implements IndexerIface {
if( writer == null ) if( writer == null )
writer = writer =
new IndexWriter(indexDir,analyzer,false, MAX_FIELD_LENGTH); new IndexWriter(indexDir,analyzer,false, MAX_FIELD_LENGTH);
indexing = true; indexing = true;
urisIndexed = new HashSet<String>();
} catch(Throwable ioe){ } catch(Throwable ioe){
try{ try{
makeNewIndex(); makeNewIndex();
@ -154,7 +157,8 @@ public class LuceneIndexer implements IndexerIface {
notifyAll(); notifyAll();
return; return;
} }
try { try {
urisIndexed = null;
log.info("ending index"); log.info("ending index");
if( writer != null ) if( writer != null )
writer.optimize(); writer.optimize();
@ -188,14 +192,22 @@ public class LuceneIndexer implements IndexerIface {
"startIndexing() before index()."); "startIndexing() before index().");
if( writer == null ) if( writer == null )
throw new IndexingException("LuceneIndexer: cannot build index," + throw new IndexingException("LuceneIndexer: cannot build index," +
"IndexWriter is null."); "IndexWriter is null.");
try { if( ind == null )
log.debug("Individual to index was null, ignoring.");
try {
if( urisIndexed.contains(ind.getURI()) ){
log.debug("already indexed " + ind.getURI() );
return;
}else
urisIndexed.add(ind.getURI());
Iterator<Obj2DocIface> it = getObj2DocList().iterator(); Iterator<Obj2DocIface> it = getObj2DocList().iterator();
while (it.hasNext()) { while (it.hasNext()) {
Obj2DocIface obj2doc = (Obj2DocIface) it.next(); Obj2DocIface obj2doc = (Obj2DocIface) it.next();
if (obj2doc.canTranslate(ind)) { if (obj2doc.canTranslate(ind)) {
Document d = (Document) obj2doc.translate(ind); Document d = (Document) obj2doc.translate(ind);
if( d != null){ if( d != null){
if( !newDoc ){ if( !newDoc ){
writer.updateDocument((Term)obj2doc.getIndexId(ind), d); writer.updateDocument((Term)obj2doc.getIndexId(ind), d);
log.debug("updated " + ind.getName() + " " + ind.getURI()); log.debug("updated " + ind.getName() + " " + ind.getURI());

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetup.java
View file

@ -143,8 +143,10 @@ public class LuceneSetup implements javax.servlet.ServletContextListener {
/** /**
* Gets run when the webApp Context gets destroyed. * Gets run when the webApp Context gets destroyed.
*/ */
public void contextDestroyed(ServletContextEvent sce) { public void contextDestroyed(ServletContextEvent sce) {
log.info("**** Running "+this.getClass().getName()+".contextDestroyed()"); log.info("**** Running "+this.getClass().getName()+".contextDestroyed()");
IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName());
builder.killIndexingThread();
} }
/** /**

7
webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetupCJK.java
View file

@ -126,8 +126,11 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener {
/** /**
* Gets run when the webApp Context gets destroyed. * Gets run when the webApp Context gets destroyed.
*/ */
public void contextDestroyed(ServletContextEvent sce) { public void contextDestroyed(ServletContextEvent sce) {
log.info("**** Running "+this.getClass().getName()+".contextDestroyed()");
log.info("**** Running "+this.getClass().getName()+".contextDestroyed()");
IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName());
builder.killIndexingThread();
} }
/** /**

33
webapp/test/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThreadTest.java Normal file
View file

@ -0,0 +1,33 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.indexing;
import junit.framework.Assert;
import org.apache.log4j.Level;
import org.junit.Test;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
public class IndexBuilderThreadTest extends AbstractTestClass {
@Test
public void testStoppingTheThread(){
setLoggerLevel(IndexBuilderThread.class, Level.OFF);
IndexBuilderThread ibt = new IndexBuilderThread(null);
ibt.start();
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
ibt.kill();
try {
Thread.sleep(100);
} catch (InterruptedException e) {
Assert.fail(e.getMessage());
}
Assert.assertFalse(ibt.isAlive());
}
}