From 1341d23a25c55d726e8b9d04cd1405056a73ab72 Mon Sep 17 00:00:00 2001 From: bdc34 Date: Wed, 9 Feb 2011 16:08:42 +0000 Subject: [PATCH] Changing the models that the SearchReindexingListener listens to. NIHVIVO-2076 --- .../vitro/webapp/dao/jena/ModelContext.java | 23 + .../dao/jena/SearchReindexingListener.java | 26 +- .../webapp/dao/jena/VClassGroupCache.java | 14 +- .../webapp/search/indexing/IndexBuilder.java | 414 +++++++++--------- .../search/indexing/IndexBuilderThread.java | 72 --- .../webapp/search/indexing/IndexerIface.java | 8 +- .../webapp/search/lucene/LuceneIndexer.java | 17 + .../webapp/search/lucene/LuceneSetup.java | 13 +- .../webapp/search/lucene/LuceneSetupCJK.java | 47 +- .../indexing/IndexBuilderThreadTest.java | 16 +- 10 files changed, 327 insertions(+), 323 deletions(-) delete mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThread.java diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java index 250837986..e9df05cde 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java @@ -5,6 +5,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena; import javax.servlet.ServletContext; import com.hp.hpl.jena.ontology.OntModel; +import com.hp.hpl.jena.rdf.model.ModelChangedListener; public class ModelContext { @@ -75,4 +76,26 @@ public class ModelContext { ctx.setAttribute(INFERENCE_ONT_MODEL, ontModel); } + /** + * Register a listener to the models needed to get changes to: + * class membership + * inferred class membership + * class group membership, + * object properties, + * data properties, + * inferred object properties, + * rdfs:label annotations + * This listener does not need: + * other annotations + * change to TBox + */ + public static void registerListenerForChanges(ServletContext ctx, ModelChangedListener ml){ + ModelContext.getJenaOntModel(ctx).register(ml); + ModelContext.getBaseOntModel(ctx).register(ml); + ModelContext.getInferenceOntModel(ctx).register(ml); + ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml); + ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml); + ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml); + } + } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java index c05536640..fb9c3004c 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java @@ -2,9 +2,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena; -import java.util.HashSet; import java.util.List; -import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,18 +30,34 @@ public class SearchReindexingListener implements ModelChangedListener { this.indexBuilder = indexBuilder; } - private synchronized void addChange(Statement stmt){ + private synchronized void addChange(Statement stmt){ if( stmt == null ) return; + if( log.isDebugEnabled() ){ + String sub="unknown"; + String pred = "unknown"; + String obj ="unknown"; + + if( stmt.getSubject().isURIResource() ){ + sub = stmt.getSubject().getURI(); + } + if( stmt.getPredicate() != null ){ + pred = stmt.getPredicate().getURI(); + } + if( stmt.getObject().isURIResource() ){ + obj = ((Resource) (stmt.getPredicate().as(Resource.class))).getURI(); + }else{ + obj = stmt.getObject().toString(); + } + log.debug("changed statement: sub='" + sub + "' pred='" + pred +"' obj='" + obj + "'"); + } + if( stmt.getSubject().isURIResource() ){ - //changedUris.add( stmt.getSubject().getURI()); indexBuilder.addToChangedUris(stmt.getSubject().getURI()); log.debug("subject: " + stmt.getSubject().getURI()); } if( stmt.getObject().isURIResource() ){ - //changedUris.add( ((Resource) stmt.getObject().as(Resource.class)).getURI() ); indexBuilder.addToChangedUris(((Resource) stmt.getObject()).getURI()); - log.debug("object: " + ((Resource) stmt.getObject().as(Resource.class)).getURI()); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java index 74711aa24..f629b9943 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java @@ -70,12 +70,14 @@ public class VClassGroupCache{ } VClassGroupCacheChangeListener bccl = new VClassGroupCacheChangeListener(this); - ModelContext.getJenaOntModel(context).register(bccl); - ModelContext.getBaseOntModel(context).register(bccl); - ModelContext.getInferenceOntModel(context).register(bccl); - ModelContext.getUnionOntModelSelector(context).getABoxModel().register(bccl); - ModelContext.getBaseOntModelSelector(context).getABoxModel().register(bccl); - ModelContext.getInferenceOntModelSelector(context).getABoxModel().register(bccl); + ModelContext.registerListenerForChanges(context, bccl); +// +// ModelContext.getJenaOntModel(context).register(bccl); +// ModelContext.getBaseOntModel(context).register(bccl); +// ModelContext.getInferenceOntModel(context).register(bccl); +// ModelContext.getUnionOntModelSelector(context).getABoxModel().register(bccl); +// ModelContext.getBaseOntModelSelector(context).getABoxModel().register(bccl); +// ModelContext.getInferenceOntModelSelector(context).getABoxModel().register(bccl); _rebuildQueue.add(REBUILD_EVERY_PORTAL); _cacheRebuildThread = new RebuildGroupCacheThread(this); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index f66a936d8..17c1cb26e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -26,38 +26,31 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch; /** * The IndexBuilder is used to rebuild or update a search index. + * There should only be one IndexBuilder in a vitro web application. * It uses an implementation of a back-end through an object that * implements IndexerIface. An example of a back-end is LuceneIndexer. * * See the class SearchReindexingListener for an example of how a model change * listener can use an IndexBuilder to keep the full text index in sncy with - * updates to a model. + * updates to a model. It calls IndexBuilder.addToChangedUris(). * - * There should be an IndexBuilder in the servlet context, try: - * - IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName()); - if( request.getParameter("update") != null ) - builder.doUpdateIndex(); - * @author bdc34 * */ -public class IndexBuilder { +public class IndexBuilder extends Thread { private List sourceList = new LinkedList(); private IndexerIface indexer = null; - private ServletContext context = null; + private ServletContext context = null; - private long lastRun = 0; - - private HashSet changedUris = null; + /* changedUris should only be accessed from synchronized blocks */ + private HashSet changedUris = null; private List updatedInds = null; - private List deletedInds = null; + private List deletedInds = null; - private IndexBuilderThread indexingThread = null; - - //shared with IndexBuilderThread - private boolean reindexRequested = false; + private boolean reindexRequested = false; + protected boolean stopRequested = false; + protected long reindexInterval = 1000 * 60 /* msec */ ; public static final boolean UPDATE_DOCS = false; public static final boolean NEW_DOCS = true; @@ -67,15 +60,20 @@ public class IndexBuilder { public IndexBuilder(ServletContext context, IndexerIface indexer, List /*ObjectSourceIface*/ sources){ + super("IndexBuilder"); this.indexer = indexer; this.sourceList = sources; this.context = context; - this.changedUris = new HashSet(); - this.indexingThread = new IndexBuilderThread(this); - this.indexingThread.start(); + this.changedUris = new HashSet(); + this.start(); } - + + protected IndexBuilder(){ + //for testing only + this( null, null, Collections.emptyList()); + } + public void addObjectSource(ObjectSourceIface osi) { if (osi != null) sourceList.add(osi); @@ -89,30 +87,26 @@ public class IndexBuilder { return sourceList; } - public void doIndexRebuild() throws IndexingException { - //set up full index rebuild - setReindexRequested( true ); - //wake up indexing thread - synchronized (this.indexingThread) { - this.indexingThread.notifyAll(); - } + public synchronized void doIndexRebuild() throws IndexingException { + //set flag for full index rebuild + this.reindexRequested = true; + //wake up + this.notifyAll(); } - /** + /** * This will re-index Individuals that changed because of modtime or because they * were added with addChangedUris(). */ - public void doUpdateIndex() { - //wake up thread - synchronized (this.indexingThread) { - this.indexingThread.notifyAll(); - } + public synchronized void doUpdateIndex() { + //wake up thread and it will attempt to index anything in changedUris + this.notifyAll(); } public synchronized void addToChangedUris(String uri){ changedUris.add(uri); } - + public synchronized void addToChangedUris(Collection uris){ changedUris.addAll(uris); } @@ -125,79 +119,47 @@ public class IndexBuilder { return isReindexRequested() || ! changedUris.isEmpty() ; } - public void killIndexingThread() { - this.indexingThread.kill(); + public synchronized void stopIndexingThread() { + stopRequested = true; + this.notifyAll(); } + + @Override + public void run() { + while(! stopRequested ){ + try{ + if( !stopRequested && isReindexRequested() ){ + log.debug("full re-index requested"); + indexRebuild(); + }else if( !stopRequested && isThereWorkToDo() ){ + Thread.sleep(250); //wait a bit to let a bit more work to come into the queue + log.debug("work found for IndexBuilder, starting update"); + updatedIndex(); + }else if( !stopRequested && ! isThereWorkToDo() ){ + log.debug("there is no indexing working to do, waiting for work"); + synchronized (this) { this.wait(reindexInterval); } + } + } catch (InterruptedException e) { + log.debug("woken up",e); + }catch(Throwable e){ + log.error(e,e); + } + } + log.info("Stopping IndexBuilder thread"); + } + + /* ******************** non-public methods ************************* */ - - private synchronized void setReindexRequested(boolean reindexRequested) { - this.reindexRequested = reindexRequested; - } - private synchronized Collection getAndEmptyChangedUris(){ Collection out = changedUris; changedUris = new HashSet(); return out; } - protected void indexRebuild() throws IndexingException { - log.info("Rebuild of search index is starting."); - - Iterator sources = sourceList.iterator(); - List listOfIterators = new LinkedList(); - while(sources.hasNext()){ - Object obj = sources.next(); - if( obj != null && obj instanceof ObjectSourceIface ) - listOfIterators.add((((ObjectSourceIface) obj) - .getAllOfThisTypeIterator())); - else - log.warn("\tskipping object of class " - + obj.getClass().getName() + "\n" - + "\tIt doesn not implement ObjectSourceIface.\n"); - } - - //clear out changed uris since we are doing a full index rebuild - getAndEmptyChangedUris(); - - if( listOfIterators.size() == 0){ log.warn("Warning: no ObjectSources found.");} - - setReindexRequested(false); - doBuild( listOfIterators, Collections.EMPTY_LIST, true, NEW_DOCS ); - log.info("Rebuild of search index is complete."); - } - - protected void updatedIndex() throws IndexingException{ - log.debug("Starting updateIndex()"); - long since = indexer.getModified() - 60000; - - Iterator sources = sourceList.iterator(); - - List> listOfIterators = - new LinkedList>(); - - while (sources.hasNext()) { - Object obj = sources.next(); - if (obj != null && obj instanceof ObjectSourceIface) - listOfIterators.add((((ObjectSourceIface) obj) - .getUpdatedSinceIterator(since))); - else - log.warn("\tskipping object of class " - + obj.getClass().getName() + "\n" - + "\tIt doesn not implement " + "ObjectSourceIface.\n"); - } - - buildAddAndDeleteLists( getAndEmptyChangedUris()); - listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) ); - - doBuild( listOfIterators, deletedInds, false, UPDATE_DOCS ); - log.debug("Ending updateIndex()"); - } - /** - * Sets updatedUris and deletedUris. - * @param changedUris + * Sets updatedUris and deletedUris lists. */ - private void buildAddAndDeleteLists( Collection uris){ + private void makeAddAndDeleteLists( Collection uris){ /* clear updateInds and deletedUris. This is the only method that should set these. */ this.updatedInds = new ArrayList(); this.deletedInds = new ArrayList(); @@ -216,45 +178,60 @@ public class IndexBuilder { } this.updatedInds = addDepResourceClasses(updatedInds); - } - + } + + protected void indexRebuild() throws IndexingException { + log.info("Rebuild of search index is starting."); - - private List addDepResourceClasses(List inds) { - WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory"); - VClassDao vClassDao = wdf.getVClassDao(); - Iterator it = inds.iterator(); - VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE); - while(it.hasNext()){ - Individual ind = it.next(); - List classes = ind.getVClasses(); - boolean isDepResource = false; - for( VClass clazz : classes){ - if( !isDepResource && VitroVocabulary.DEPENDENT_RESORUCE.equals( clazz.getURI() ) ){ - isDepResource = true; - break; - } - } - if( ! isDepResource ){ - for( VClass clazz : classes){ - List superClassUris = vClassDao.getAllSuperClassURIs(clazz.getURI()); - for( String uri : superClassUris){ - if( VitroVocabulary.DEPENDENT_RESORUCE.equals( uri ) ){ - isDepResource = true; - break; - } - } - if( isDepResource ) - break; - } - } - if( isDepResource){ - classes.add(depResVClass); - ind.setVClasses(classes, true); - } - } - return inds; - } + Iterator sources = sourceList.iterator(); + List listOfIterators = new LinkedList(); + while (sources.hasNext()) { + Object obj = sources.next(); + if (obj != null && obj instanceof ObjectSourceIface) + listOfIterators.add((((ObjectSourceIface) obj) + .getAllOfThisTypeIterator())); + else + log.warn("\tskipping object of class " + + obj.getClass().getName() + "\n" + + "\tIt doesn not implement ObjectSourceIface.\n"); + } + + // clear out changed uris since we are doing a full index rebuild + getAndEmptyChangedUris(); + + if (listOfIterators.size() == 0) + log.warn("Warning: no ObjectSources found."); + + doBuild(listOfIterators, Collections.EMPTY_LIST ); + if( log != null ) //log might be null if system is shutting down. + log.info("Rebuild of search index is complete."); + } + + protected void updatedIndex() throws IndexingException{ + log.debug("Starting updateIndex()"); + long since = indexer.getModified() - 60000; + + Iterator sources = sourceList.iterator(); + List> listOfIterators = + new LinkedList>(); + + while (sources.hasNext()) { + Object obj = sources.next(); + if (obj != null && obj instanceof ObjectSourceIface) + listOfIterators.add((((ObjectSourceIface) obj) + .getUpdatedSinceIterator(since))); + else + log.warn("\tskipping object of class " + + obj.getClass().getName() + "\n" + + "\tIt doesn not implement " + "ObjectSourceIface.\n"); + } + + makeAddAndDeleteLists( getAndEmptyChangedUris()); + listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) ); + + doBuild( listOfIterators, deletedInds ); + log.debug("Ending updateIndex()"); + } /** * For each sourceIterator, get all of the objects and attempt to @@ -270,17 +247,22 @@ public class IndexBuilder { * to false, and a check is made before adding, it will work fine; but * checking if an object is on the index is slow. */ - private void doBuild(List sourceIterators, Collection deletes, boolean forceNewIndex, boolean newDocs ){ + private void doBuild(List> sourceIterators, Collection deletes ){ + boolean aborted = false; + boolean newDocs = reindexRequested; + boolean forceNewIndex = reindexRequested; + try { - if( forceNewIndex ) + if( reindexRequested ) indexer.prepareForRebuild(); indexer.startIndexing(); - + reindexRequested = false; + if( ! forceNewIndex ){ - for(Individual deleteMe : deletes ){ - indexer.removeFromIndex(deleteMe); - } + for(Individual deleteMe : deletes ){ + indexer.removeFromIndex(deleteMe); + } } //get an iterator for all of the sources of indexable objects @@ -289,35 +271,50 @@ public class IndexBuilder { while (sourceIters.hasNext()) { obj = sourceIters.next(); if (obj == null || !(obj instanceof Iterator)) { - log.warn("\tskipping object of class " - + obj.getClass().getName() + "\n" - + "\tIt doesn not implement " - + "Iterator.\n"); + log.warn("skipping object of class " + obj.getClass().getName() + + "It doesn not implement Iterator."); continue; } indexForSource((Iterator)obj, newDocs); } - } catch (IndexingException ex) { - log.error(ex,ex); + } catch (AbortIndexing abort){ + if( log != null) + log.debug("aborting the indexing because thread stop was requested"); + aborted = true; } catch (Exception e) { log.error(e,e); - } finally { + } + + if( aborted && forceNewIndex ){ + indexer.abortIndexingAndCleanUp(); + }else{ indexer.endIndexing(); } + } /** * Use the back end indexer to index each object that the Iterator returns. - * @param items - * @return + * @throws AbortIndexing */ - private void indexForSource(Iterator individuals , boolean newDocs){ - if( individuals == null ) return; - + private void indexForSource(Iterator individuals , boolean newDocs) throws AbortIndexing{ long starttime = System.currentTimeMillis(); long count = 0; - while(individuals.hasNext()){ - indexItem(individuals.next(), newDocs); + while(individuals.hasNext()){ + if( stopRequested ) + throw new AbortIndexing(); + + Individual ind = null; + try{ + ind = individuals.next(); + indexer.index(ind, newDocs); + }catch(Throwable ex){ + if( stopRequested || log == null){//log might be null if system is shutting down. + throw new AbortIndexing(); + } + String uri = ind!=null?ind.getURI():"null"; + log.warn("Error indexing individual " + uri + " " + ex.getMessage()); + } count++; if( log.isDebugEnabled() ){ if( (count % 100 ) == 0 && count > 0 ){ @@ -330,55 +327,74 @@ public class IndexBuilder { log.info( "individuals indexed: " + count + " in " + (System.currentTimeMillis() - starttime) + " msec" + - (count!=0?(" time per individual = " + (System.currentTimeMillis() - starttime)/ count + " msec"):"") + (count!=0?(" time per individual = " + (System.currentTimeMillis() - starttime)/ count + " msec"):"") ); - } + } /** - * Use the backend indexer to index a single item. - * @param item - * @return + * For a list of individuals, this builds a list of dependent resources and returns it. */ - private void indexItem( Individual ind, boolean newDoc){ - try{ - if( ind != null ){ - indexer.index(ind, newDoc); - } - }catch(Throwable ex){ - log.warn("IndexBuilder.indexItem() Error indexing " - + ind + "\n" +ex); + private List addDepResourceClasses(List inds) { + WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory"); + VClassDao vClassDao = wdf.getVClassDao(); + Iterator it = inds.iterator(); + VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE); + while(it.hasNext()){ + Individual ind = it.next(); + List classes = ind.getVClasses(); + boolean isDepResource = false; + for( VClass clazz : classes){ + if( !isDepResource && VitroVocabulary.DEPENDENT_RESORUCE.equals( clazz.getURI() ) ){ + isDepResource = true; + break; + } + } + if( ! isDepResource ){ + for( VClass clazz : classes){ + List superClassUris = vClassDao.getAllSuperClassURIs(clazz.getURI()); + for( String uri : superClassUris){ + if( VitroVocabulary.DEPENDENT_RESORUCE.equals( uri ) ){ + isDepResource = true; + break; + } + } + if( isDepResource ) + break; + } + } + if( isDepResource){ + classes.add(depResVClass); + ind.setVClasses(classes, true); + } } - return ; - } + return inds; + } + /* maybe ObjectSourceIface should be replaced with just an iterator. */ private class BuilderObjectSource implements ObjectSourceIface { - private final List individuals; - public BuilderObjectSource( List individuals){ - this.individuals=individuals; - } - - public Iterator getAllOfThisTypeIterator() { - return new Iterator(){ - final Iterator it = individuals.iterator(); - - public boolean hasNext() { - return it.hasNext(); - } - - public Object next() { - return it.next(); - } - - public void remove() { /* not implemented */} - }; - } - - public Iterator getUpdatedSinceIterator(long msSinceEpoc) { - return getAllOfThisTypeIterator(); - } + private final List individuals; + public BuilderObjectSource( List individuals){ + this.individuals=individuals; + } + public Iterator getAllOfThisTypeIterator() { + return new Iterator(){ + final Iterator it = individuals.iterator(); + + public boolean hasNext() { + return it.hasNext(); + } + + public Object next() { + return it.next(); + } + + public void remove() { /* not implemented */} + }; + } + public Iterator getUpdatedSinceIterator(long msSinceEpoc) { + return getAllOfThisTypeIterator(); + } } - - - - + + private class AbortIndexing extends Exception { } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThread.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThread.java deleted file mode 100644 index 19c3fe286..000000000 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThread.java +++ /dev/null @@ -1,72 +0,0 @@ -/* $This file is distributed under the terms of the license in /doc/license.txt$ */ - -package edu.cornell.mannlib.vitro.webapp.search.indexing; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * Thread that executes the methods in IndexBuilder. - * - * @author bdc34 - * - */ -public class IndexBuilderThread extends Thread{ - private IndexBuilder indexBuilder; - protected boolean stopRequested = false; - protected long reindexInterval = 1000 * 60 /* msec */ ; - - private static final Log log = LogFactory.getLog(IndexBuilderThread.class.getName()); - - public IndexBuilderThread(IndexBuilder ib){ - super("IndexBuilderThread"); - if( ib == null ) - log.error("IndexBuilderThread needs an IndexBuilder, search is not configured."); - this.indexBuilder = ib; - } - - @Override - public void run() { - while(true){ - if( stopRequested ){ - log.info("Stopping IndexBuilderThread "); - return; - } - - if( indexBuilder == null ) - log.warn("IndexBuilderThread needs a IndexBuilder, search may not be configured."); - - try{ - if( indexBuilder != null && indexBuilder.isReindexRequested() ){ - log.debug("full re-index requested"); - indexBuilder.indexRebuild(); - }else{ - if( indexBuilder != null && indexBuilder.isThereWorkToDo() ){ - Thread.sleep(250); //wait a bit to let a bit more work to come into the queue - log.debug("work found for IndexBuilder, starting update"); - indexBuilder.updatedIndex(); - } - } - }catch (Throwable e) { - log.error(e,e); - } - - if( indexBuilder != null && ! indexBuilder.isThereWorkToDo() ){ - log.debug("there is no indexing working to do, going to sleep"); - try { - synchronized (this) { - this.wait(reindexInterval); - } - } catch (InterruptedException e) { - log.debug(" woken up",e); - } - } - } - } - - public synchronized void kill(){ - log.debug("Attempting to kill IndexBuilderThread "); - stopRequested = true; - this.notifyAll(); - } -} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java index 326c14dbc..a456cfbad 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java @@ -53,5 +53,11 @@ public interface IndexerIface { public void startIndexing() throws IndexingException; public void endIndexing(); - public long getModified(); + public long getModified(); + + /** + * Ends the indexing and removes any temporary files. + * This may be called instead of endIndexing() + */ + public void abortIndexingAndCleanUp(); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneIndexer.java index 18102d518..270932831 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneIndexer.java @@ -455,5 +455,22 @@ public class LuceneIndexer implements IndexerIface { public boolean isIndexCorroupt(){ //if it is clear it out but don't rebuild. return false; + } + + @Override + public synchronized void abortIndexingAndCleanUp() { + if( ! indexing ) + log.error("abortIndexingAndCleanUp() should only be called if LuceneIndexer is indexing."); + else if( ! fullRebuild ) + log.error("abortIndexingAndCleanUp() should only be called if LuceneIndexer to end an aborted full index rebuild"); + else{ + closeWriter(); + File offLineDir = new File(currentOffLineDir); + boolean deleted = deleteDir(offLineDir); + //log might be null if system is shutting down. + if( ! deleted ){ + System.out.println("Could not clean up temp indexing dir " + currentOffLineDir); + } + } } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetup.java index 9ee269e42..3ef35259c 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetup.java @@ -134,17 +134,12 @@ public class LuceneSetup implements javax.servlet.ServletContextListener { // set up listeners so search index builder is notified of changes to model ServletContext ctx = sce.getServletContext(); SearchReindexingListener srl = new SearchReindexingListener(builder); - ModelContext.getBaseOntModel(ctx).getBaseModel().register(srl); - ModelContext.getJenaOntModel(ctx).getBaseModel().register(srl); - ModelContext.getInferenceOntModel(ctx).register(srl); - ModelContext.getUnionOntModelSelector(ctx).getABoxModel() - .getBaseModel().register(srl); - + ModelContext.registerListenerForChanges(ctx, srl); + if( (Boolean)sce.getServletContext().getAttribute(INDEX_REBUILD_REQUESTED_AT_STARTUP) instanceof Boolean && (Boolean)sce.getServletContext().getAttribute(INDEX_REBUILD_REQUESTED_AT_STARTUP) ){ log.info("Rebuild of lucene index required before startup."); - builder.doIndexRebuild(); - Thread.currentThread().sleep(500); + builder.doIndexRebuild(); int n = 0; while( builder.isReindexRequested() || builder.isIndexing() ){ n++; @@ -169,7 +164,7 @@ public class LuceneSetup implements javax.servlet.ServletContextListener { log.debug("**** Running " + this.getClass().getName() + ".contextDestroyed()"); IndexBuilder builder = (IndexBuilder) sce.getServletContext().getAttribute(IndexBuilder.class.getName()); if( builder != null){ - builder.killIndexingThread(); + builder.stopIndexingThread(); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetupCJK.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetupCJK.java index e3a2409a5..c6e840ea0 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetupCJK.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/lucene/LuceneSetupCJK.java @@ -3,31 +3,31 @@ package edu.cornell.mannlib.vitro.webapp.search.lucene; import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; - -import javax.servlet.ServletContext; -import javax.servlet.ServletContextEvent; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.cjk.CJKAnalyzer; -import org.apache.lucene.search.BooleanQuery; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; + +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.cjk.CJKAnalyzer; +import org.apache.lucene.search.BooleanQuery; import com.hp.hpl.jena.ontology.OntModel; - + import edu.cornell.mannlib.vitro.webapp.ConfigurationProperties; -import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel; -import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; -import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering; -import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; -import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; +import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel; +import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; +import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering; +import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; +import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; +import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext; import edu.cornell.mannlib.vitro.webapp.dao.jena.SearchReindexingListener; -import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; /** * Setup objects for lucene searching and indexing. @@ -108,8 +108,7 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener { OntModel baseOntModel = (OntModel)sce.getServletContext().getAttribute("baseOntModel"); OntModel jenaOntModel = (OntModel)sce.getServletContext().getAttribute("jenaOntModel"); SearchReindexingListener srl = new SearchReindexingListener( builder ); - baseOntModel.getBaseModel().register(srl); - jenaOntModel.getBaseModel().register(srl); + ModelContext.registerListenerForChanges(sce.getServletContext(), srl); }catch(Exception ex){ log.error("Could not setup lucene full text search." , ex); @@ -125,7 +124,7 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener { log.info("**** Running "+this.getClass().getName()+".contextDestroyed()"); IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName()); - builder.killIndexingThread(); + builder.stopIndexingThread(); } /** diff --git a/webapp/test/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThreadTest.java b/webapp/test/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThreadTest.java index a2f819ec8..450694ed1 100644 --- a/webapp/test/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThreadTest.java +++ b/webapp/test/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilderThreadTest.java @@ -13,21 +13,25 @@ import edu.cornell.mannlib.vitro.testing.AbstractTestClass; public class IndexBuilderThreadTest extends AbstractTestClass { @Test - public void testStoppingTheThread(){ - setLoggerLevel(IndexBuilderThread.class, Level.OFF); - IndexBuilderThread ibt = new IndexBuilderThread(null); - ibt.start(); + public void testStoppingTheThread(){ + setLoggerLevel(IndexBuilder.class, Level.OFF); + + IndexBuilder ib = new IndexBuilder(); + Assert.assertNotSame(Thread.State.NEW, ib.getState() ); + Assert.assertNotSame(Thread.State.TERMINATED, ib.getState() ); + try { Thread.sleep(100); } catch (InterruptedException e) { Assert.fail(e.getMessage()); } - ibt.kill(); + ib.stopIndexingThread(); try { Thread.sleep(100); } catch (InterruptedException e) { Assert.fail(e.getMessage()); } - Assert.assertFalse(ibt.isAlive()); + Assert.assertFalse(ib.isAlive()); + Assert.assertSame(Thread.State.TERMINATED, ib.getState() ); } }