Changing the models that the SearchReindexingListener listens to. NIHVIVO-2076
This commit is contained in:
bdc34
parent
61812f7a1e
commit
1341d23a25
10 changed files with 327 additions and 323 deletions
|
|
@ -5,6 +5,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
|
|||
import javax.servlet.ServletContext;
|
||||
|
||||
import com.hp.hpl.jena.ontology.OntModel;
|
||||
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
|
||||
|
||||
public class ModelContext {
|
||||
|
||||
|
|
@ -75,4 +76,26 @@ public class ModelContext {
|
|||
ctx.setAttribute(INFERENCE_ONT_MODEL, ontModel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a listener to the models needed to get changes to:
|
||||
* class membership
|
||||
* inferred class membership
|
||||
* class group membership,
|
||||
* object properties,
|
||||
* data properties,
|
||||
* inferred object properties,
|
||||
* rdfs:label annotations
|
||||
* This listener does not need:
|
||||
* other annotations
|
||||
* change to TBox
|
||||
*/
|
||||
public static void registerListenerForChanges(ServletContext ctx, ModelChangedListener ml){
|
||||
ModelContext.getJenaOntModel(ctx).register(ml);
|
||||
ModelContext.getBaseOntModel(ctx).register(ml);
|
||||
ModelContext.getInferenceOntModel(ctx).register(ml);
|
||||
ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml);
|
||||
ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml);
|
||||
ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,9 +2,7 @@
|
|||
|
||||
package edu.cornell.mannlib.vitro.webapp.dao.jena;
|
||||
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
|
@ -32,18 +30,34 @@ public class SearchReindexingListener implements ModelChangedListener {
|
|||
this.indexBuilder = indexBuilder;
|
||||
}
|
||||
|
||||
private synchronized void addChange(Statement stmt){
|
||||
private synchronized void addChange(Statement stmt){
|
||||
if( stmt == null ) return;
|
||||
if( log.isDebugEnabled() ){
|
||||
String sub="unknown";
|
||||
String pred = "unknown";
|
||||
String obj ="unknown";
|
||||
|
||||
if( stmt.getSubject().isURIResource() ){
|
||||
sub = stmt.getSubject().getURI();
|
||||
}
|
||||
if( stmt.getPredicate() != null ){
|
||||
pred = stmt.getPredicate().getURI();
|
||||
}
|
||||
if( stmt.getObject().isURIResource() ){
|
||||
obj = ((Resource) (stmt.getPredicate().as(Resource.class))).getURI();
|
||||
}else{
|
||||
obj = stmt.getObject().toString();
|
||||
}
|
||||
log.debug("changed statement: sub='" + sub + "' pred='" + pred +"' obj='" + obj + "'");
|
||||
}
|
||||
|
||||
if( stmt.getSubject().isURIResource() ){
|
||||
//changedUris.add( stmt.getSubject().getURI());
|
||||
indexBuilder.addToChangedUris(stmt.getSubject().getURI());
|
||||
log.debug("subject: " + stmt.getSubject().getURI());
|
||||
}
|
||||
|
||||
if( stmt.getObject().isURIResource() ){
|
||||
//changedUris.add( ((Resource) stmt.getObject().as(Resource.class)).getURI() );
|
||||
indexBuilder.addToChangedUris(((Resource) stmt.getObject()).getURI());
|
||||
log.debug("object: " + ((Resource) stmt.getObject().as(Resource.class)).getURI());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -70,12 +70,14 @@ public class VClassGroupCache{
|
|||
}
|
||||
|
||||
VClassGroupCacheChangeListener bccl = new VClassGroupCacheChangeListener(this);
|
||||
ModelContext.getJenaOntModel(context).register(bccl);
|
||||
ModelContext.getBaseOntModel(context).register(bccl);
|
||||
ModelContext.getInferenceOntModel(context).register(bccl);
|
||||
ModelContext.getUnionOntModelSelector(context).getABoxModel().register(bccl);
|
||||
ModelContext.getBaseOntModelSelector(context).getABoxModel().register(bccl);
|
||||
ModelContext.getInferenceOntModelSelector(context).getABoxModel().register(bccl);
|
||||
ModelContext.registerListenerForChanges(context, bccl);
|
||||
//
|
||||
// ModelContext.getJenaOntModel(context).register(bccl);
|
||||
// ModelContext.getBaseOntModel(context).register(bccl);
|
||||
// ModelContext.getInferenceOntModel(context).register(bccl);
|
||||
// ModelContext.getUnionOntModelSelector(context).getABoxModel().register(bccl);
|
||||
// ModelContext.getBaseOntModelSelector(context).getABoxModel().register(bccl);
|
||||
// ModelContext.getInferenceOntModelSelector(context).getABoxModel().register(bccl);
|
||||
|
||||
_rebuildQueue.add(REBUILD_EVERY_PORTAL);
|
||||
_cacheRebuildThread = new RebuildGroupCacheThread(this);
|
||||
|
|
|
|||
|
|
@ -26,38 +26,31 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
|
|||
|
||||
/**
|
||||
* The IndexBuilder is used to rebuild or update a search index.
|
||||
* There should only be one IndexBuilder in a vitro web application.
|
||||
* It uses an implementation of a back-end through an object that
|
||||
* implements IndexerIface. An example of a back-end is LuceneIndexer.
|
||||
*
|
||||
* See the class SearchReindexingListener for an example of how a model change
|
||||
* listener can use an IndexBuilder to keep the full text index in sncy with
|
||||
* updates to a model.
|
||||
* updates to a model. It calls IndexBuilder.addToChangedUris().
|
||||
*
|
||||
* There should be an IndexBuilder in the servlet context, try:
|
||||
*
|
||||
IndexBuilder builder = (IndexBuilder)getServletContext().getAttribute(IndexBuilder.class.getName());
|
||||
if( request.getParameter("update") != null )
|
||||
builder.doUpdateIndex();
|
||||
|
||||
* @author bdc34
|
||||
*
|
||||
*/
|
||||
public class IndexBuilder {
|
||||
public class IndexBuilder extends Thread {
|
||||
private List<ObjectSourceIface> sourceList = new LinkedList<ObjectSourceIface>();
|
||||
private IndexerIface indexer = null;
|
||||
private ServletContext context = null;
|
||||
private ServletContext context = null;
|
||||
|
||||
private long lastRun = 0;
|
||||
|
||||
private HashSet<String> changedUris = null;
|
||||
/* changedUris should only be accessed from synchronized blocks */
|
||||
private HashSet<String> changedUris = null;
|
||||
|
||||
private List<Individual> updatedInds = null;
|
||||
private List<Individual> deletedInds = null;
|
||||
private List<Individual> deletedInds = null;
|
||||
|
||||
private IndexBuilderThread indexingThread = null;
|
||||
|
||||
//shared with IndexBuilderThread
|
||||
private boolean reindexRequested = false;
|
||||
private boolean reindexRequested = false;
|
||||
protected boolean stopRequested = false;
|
||||
protected long reindexInterval = 1000 * 60 /* msec */ ;
|
||||
|
||||
public static final boolean UPDATE_DOCS = false;
|
||||
public static final boolean NEW_DOCS = true;
|
||||
|
|
@ -67,15 +60,20 @@ public class IndexBuilder {
|
|||
public IndexBuilder(ServletContext context,
|
||||
IndexerIface indexer,
|
||||
List /*ObjectSourceIface*/ sources){
|
||||
super("IndexBuilder");
|
||||
this.indexer = indexer;
|
||||
this.sourceList = sources;
|
||||
this.context = context;
|
||||
|
||||
this.changedUris = new HashSet<String>();
|
||||
this.indexingThread = new IndexBuilderThread(this);
|
||||
this.indexingThread.start();
|
||||
this.changedUris = new HashSet<String>();
|
||||
this.start();
|
||||
}
|
||||
|
||||
|
||||
protected IndexBuilder(){
|
||||
//for testing only
|
||||
this( null, null, Collections.emptyList());
|
||||
}
|
||||
|
||||
public void addObjectSource(ObjectSourceIface osi) {
|
||||
if (osi != null)
|
||||
sourceList.add(osi);
|
||||
|
|
@ -89,30 +87,26 @@ public class IndexBuilder {
|
|||
return sourceList;
|
||||
}
|
||||
|
||||
public void doIndexRebuild() throws IndexingException {
|
||||
//set up full index rebuild
|
||||
setReindexRequested( true );
|
||||
//wake up indexing thread
|
||||
synchronized (this.indexingThread) {
|
||||
this.indexingThread.notifyAll();
|
||||
}
|
||||
public synchronized void doIndexRebuild() throws IndexingException {
|
||||
//set flag for full index rebuild
|
||||
this.reindexRequested = true;
|
||||
//wake up
|
||||
this.notifyAll();
|
||||
}
|
||||
|
||||
/**
|
||||
/**
|
||||
* This will re-index Individuals that changed because of modtime or because they
|
||||
* were added with addChangedUris().
|
||||
*/
|
||||
public void doUpdateIndex() {
|
||||
//wake up thread
|
||||
synchronized (this.indexingThread) {
|
||||
this.indexingThread.notifyAll();
|
||||
}
|
||||
public synchronized void doUpdateIndex() {
|
||||
//wake up thread and it will attempt to index anything in changedUris
|
||||
this.notifyAll();
|
||||
}
|
||||
|
||||
public synchronized void addToChangedUris(String uri){
|
||||
changedUris.add(uri);
|
||||
}
|
||||
|
||||
|
||||
public synchronized void addToChangedUris(Collection<String> uris){
|
||||
changedUris.addAll(uris);
|
||||
}
|
||||
|
|
@ -125,79 +119,47 @@ public class IndexBuilder {
|
|||
return isReindexRequested() || ! changedUris.isEmpty() ;
|
||||
}
|
||||
|
||||
public void killIndexingThread() {
|
||||
this.indexingThread.kill();
|
||||
public synchronized void stopIndexingThread() {
|
||||
stopRequested = true;
|
||||
this.notifyAll();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
while(! stopRequested ){
|
||||
try{
|
||||
if( !stopRequested && isReindexRequested() ){
|
||||
log.debug("full re-index requested");
|
||||
indexRebuild();
|
||||
}else if( !stopRequested && isThereWorkToDo() ){
|
||||
Thread.sleep(250); //wait a bit to let a bit more work to come into the queue
|
||||
log.debug("work found for IndexBuilder, starting update");
|
||||
updatedIndex();
|
||||
}else if( !stopRequested && ! isThereWorkToDo() ){
|
||||
log.debug("there is no indexing working to do, waiting for work");
|
||||
synchronized (this) { this.wait(reindexInterval); }
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.debug("woken up",e);
|
||||
}catch(Throwable e){
|
||||
log.error(e,e);
|
||||
}
|
||||
}
|
||||
log.info("Stopping IndexBuilder thread");
|
||||
}
|
||||
|
||||
|
||||
/* ******************** non-public methods ************************* */
|
||||
|
||||
private synchronized void setReindexRequested(boolean reindexRequested) {
|
||||
this.reindexRequested = reindexRequested;
|
||||
}
|
||||
|
||||
private synchronized Collection<String> getAndEmptyChangedUris(){
|
||||
Collection<String> out = changedUris;
|
||||
changedUris = new HashSet<String>();
|
||||
return out;
|
||||
}
|
||||
|
||||
protected void indexRebuild() throws IndexingException {
|
||||
log.info("Rebuild of search index is starting.");
|
||||
|
||||
Iterator<ObjectSourceIface> sources = sourceList.iterator();
|
||||
List listOfIterators = new LinkedList();
|
||||
while(sources.hasNext()){
|
||||
Object obj = sources.next();
|
||||
if( obj != null && obj instanceof ObjectSourceIface )
|
||||
listOfIterators.add((((ObjectSourceIface) obj)
|
||||
.getAllOfThisTypeIterator()));
|
||||
else
|
||||
log.warn("\tskipping object of class "
|
||||
+ obj.getClass().getName() + "\n"
|
||||
+ "\tIt doesn not implement ObjectSourceIface.\n");
|
||||
}
|
||||
|
||||
//clear out changed uris since we are doing a full index rebuild
|
||||
getAndEmptyChangedUris();
|
||||
|
||||
if( listOfIterators.size() == 0){ log.warn("Warning: no ObjectSources found.");}
|
||||
|
||||
setReindexRequested(false);
|
||||
doBuild( listOfIterators, Collections.EMPTY_LIST, true, NEW_DOCS );
|
||||
log.info("Rebuild of search index is complete.");
|
||||
}
|
||||
|
||||
protected void updatedIndex() throws IndexingException{
|
||||
log.debug("Starting updateIndex()");
|
||||
long since = indexer.getModified() - 60000;
|
||||
|
||||
Iterator<ObjectSourceIface> sources = sourceList.iterator();
|
||||
|
||||
List<Iterator<Individual>> listOfIterators =
|
||||
new LinkedList<Iterator<Individual>>();
|
||||
|
||||
while (sources.hasNext()) {
|
||||
Object obj = sources.next();
|
||||
if (obj != null && obj instanceof ObjectSourceIface)
|
||||
listOfIterators.add((((ObjectSourceIface) obj)
|
||||
.getUpdatedSinceIterator(since)));
|
||||
else
|
||||
log.warn("\tskipping object of class "
|
||||
+ obj.getClass().getName() + "\n"
|
||||
+ "\tIt doesn not implement " + "ObjectSourceIface.\n");
|
||||
}
|
||||
|
||||
buildAddAndDeleteLists( getAndEmptyChangedUris());
|
||||
listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) );
|
||||
|
||||
doBuild( listOfIterators, deletedInds, false, UPDATE_DOCS );
|
||||
log.debug("Ending updateIndex()");
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets updatedUris and deletedUris.
|
||||
* @param changedUris
|
||||
* Sets updatedUris and deletedUris lists.
|
||||
*/
|
||||
private void buildAddAndDeleteLists( Collection<String> uris){
|
||||
private void makeAddAndDeleteLists( Collection<String> uris){
|
||||
/* clear updateInds and deletedUris. This is the only method that should set these. */
|
||||
this.updatedInds = new ArrayList<Individual>();
|
||||
this.deletedInds = new ArrayList<Individual>();
|
||||
|
|
@ -216,45 +178,60 @@ public class IndexBuilder {
|
|||
}
|
||||
|
||||
this.updatedInds = addDepResourceClasses(updatedInds);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
protected void indexRebuild() throws IndexingException {
|
||||
log.info("Rebuild of search index is starting.");
|
||||
|
||||
|
||||
private List<Individual> addDepResourceClasses(List<Individual> inds) {
|
||||
WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory");
|
||||
VClassDao vClassDao = wdf.getVClassDao();
|
||||
Iterator<Individual> it = inds.iterator();
|
||||
VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE);
|
||||
while(it.hasNext()){
|
||||
Individual ind = it.next();
|
||||
List<VClass> classes = ind.getVClasses();
|
||||
boolean isDepResource = false;
|
||||
for( VClass clazz : classes){
|
||||
if( !isDepResource && VitroVocabulary.DEPENDENT_RESORUCE.equals( clazz.getURI() ) ){
|
||||
isDepResource = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( ! isDepResource ){
|
||||
for( VClass clazz : classes){
|
||||
List<String> superClassUris = vClassDao.getAllSuperClassURIs(clazz.getURI());
|
||||
for( String uri : superClassUris){
|
||||
if( VitroVocabulary.DEPENDENT_RESORUCE.equals( uri ) ){
|
||||
isDepResource = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( isDepResource )
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( isDepResource){
|
||||
classes.add(depResVClass);
|
||||
ind.setVClasses(classes, true);
|
||||
}
|
||||
}
|
||||
return inds;
|
||||
}
|
||||
Iterator<ObjectSourceIface> sources = sourceList.iterator();
|
||||
List listOfIterators = new LinkedList();
|
||||
while (sources.hasNext()) {
|
||||
Object obj = sources.next();
|
||||
if (obj != null && obj instanceof ObjectSourceIface)
|
||||
listOfIterators.add((((ObjectSourceIface) obj)
|
||||
.getAllOfThisTypeIterator()));
|
||||
else
|
||||
log.warn("\tskipping object of class "
|
||||
+ obj.getClass().getName() + "\n"
|
||||
+ "\tIt doesn not implement ObjectSourceIface.\n");
|
||||
}
|
||||
|
||||
// clear out changed uris since we are doing a full index rebuild
|
||||
getAndEmptyChangedUris();
|
||||
|
||||
if (listOfIterators.size() == 0)
|
||||
log.warn("Warning: no ObjectSources found.");
|
||||
|
||||
doBuild(listOfIterators, Collections.EMPTY_LIST );
|
||||
if( log != null ) //log might be null if system is shutting down.
|
||||
log.info("Rebuild of search index is complete.");
|
||||
}
|
||||
|
||||
protected void updatedIndex() throws IndexingException{
|
||||
log.debug("Starting updateIndex()");
|
||||
long since = indexer.getModified() - 60000;
|
||||
|
||||
Iterator<ObjectSourceIface> sources = sourceList.iterator();
|
||||
List<Iterator<Individual>> listOfIterators =
|
||||
new LinkedList<Iterator<Individual>>();
|
||||
|
||||
while (sources.hasNext()) {
|
||||
Object obj = sources.next();
|
||||
if (obj != null && obj instanceof ObjectSourceIface)
|
||||
listOfIterators.add((((ObjectSourceIface) obj)
|
||||
.getUpdatedSinceIterator(since)));
|
||||
else
|
||||
log.warn("\tskipping object of class "
|
||||
+ obj.getClass().getName() + "\n"
|
||||
+ "\tIt doesn not implement " + "ObjectSourceIface.\n");
|
||||
}
|
||||
|
||||
makeAddAndDeleteLists( getAndEmptyChangedUris());
|
||||
listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) );
|
||||
|
||||
doBuild( listOfIterators, deletedInds );
|
||||
log.debug("Ending updateIndex()");
|
||||
}
|
||||
|
||||
/**
|
||||
* For each sourceIterator, get all of the objects and attempt to
|
||||
|
|
@ -270,17 +247,22 @@ public class IndexBuilder {
|
|||
* to false, and a check is made before adding, it will work fine; but
|
||||
* checking if an object is on the index is slow.
|
||||
*/
|
||||
private void doBuild(List sourceIterators, Collection<Individual> deletes, boolean forceNewIndex, boolean newDocs ){
|
||||
private void doBuild(List<Iterator<Individual>> sourceIterators, Collection<Individual> deletes ){
|
||||
boolean aborted = false;
|
||||
boolean newDocs = reindexRequested;
|
||||
boolean forceNewIndex = reindexRequested;
|
||||
|
||||
try {
|
||||
if( forceNewIndex )
|
||||
if( reindexRequested )
|
||||
indexer.prepareForRebuild();
|
||||
|
||||
indexer.startIndexing();
|
||||
|
||||
reindexRequested = false;
|
||||
|
||||
if( ! forceNewIndex ){
|
||||
for(Individual deleteMe : deletes ){
|
||||
indexer.removeFromIndex(deleteMe);
|
||||
}
|
||||
for(Individual deleteMe : deletes ){
|
||||
indexer.removeFromIndex(deleteMe);
|
||||
}
|
||||
}
|
||||
|
||||
//get an iterator for all of the sources of indexable objects
|
||||
|
|
@ -289,35 +271,50 @@ public class IndexBuilder {
|
|||
while (sourceIters.hasNext()) {
|
||||
obj = sourceIters.next();
|
||||
if (obj == null || !(obj instanceof Iterator)) {
|
||||
log.warn("\tskipping object of class "
|
||||
+ obj.getClass().getName() + "\n"
|
||||
+ "\tIt doesn not implement "
|
||||
+ "Iterator.\n");
|
||||
log.warn("skipping object of class " + obj.getClass().getName()
|
||||
+ "It doesn not implement Iterator.");
|
||||
continue;
|
||||
}
|
||||
indexForSource((Iterator)obj, newDocs);
|
||||
}
|
||||
} catch (IndexingException ex) {
|
||||
log.error(ex,ex);
|
||||
} catch (AbortIndexing abort){
|
||||
if( log != null)
|
||||
log.debug("aborting the indexing because thread stop was requested");
|
||||
aborted = true;
|
||||
} catch (Exception e) {
|
||||
log.error(e,e);
|
||||
} finally {
|
||||
}
|
||||
|
||||
if( aborted && forceNewIndex ){
|
||||
indexer.abortIndexingAndCleanUp();
|
||||
}else{
|
||||
indexer.endIndexing();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Use the back end indexer to index each object that the Iterator returns.
|
||||
* @param items
|
||||
* @return
|
||||
* @throws AbortIndexing
|
||||
*/
|
||||
private void indexForSource(Iterator<Individual> individuals , boolean newDocs){
|
||||
if( individuals == null ) return;
|
||||
|
||||
private void indexForSource(Iterator<Individual> individuals , boolean newDocs) throws AbortIndexing{
|
||||
long starttime = System.currentTimeMillis();
|
||||
long count = 0;
|
||||
while(individuals.hasNext()){
|
||||
indexItem(individuals.next(), newDocs);
|
||||
while(individuals.hasNext()){
|
||||
if( stopRequested )
|
||||
throw new AbortIndexing();
|
||||
|
||||
Individual ind = null;
|
||||
try{
|
||||
ind = individuals.next();
|
||||
indexer.index(ind, newDocs);
|
||||
}catch(Throwable ex){
|
||||
if( stopRequested || log == null){//log might be null if system is shutting down.
|
||||
throw new AbortIndexing();
|
||||
}
|
||||
String uri = ind!=null?ind.getURI():"null";
|
||||
log.warn("Error indexing individual " + uri + " " + ex.getMessage());
|
||||
}
|
||||
count++;
|
||||
if( log.isDebugEnabled() ){
|
||||
if( (count % 100 ) == 0 && count > 0 ){
|
||||
|
|
@ -330,55 +327,74 @@ public class IndexBuilder {
|
|||
|
||||
log.info(
|
||||
"individuals indexed: " + count + " in " + (System.currentTimeMillis() - starttime) + " msec" +
|
||||
(count!=0?(" time per individual = " + (System.currentTimeMillis() - starttime)/ count + " msec"):"")
|
||||
(count!=0?(" time per individual = " + (System.currentTimeMillis() - starttime)/ count + " msec"):"")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Use the backend indexer to index a single item.
|
||||
* @param item
|
||||
* @return
|
||||
* For a list of individuals, this builds a list of dependent resources and returns it.
|
||||
*/
|
||||
private void indexItem( Individual ind, boolean newDoc){
|
||||
try{
|
||||
if( ind != null ){
|
||||
indexer.index(ind, newDoc);
|
||||
}
|
||||
}catch(Throwable ex){
|
||||
log.warn("IndexBuilder.indexItem() Error indexing "
|
||||
+ ind + "\n" +ex);
|
||||
private List<Individual> addDepResourceClasses(List<Individual> inds) {
|
||||
WebappDaoFactory wdf = (WebappDaoFactory)context.getAttribute("webappDaoFactory");
|
||||
VClassDao vClassDao = wdf.getVClassDao();
|
||||
Iterator<Individual> it = inds.iterator();
|
||||
VClass depResVClass = new VClass(VitroVocabulary.DEPENDENT_RESORUCE);
|
||||
while(it.hasNext()){
|
||||
Individual ind = it.next();
|
||||
List<VClass> classes = ind.getVClasses();
|
||||
boolean isDepResource = false;
|
||||
for( VClass clazz : classes){
|
||||
if( !isDepResource && VitroVocabulary.DEPENDENT_RESORUCE.equals( clazz.getURI() ) ){
|
||||
isDepResource = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( ! isDepResource ){
|
||||
for( VClass clazz : classes){
|
||||
List<String> superClassUris = vClassDao.getAllSuperClassURIs(clazz.getURI());
|
||||
for( String uri : superClassUris){
|
||||
if( VitroVocabulary.DEPENDENT_RESORUCE.equals( uri ) ){
|
||||
isDepResource = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( isDepResource )
|
||||
break;
|
||||
}
|
||||
}
|
||||
if( isDepResource){
|
||||
classes.add(depResVClass);
|
||||
ind.setVClasses(classes, true);
|
||||
}
|
||||
}
|
||||
return ;
|
||||
}
|
||||
return inds;
|
||||
}
|
||||
|
||||
/* maybe ObjectSourceIface should be replaced with just an iterator. */
|
||||
private class BuilderObjectSource implements ObjectSourceIface {
|
||||
private final List<Individual> individuals;
|
||||
public BuilderObjectSource( List<Individual> individuals){
|
||||
this.individuals=individuals;
|
||||
}
|
||||
|
||||
public Iterator getAllOfThisTypeIterator() {
|
||||
return new Iterator(){
|
||||
final Iterator it = individuals.iterator();
|
||||
|
||||
public boolean hasNext() {
|
||||
return it.hasNext();
|
||||
}
|
||||
|
||||
public Object next() {
|
||||
return it.next();
|
||||
}
|
||||
|
||||
public void remove() { /* not implemented */}
|
||||
};
|
||||
}
|
||||
|
||||
public Iterator getUpdatedSinceIterator(long msSinceEpoc) {
|
||||
return getAllOfThisTypeIterator();
|
||||
}
|
||||
private final List<Individual> individuals;
|
||||
public BuilderObjectSource( List<Individual> individuals){
|
||||
this.individuals=individuals;
|
||||
}
|
||||
public Iterator getAllOfThisTypeIterator() {
|
||||
return new Iterator(){
|
||||
final Iterator it = individuals.iterator();
|
||||
|
||||
public boolean hasNext() {
|
||||
return it.hasNext();
|
||||
}
|
||||
|
||||
public Object next() {
|
||||
return it.next();
|
||||
}
|
||||
|
||||
public void remove() { /* not implemented */}
|
||||
};
|
||||
}
|
||||
public Iterator getUpdatedSinceIterator(long msSinceEpoc) {
|
||||
return getAllOfThisTypeIterator();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
private class AbortIndexing extends Exception { }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,72 +0,0 @@
|
|||
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
|
||||
|
||||
package edu.cornell.mannlib.vitro.webapp.search.indexing;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
|
||||
/**
|
||||
* Thread that executes the methods in IndexBuilder.
|
||||
*
|
||||
* @author bdc34
|
||||
*
|
||||
*/
|
||||
public class IndexBuilderThread extends Thread{
|
||||
private IndexBuilder indexBuilder;
|
||||
protected boolean stopRequested = false;
|
||||
protected long reindexInterval = 1000 * 60 /* msec */ ;
|
||||
|
||||
private static final Log log = LogFactory.getLog(IndexBuilderThread.class.getName());
|
||||
|
||||
public IndexBuilderThread(IndexBuilder ib){
|
||||
super("IndexBuilderThread");
|
||||
if( ib == null )
|
||||
log.error("IndexBuilderThread needs an IndexBuilder, search is not configured.");
|
||||
this.indexBuilder = ib;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run() {
|
||||
while(true){
|
||||
if( stopRequested ){
|
||||
log.info("Stopping IndexBuilderThread ");
|
||||
return;
|
||||
}
|
||||
|
||||
if( indexBuilder == null )
|
||||
log.warn("IndexBuilderThread needs a IndexBuilder, search may not be configured.");
|
||||
|
||||
try{
|
||||
if( indexBuilder != null && indexBuilder.isReindexRequested() ){
|
||||
log.debug("full re-index requested");
|
||||
indexBuilder.indexRebuild();
|
||||
}else{
|
||||
if( indexBuilder != null && indexBuilder.isThereWorkToDo() ){
|
||||
Thread.sleep(250); //wait a bit to let a bit more work to come into the queue
|
||||
log.debug("work found for IndexBuilder, starting update");
|
||||
indexBuilder.updatedIndex();
|
||||
}
|
||||
}
|
||||
}catch (Throwable e) {
|
||||
log.error(e,e);
|
||||
}
|
||||
|
||||
if( indexBuilder != null && ! indexBuilder.isThereWorkToDo() ){
|
||||
log.debug("there is no indexing working to do, going to sleep");
|
||||
try {
|
||||
synchronized (this) {
|
||||
this.wait(reindexInterval);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.debug(" woken up",e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public synchronized void kill(){
|
||||
log.debug("Attempting to kill IndexBuilderThread ");
|
||||
stopRequested = true;
|
||||
this.notifyAll();
|
||||
}
|
||||
}
|
||||
|
|
@ -53,5 +53,11 @@ public interface IndexerIface {
|
|||
public void startIndexing() throws IndexingException;
|
||||
public void endIndexing();
|
||||
|
||||
public long getModified();
|
||||
public long getModified();
|
||||
|
||||
/**
|
||||
* Ends the indexing and removes any temporary files.
|
||||
* This may be called instead of endIndexing()
|
||||
*/
|
||||
public void abortIndexingAndCleanUp();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -455,5 +455,22 @@ public class LuceneIndexer implements IndexerIface {
|
|||
public boolean isIndexCorroupt(){
|
||||
//if it is clear it out but don't rebuild.
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void abortIndexingAndCleanUp() {
|
||||
if( ! indexing )
|
||||
log.error("abortIndexingAndCleanUp() should only be called if LuceneIndexer is indexing.");
|
||||
else if( ! fullRebuild )
|
||||
log.error("abortIndexingAndCleanUp() should only be called if LuceneIndexer to end an aborted full index rebuild");
|
||||
else{
|
||||
closeWriter();
|
||||
File offLineDir = new File(currentOffLineDir);
|
||||
boolean deleted = deleteDir(offLineDir);
|
||||
//log might be null if system is shutting down.
|
||||
if( ! deleted ){
|
||||
System.out.println("Could not clean up temp indexing dir " + currentOffLineDir);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -134,17 +134,12 @@ public class LuceneSetup implements javax.servlet.ServletContextListener {
|
|||
// set up listeners so search index builder is notified of changes to model
|
||||
ServletContext ctx = sce.getServletContext();
|
||||
SearchReindexingListener srl = new SearchReindexingListener(builder);
|
||||
ModelContext.getBaseOntModel(ctx).getBaseModel().register(srl);
|
||||
ModelContext.getJenaOntModel(ctx).getBaseModel().register(srl);
|
||||
ModelContext.getInferenceOntModel(ctx).register(srl);
|
||||
ModelContext.getUnionOntModelSelector(ctx).getABoxModel()
|
||||
.getBaseModel().register(srl);
|
||||
|
||||
ModelContext.registerListenerForChanges(ctx, srl);
|
||||
|
||||
if( (Boolean)sce.getServletContext().getAttribute(INDEX_REBUILD_REQUESTED_AT_STARTUP) instanceof Boolean &&
|
||||
(Boolean)sce.getServletContext().getAttribute(INDEX_REBUILD_REQUESTED_AT_STARTUP) ){
|
||||
log.info("Rebuild of lucene index required before startup.");
|
||||
builder.doIndexRebuild();
|
||||
Thread.currentThread().sleep(500);
|
||||
builder.doIndexRebuild();
|
||||
int n = 0;
|
||||
while( builder.isReindexRequested() || builder.isIndexing() ){
|
||||
n++;
|
||||
|
|
@ -169,7 +164,7 @@ public class LuceneSetup implements javax.servlet.ServletContextListener {
|
|||
log.debug("**** Running " + this.getClass().getName() + ".contextDestroyed()");
|
||||
IndexBuilder builder = (IndexBuilder) sce.getServletContext().getAttribute(IndexBuilder.class.getName());
|
||||
if( builder != null){
|
||||
builder.killIndexingThread();
|
||||
builder.stopIndexingThread();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,31 +3,31 @@
|
|||
package edu.cornell.mannlib.vitro.webapp.search.lucene;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.ServletContext;
|
||||
import javax.servlet.ServletContextEvent;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
|
||||
import javax.servlet.ServletContext;
|
||||
import javax.servlet.ServletContextEvent;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
|
||||
import com.hp.hpl.jena.ontology.OntModel;
|
||||
|
||||
|
||||
import edu.cornell.mannlib.vitro.webapp.ConfigurationProperties;
|
||||
import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
|
||||
import edu.cornell.mannlib.vitro.webapp.beans.BaseResourceBean.RoleLevel;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
|
||||
import edu.cornell.mannlib.vitro.webapp.dao.jena.SearchReindexingListener;
|
||||
import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher;
|
||||
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
|
||||
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
|
||||
|
||||
/**
|
||||
* Setup objects for lucene searching and indexing.
|
||||
|
|
@ -108,8 +108,7 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener {
|
|||
OntModel baseOntModel = (OntModel)sce.getServletContext().getAttribute("baseOntModel");
|
||||
OntModel jenaOntModel = (OntModel)sce.getServletContext().getAttribute("jenaOntModel");
|
||||
SearchReindexingListener srl = new SearchReindexingListener( builder );
|
||||
baseOntModel.getBaseModel().register(srl);
|
||||
jenaOntModel.getBaseModel().register(srl);
|
||||
ModelContext.registerListenerForChanges(sce.getServletContext(), srl);
|
||||
|
||||
}catch(Exception ex){
|
||||
log.error("Could not setup lucene full text search." , ex);
|
||||
|
|
@ -125,7 +124,7 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener {
|
|||
|
||||
log.info("**** Running "+this.getClass().getName()+".contextDestroyed()");
|
||||
IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName());
|
||||
builder.killIndexingThread();
|
||||
builder.stopIndexingThread();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -13,21 +13,25 @@ import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
|
|||
public class IndexBuilderThreadTest extends AbstractTestClass {
|
||||
|
||||
@Test
|
||||
public void testStoppingTheThread(){
|
||||
setLoggerLevel(IndexBuilderThread.class, Level.OFF);
|
||||
IndexBuilderThread ibt = new IndexBuilderThread(null);
|
||||
ibt.start();
|
||||
public void testStoppingTheThread(){
|
||||
setLoggerLevel(IndexBuilder.class, Level.OFF);
|
||||
|
||||
IndexBuilder ib = new IndexBuilder();
|
||||
Assert.assertNotSame(Thread.State.NEW, ib.getState() );
|
||||
Assert.assertNotSame(Thread.State.TERMINATED, ib.getState() );
|
||||
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Assert.fail(e.getMessage());
|
||||
}
|
||||
ibt.kill();
|
||||
ib.stopIndexingThread();
|
||||
try {
|
||||
Thread.sleep(100);
|
||||
} catch (InterruptedException e) {
|
||||
Assert.fail(e.getMessage());
|
||||
}
|
||||
Assert.assertFalse(ibt.isAlive());
|
||||
Assert.assertFalse(ib.isAlive());
|
||||
Assert.assertSame(Thread.State.TERMINATED, ib.getState() );
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue