diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/AdditionalURIsToIndex.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/AdditionalURIsToIndex.java new file mode 100644 index 000000000..ce99b4628 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/AdditionalURIsToIndex.java @@ -0,0 +1,15 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.search.beans; + +import java.util.List; + +import com.hp.hpl.jena.rdf.model.Statement; + +/** + * Interface to use with IndexBuilder to find more URIs to index given a changed statement. + * The stmt may have been added or removed from the model. + */ +public interface AdditionalURIsToIndex { + List findAdditionalURIsToIndex(Statement stmt); +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/IndexerIface.java similarity index 94% rename from webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java rename to webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/IndexerIface.java index 50ebdfac7..be5899509 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/IndexerIface.java @@ -1,6 +1,6 @@ /* $This file is distributed under the terms of the license in /doc/license.txt$ */ -package edu.cornell.mannlib.vitro.webapp.search.indexing; +package edu.cornell.mannlib.vitro.webapp.search.beans; import java.util.List; diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsForContextNodes.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsForContextNodes.java index a6e84245f..91e261d99 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsForContextNodes.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsForContextNodes.java @@ -22,8 +22,11 @@ import com.hp.hpl.jena.query.Syntax; import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.Resource; import com.hp.hpl.jena.rdf.model.ResourceFactory; +import com.hp.hpl.jena.rdf.model.Statement; import com.hp.hpl.jena.shared.Lock; +import edu.cornell.mannlib.vitro.webapp.search.beans.AdditionalURIsToIndex; + public class AdditionalURIsForContextNodes implements AdditionalURIsToIndex { private OntModel model; @@ -39,8 +42,30 @@ public class AdditionalURIsForContextNodes implements AdditionalURIsToIndex { } @Override - public List findAdditionalURIsToIndex(String uri) { - + public List findAdditionalURIsToIndex(Statement stmt) { + + if( stmt != null ){ + ListurisToIndex = new ArrayList(); + if(stmt.getSubject() != null && stmt.getSubject().isURIResource() ){ + String subjUri = stmt.getSubject().getURI(); + if( subjUri != null){ + urisToIndex.addAll( findAdditionalURIsToIndex(subjUri)); + } + } + + if( stmt.getObject() != null && stmt.getObject().isURIResource() ){ + String objUri = stmt.getSubject().getURI(); + if( objUri != null){ + urisToIndex.addAll( findAdditionalURIsToIndex(objUri)); + } + } + return urisToIndex; + }else{ + return Collections.emptyList(); + } + } + + protected List findAdditionalURIsToIndex(String uri) { List uriList = new ArrayList(); @@ -307,4 +332,7 @@ public class AdditionalURIsForContextNodes implements AdditionalURIsToIndex { queryList = Collections.unmodifiableList(tmpList); } + + + } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsToIndex.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsToIndex.java deleted file mode 100644 index 37f56fb1a..000000000 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/AdditionalURIsToIndex.java +++ /dev/null @@ -1,13 +0,0 @@ -/* $This file is distributed under the terms of the license in /doc/license.txt$ */ - -package edu.cornell.mannlib.vitro.webapp.search.indexing; - -import java.util.List; - -/** - * Interface to use with IndexBuilder to find more URIs to index given a URI. - * - */ -public interface AdditionalURIsToIndex { - List findAdditionalURIsToIndex(String uri); -} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index 71cce8857..19e5312b6 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -19,6 +19,8 @@ import com.hp.hpl.jena.query.QueryParseException; import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; +import edu.cornell.mannlib.vitro.webapp.search.beans.AdditionalURIsToIndex; +import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; /** @@ -50,7 +52,6 @@ public class IndexBuilder extends Thread { protected long reindexInterval = 1000 * 60 /* msec */ ; protected int numberOfThreads = 10; - protected List additionalURIsFinders; public static final boolean UPDATE_DOCS = false; public static final boolean NEW_DOCS = true; @@ -60,20 +61,18 @@ public class IndexBuilder extends Thread { public IndexBuilder( ServletContext context, IndexerIface indexer, - WebappDaoFactory wdf, - List additionalURIsFinders){ + WebappDaoFactory wdf ){ super("IndexBuilder"); this.indexer = indexer; this.wdf = wdf; this.context = context; - this.additionalURIsFinders = additionalURIsFinders; this.changedUris = new HashSet(); this.start(); } protected IndexBuilder(){ //for testing only - this( null, null, null, null); + this( null, null, null); } public void setWdf(WebappDaoFactory wdf){ @@ -173,9 +172,7 @@ public class IndexBuilder extends Thread { /** * Sets updatedUris and deletedUris lists. */ - private void makeAddAndDeleteLists( Collection uris){ - - uris.addAll(getAdditionalURIsToIndex(uris)); + private void makeAddAndDeleteLists( Collection uris){ /* clear updateInds and deletedUris. This is the only method that should set these. */ this.updatedInds = new ArrayList(); @@ -197,18 +194,6 @@ public class IndexBuilder extends Thread { } } } - - private Set getAdditionalURIsToIndex(Collection uris) { - - Set listOfAdditionalURIs = new HashSet(); - - for(String uri: uris){ - //grab the uris from AdditionalURIsToIndex - listOfAdditionalURIs.addAll(additionalURIsFinders.get(0).findAdditionalURIsToIndex(uri)); - } - - return listOfAdditionalURIs; - } /** * This rebuilds the whole index. diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java index 33b34080e..11b6b370d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java @@ -9,6 +9,7 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.search.IndexingException; +import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.solr.IndividualToSolrDocument; class IndexWorkerThread extends Thread{ diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/SearchReindexingListener.java similarity index 86% rename from webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java rename to webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/SearchReindexingListener.java index c3a6895a2..3774dae58 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SearchReindexingListener.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/SearchReindexingListener.java @@ -1,7 +1,8 @@ /* $This file is distributed under the terms of the license in /doc/license.txt$ */ -package edu.cornell.mannlib.vitro.webapp.dao.jena; +package edu.cornell.mannlib.vitro.webapp.search.indexing; +import java.util.Collections; import java.util.List; import org.apache.commons.logging.Log; @@ -15,7 +16,7 @@ import com.hp.hpl.jena.rdf.model.StmtIterator; import com.hp.hpl.jena.shared.Lock; import edu.cornell.mannlib.vitro.webapp.dao.jena.event.EditEvent; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.search.beans.AdditionalURIsToIndex; /** * This class is thread safe. Notice that doAsyncIndexBuild() is frequently @@ -23,11 +24,16 @@ import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; */ public class SearchReindexingListener implements ModelChangedListener { private IndexBuilder indexBuilder; + private List additionalUriFinders; - public SearchReindexingListener(IndexBuilder indexBuilder) { + public SearchReindexingListener(IndexBuilder indexBuilder, List addUrisList ) { if(indexBuilder == null ) throw new IllegalArgumentException("Constructor parameter indexBuilder must not be null"); - this.indexBuilder = indexBuilder; + this.indexBuilder = indexBuilder; + if( addUrisList != null ) + this.additionalUriFinders = addUrisList; + else + this.additionalUriFinders = Collections.emptyList(); } private synchronized void addChange(Statement stmt){ @@ -59,6 +65,10 @@ public class SearchReindexingListener implements ModelChangedListener { if( stmt.getObject().isURIResource() ){ indexBuilder.addToChangedUris(((Resource) stmt.getObject()).getURI()); } + + for( AdditionalURIsToIndex au : additionalUriFinders ){ + indexBuilder.addToChangedUris( au.findAdditionalURIsToIndex(stmt) ); + } } private void requestAsyncIndexUpdate(){ diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java index a54dc617d..c3a21011e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java @@ -18,8 +18,8 @@ import org.apache.solr.common.SolrInputDocument; import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.search.IndexingException; +import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface; -import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexerIface; public class SolrIndexer implements IndexerIface { diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java index 0ba29a592..ae323c084 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java @@ -27,14 +27,14 @@ import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDao; import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext; -import edu.cornell.mannlib.vitro.webapp.dao.jena.SearchReindexingListener; import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactoryJena; import edu.cornell.mannlib.vitro.webapp.search.IndexConstants; +import edu.cornell.mannlib.vitro.webapp.search.beans.AdditionalURIsToIndex; import edu.cornell.mannlib.vitro.webapp.search.beans.IndividualProhibitedFromSearchImpl; import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch; import edu.cornell.mannlib.vitro.webapp.search.indexing.AdditionalURIsForContextNodes; -import edu.cornell.mannlib.vitro.webapp.search.indexing.AdditionalURIsToIndex; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; +import edu.cornell.mannlib.vitro.webapp.search.indexing.SearchReindexingListener; import edu.cornell.mannlib.vitro.webapp.servlet.setup.AbortStartup; public class SolrSetup implements javax.servlet.ServletContextListener{ @@ -61,8 +61,8 @@ public class SolrSetup implements javax.servlet.ServletContextListener{ return; } CommonsHttpSolrServer server; + //It would be nice to use the default binary handler but there seem to be library problems server = new CommonsHttpSolrServer(new URL( solrServerUrl ),null,new XMLResponseParser(),false); - //server = new CommonsHttpSolrServer(new URL( solrServerUrl )); server.setSoTimeout(10000); // socket read timeout server.setConnectionTimeout(10000); server.setDefaultMaxConnectionsPerHost(100); @@ -90,11 +90,7 @@ public class SolrSetup implements javax.servlet.ServletContextListener{ modifiers); /* setup solr indexer */ - SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc); - /* if( solrIndexer.isIndexEmpty() ){ - log.info("solr index is empty, requesting rebuild"); - sce.getServletContext().setAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP, Boolean.TRUE); - } */ + SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc); // This is where the builder gets the list of places to try to // get objects to index. It is filtered so that non-public text @@ -102,32 +98,19 @@ public class SolrSetup implements javax.servlet.ServletContextListener{ WebappDaoFactory wadf = (WebappDaoFactory) context.getAttribute("webappDaoFactory"); VitroFilters vf = VitroFilterUtils.getPublicFilter(context); wadf = new WebappDaoFactoryFiltering(wadf, vf); + + IndexBuilder builder = new IndexBuilder(context, solrIndexer, wadf ); + // to the servlet context so we can access it later in the webapp. + context.setAttribute(IndexBuilder.class.getName(), builder); //make objects that will find additional URIs for context nodes etc List uriFinders = new ArrayList(); uriFinders.add( new AdditionalURIsForContextNodes(jenaOntModel) ); - IndexBuilder builder = new IndexBuilder(context, solrIndexer, wadf, uriFinders); - // to the servlet context so we can access it later in the webapp. - context.setAttribute(IndexBuilder.class.getName(), builder); - // set up listeners so search index builder is notified of changes to model ServletContext ctx = sce.getServletContext(); - SearchReindexingListener srl = new SearchReindexingListener(builder); + SearchReindexingListener srl = new SearchReindexingListener(builder, uriFinders); ModelContext.registerListenerForChanges(ctx, srl); - - /* if( sce.getServletContext().getAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP) instanceof Boolean && - (Boolean)sce.getServletContext().getAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP) ){ - log.info("Rebuild of solr index required before startup."); - builder.doIndexRebuild(); - int n = 0; - while( builder.isReindexRequested() || builder.isIndexing() ){ - n++; - if( n % 20 == 0 ) //output message every 10 sec. - log.info("Still rebuilding solr index"); - Thread.sleep(500); - } - }*/ log.info("Setup of Solr index completed."); } catch (Throwable e) {