diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index 0e67fdf2a..a6c7928ad 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -243,18 +243,19 @@ public class IndexBuilder extends Thread { indexer.startIndexing(); reindexRequested = false; - if( ! forceNewIndex ){ + if( ! forceNewIndex ){ + //if this is not a full reindex, deleted indivdiuals + //need to be removed from the index for(String deleteMe : deletes ){ try{ indexer.removeFromIndex(deleteMe); - }catch(Exception ex){ - log.debug(ex.getMessage()); + }catch(Exception ex){ log.debug("could not remove individual " + deleteMe + " from index, usually this is harmless",ex); } } - } - + } + indexUriList(updates, newDocs); } catch (AbortIndexing abort){ if( log != null) @@ -313,10 +314,7 @@ public class IndexBuilder extends Thread { } IndexWorkerThread.resetCount(); - } - - - + } /* maybe ObjectSourceIface should be replaced with just an iterator. */ protected class UriToIndividualIterator implements Iterator{ diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java index 0fa5938bf..50ebdfac7 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexerIface.java @@ -21,12 +21,8 @@ import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface; */ public interface IndexerIface { - public void addObj2Doc(Obj2DocIface o2d); - public List getObj2DocList(); - /** - * Check if indexing is currently running in a different thread. - * @return + * Check if indexing is currently running. */ public boolean isIndexing(); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java index e1947f03d..6632287c7 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java @@ -98,7 +98,7 @@ public class IndividualToSolrDocument { addThumbnailExistance(ind, doc); //time of index in millis past epoc - doc.addField(term.INDEXEDTIME,(new DateTime()).getMillis()); + doc.addField(term.INDEXEDTIME, new Long( (new DateTime()).getMillis() ) ); if(!prohibited){ addAllText( ind, doc, classPublicNames, objectNames ); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java index 48ce7c454..a54dc617d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java @@ -21,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.search.IndexingException; import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexerIface; + public class SolrIndexer implements IndexerIface { private final static Log log = LogFactory.getLog(SolrIndexer.class); @@ -29,6 +30,25 @@ public class SolrIndexer implements IndexerIface { protected HashSet urisIndexed; protected IndividualToSolrDocument individualToSolrDoc; + /** + * System is shutting down if true. + */ + protected boolean shutdownRequested = false; + + /** + * This records when a full re-index starts so that once it is done + * all the documents on the Solr service that are earlier than the + * reindexStart can be removed. + */ + protected long reindexStart = 0L; + + /** + * If true, then a full index rebuild was requested and reindexStart + * will be used to determine what documents to remove from the index + * once the re-index is complete. + */ + protected boolean doingFullIndexRebuild = false; + public SolrIndexer( SolrServer server, IndividualToSolrDocument indToDoc){ this.server = server; this.individualToSolrDoc = indToDoc; @@ -56,57 +76,18 @@ public class SolrIndexer implements IndexerIface { solrDoc = individualToSolrDoc.translate(ind); if( solrDoc != null){ - //sending each doc individually is inefficient - // Collection docs = new ArrayList(); - // docs.add( solrDoc ); UpdateResponse res = server.add( solrDoc ); log.debug("response after adding docs to server: "+ res); }else{ - log.debug("removing from index " + ind.getURI()); - //TODO: how do we delete document? - //writer.deleteDocuments((Term)obj2doc.getIndexId(ind)); + log.debug("removing from index " + ind.getURI()); + removeFromIndex(ind.getURI()); } } } catch (IOException ex) { throw new IndexingException(ex.getMessage()); } catch (SolrServerException ex) { throw new IndexingException(ex.getMessage()); - } - - if( ! indexing ) - throw new IndexingException("SolrIndexer: must call " + - "startIndexing() before index()."); - - if( ind == null ) - log.debug("Individual to index was null, ignoring."); - - try{ - if( urisIndexed.contains(ind.getURI()) ){ - log.debug("already indexed " + ind.getURI() ); - return; - }else{ - SolrInputDocument solrDoc = null; - synchronized(this){ - urisIndexed.add(ind.getURI()); - } - log.debug("indexing " + ind.getURI()); - - solrDoc = individualToSolrDoc.translate(ind); - - if( solrDoc != null){ - UpdateResponse res = server.add( solrDoc ); - log.debug("response after adding docs to server: "+ res); - }else{ - log.debug("removing from index " + ind.getURI()); - //TODO: how do we delete document? - //writer.deleteDocuments((Term)obj2doc.getIndexId(ind)); - } - } - } catch (IOException ex) { - throw new IndexingException(ex.getMessage()); - } catch (SolrServerException ex) { - throw new IndexingException(ex.getMessage()); - } + } } @Override @@ -116,7 +97,8 @@ public class SolrIndexer implements IndexerIface { @Override public void prepareForRebuild() throws IndexingException { - // TODO Auto-generated method stub + reindexStart = System.currentTimeMillis(); + doingFullIndexRebuild = true; } @Override @@ -134,30 +116,23 @@ public class SolrIndexer implements IndexerIface { } @Override - public synchronized void startIndexing() throws IndexingException { - while( indexing ){ //wait for indexing to end. - log.debug("SolrIndexer.startIndexing() waiting..."); - try{ wait(); } catch(InterruptedException ex){} + public synchronized void startIndexing() throws IndexingException { + if( indexing) + log.debug("SolrIndexer.startIndexing() Indexing in progress, waiting for completion..."); + while( indexing && ! shutdownRequested ){ //wait for indexing to end. + try{ wait( 250 ); } + catch(InterruptedException ex){} } - log.debug("Starting to index"); + log.debug("Starting to index"); indexing = true; urisIndexed = new HashSet(); notifyAll(); - } - - - public synchronized void addObj2Doc(Obj2DocIface o2d) { - //no longer used - } - - public synchronized List getObj2DocList() { - //no longer used - return null; - } + } @Override public void abortIndexingAndCleanUp() { + shutdownRequested = true; try{ server.commit(); }catch(SolrServerException e){ @@ -187,15 +162,25 @@ public class SolrIndexer implements IndexerIface { } catch(IOException e){ log.error("Could not commit to solr server", e); } -// try { -// server.optimize(); -// } catch (Exception e) { -// log.error("Could not optimize solr server", e); -// } + if( doingFullIndexRebuild ){ + removeDocumentsFromBeforeRebuild( ); + } indexing = false; notifyAll(); } + protected void removeDocumentsFromBeforeRebuild(){ + try { + server.deleteByQuery("indexedTime:[ * TO " + reindexStart + " ]"); + server.commit(); + } catch (SolrServerException e) { + log.error("could not delete documents from before rebuild.",e); + } catch (IOException e) { + log.error("could not delete documents from before rebuild.",e); + } + } + + @Override public long getModified() { long modified = 0; @@ -211,7 +196,6 @@ public class SolrIndexer implements IndexerIface { modified = (Long)docs.get(0).getFieldValue("indexedTime"); } } catch (SolrServerException e) { - // TODO Auto-generated catch block log.error(e,e); } @@ -228,14 +212,9 @@ public class SolrIndexer implements IndexerIface { return true; } } catch (SolrServerException e) { - // TODO Auto-generated catch block log.error(e,e); } return false; } - - - - }