[VIVO-1397] Faster and more reliable Solr indexing

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java

@@ -12,6 +12,7 @@ import javax.servlet.ServletContext;
 
 import org.apache.solr.client.solrj.SolrQuery;
 import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer;
 import org.apache.solr.client.solrj.impl.HttpSolrServer;
 import org.apache.solr.client.solrj.response.QueryResponse;
 
@@ -31,7 +32,8 @@ import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery;
  * The Solr-based implementation of SearchEngine.
  */
 public class SolrSearchEngine implements SearchEngine {
-	private HttpSolrServer server;
+	private HttpSolrServer queryEngine;
+	private ConcurrentUpdateSolrServer updateEngine;
 
 	/**
 	 * Set up the http connection with the solr server
@@ -51,14 +53,18 @@ public class SolrSearchEngine implements SearchEngine {
 		}
 
 		try {
-			server = new HttpSolrServer(solrServerUrlString);
-			server.setSoTimeout(10000); // socket read timeout
-			server.setConnectionTimeout(10000);
-			server.setDefaultMaxConnectionsPerHost(100);
-			server.setMaxTotalConnections(100);
-			server.setMaxRetries(1);
-			css.info("Set up the Solr search engine; URL = '"
-					+ solrServerUrlString + "'.");
+			queryEngine = new HttpSolrServer(solrServerUrlString);
+			queryEngine.setSoTimeout(10000); // socket read timeout
+			queryEngine.setConnectionTimeout(10000);
+			queryEngine.setDefaultMaxConnectionsPerHost(100);
+			queryEngine.setMaxTotalConnections(100);
+			queryEngine.setMaxRetries(1);
+
+			updateEngine = new ConcurrentUpdateSolrServer(solrServerUrlString, 100, 1);
+			updateEngine.setConnectionTimeout(10000);
+			updateEngine.setPollQueueTime(25);
+
+			css.info("Set up the Solr search engine; URL = '" + solrServerUrlString + "'.");
 		} catch (Exception e) {
 			css.fatal("Could not set up the Solr search engine", e);
 		}
@@ -66,13 +72,14 @@ public class SolrSearchEngine implements SearchEngine {
 
 	@Override
 	public void shutdown(Application application) {
-		server.shutdown();
+		queryEngine.shutdown();
+		updateEngine.shutdown();
 	}
 
 	@Override
 	public void ping() throws SearchEngineException {
 		try {
-			server.ping();
+			queryEngine.ping();
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException("Solr server did not respond to ping.",
 					e);
@@ -93,7 +100,7 @@ public class SolrSearchEngine implements SearchEngine {
 	public void add(Collection<SearchInputDocument> docs)
 			throws SearchEngineException {
 		try {
-			server.add(SolrConversionUtils.convertToSolrInputDocuments(docs));
+			updateEngine.add(SolrConversionUtils.convertToSolrInputDocuments(docs), 100);
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException("Solr server failed to add documents "
 					+ docs, e);
@@ -103,7 +110,8 @@ public class SolrSearchEngine implements SearchEngine {
 	@Override
 	public void commit() throws SearchEngineException {
 		try {
-			server.commit();
+			updateEngine.commit();
+			updateEngine.optimize();
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException("Failed to commit to Solr server.", e);
 		}
@@ -112,7 +120,8 @@ public class SolrSearchEngine implements SearchEngine {
 	@Override
 	public void commit(boolean wait) throws SearchEngineException {
 		try {
-			server.commit(wait, wait);
+			updateEngine.commit(wait, wait);
+			updateEngine.optimize(wait, wait);
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException("Failed to commit to Solr server.", e);
 		}
@@ -126,7 +135,7 @@ public class SolrSearchEngine implements SearchEngine {
 	@Override
 	public void deleteById(Collection<String> ids) throws SearchEngineException {
 		try {
-			server.deleteById(new ArrayList<>(ids));
+			updateEngine.deleteById(new ArrayList<>(ids), 100);
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException(
 					"Solr server failed to delete documents: " + ids, e);
@@ -136,7 +145,7 @@ public class SolrSearchEngine implements SearchEngine {
 	@Override
 	public void deleteByQuery(String query) throws SearchEngineException {
 		try {
-			server.deleteByQuery(query);
+			updateEngine.deleteByQuery(query, 100);
 		} catch (SolrServerException | IOException e) {
 			throw appropriateException(
 					"Solr server failed to delete documents: " + query, e);
@@ -159,7 +168,7 @@ public class SolrSearchEngine implements SearchEngine {
 	public SearchResponse query(SearchQuery query) throws SearchEngineException {
 		try {
 			SolrQuery solrQuery = SolrConversionUtils.convertToSolrQuery(query);
-			QueryResponse response = server.query(solrQuery);
+			QueryResponse response = queryEngine.query(solrQuery);
 			return SolrConversionUtils.convertToSearchResponse(response);
 		} catch (SolrServerException e) {
 			throw appropriateException(

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java

@@ -112,6 +112,7 @@ public class RebuildIndexTask implements Task {
                 if (!isInterrupted()) {
                     deleteOutdatedDocuments();
                 }
+                finalizeIndexing();
             }
 
             status = buildStatus(REBUILDING, getDocumentCount());
@@ -135,11 +136,22 @@ public class RebuildIndexTask implements Task {
             UpdateUrisTask.runNow(uris, excluders, modifiers, indDao, listeners, pool);
         }
 
+        private void finalizeIndexing() {
+            try {
+                searchEngine.commit();
+            } catch (SearchEngineNotRespondingException e) {
+                log.warn("Failed to finalize search index: "
+                        + "the search engine is not responding.");
+            } catch (SearchEngineException e) {
+                log.warn("Failed to finalize "
+                        + "from the search index", e);
+            }
+        }
+
         private void deleteOutdatedDocuments() {
             String query = "indexedTime:[ * TO " + requestedAt.getTime() + " ]";
             try {
                 searchEngine.deleteByQuery(query);
-                searchEngine.commit();
             } catch (SearchEngineNotRespondingException e) {
                 log.warn("Failed to delete outdated documents from the search index: "
                         + "the search engine is not responding.");

api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java

@@ -153,8 +153,6 @@ public class UpdateUrisTask implements Task {
             }
             pool.waitUntilIdle();
 
-            commitChanges();
-
             excluders.stopIndexing();
             modifiers.stopIndexing();
             listeners.fireEvent(new Event(STOP_URIS, status.getSearchIndexerStatus()));
@@ -221,9 +219,6 @@ public class UpdateUrisTask implements Task {
 
         private void fireEvent(Event event) {
             listeners.fireEvent(event);
-            if (event.getType() == PROGRESS || event.getType() == STOP_URIS) {
-                commitChanges();
-            }
         }
 
         private void commitChanges() {

home/src/main/resources/solr/conf/solrconfig.xml

@@ -35,7 +35,7 @@
        that you fully re-index after changing this setting as it can
        affect both how text is indexed and queried.
   -->
-  <luceneMatchVersion>4.7</luceneMatchVersion>
+  <luceneMatchVersion>4.10.4</luceneMatchVersion>
 
   <!-- <lib/> directives can be used to instruct Solr to load an Jars
        identified and use them to resolve any "plugins" specified in
@@ -387,12 +387,11 @@
          If the updateLog is enabled, then it's highly recommended to
          have some sort of hard autoCommit to limit the log size.
       -->
-      <!-- 
-     <autoCommit> 
-       <maxTime>${solr.autoCommit.maxTime:15000}</maxTime> 
+     <autoCommit>
+       <maxTime>60000</maxTime>
        <openSearcher>false</openSearcher> 
      </autoCommit>
--->
+
     <!-- softAutoCommit is like autoCommit except it causes a
          'soft' commit which only ensures that changes are visible
          but does not ensure that data is synced to disk.  This is