diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java index f1e31b18c..be3738ea4 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchengine/solr/SolrSearchEngine.java @@ -12,6 +12,7 @@ import javax.servlet.ServletContext; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrServer; import org.apache.solr.client.solrj.impl.HttpSolrServer; import org.apache.solr.client.solrj.response.QueryResponse; @@ -31,7 +32,8 @@ import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchQuery; * The Solr-based implementation of SearchEngine. */ public class SolrSearchEngine implements SearchEngine { - private HttpSolrServer server; + private HttpSolrServer queryEngine; + private ConcurrentUpdateSolrServer updateEngine; /** * Set up the http connection with the solr server @@ -51,14 +53,18 @@ public class SolrSearchEngine implements SearchEngine { } try { - server = new HttpSolrServer(solrServerUrlString); - server.setSoTimeout(10000); // socket read timeout - server.setConnectionTimeout(10000); - server.setDefaultMaxConnectionsPerHost(100); - server.setMaxTotalConnections(100); - server.setMaxRetries(1); - css.info("Set up the Solr search engine; URL = '" - + solrServerUrlString + "'."); + queryEngine = new HttpSolrServer(solrServerUrlString); + queryEngine.setSoTimeout(10000); // socket read timeout + queryEngine.setConnectionTimeout(10000); + queryEngine.setDefaultMaxConnectionsPerHost(100); + queryEngine.setMaxTotalConnections(100); + queryEngine.setMaxRetries(1); + + updateEngine = new ConcurrentUpdateSolrServer(solrServerUrlString, 100, 1); + updateEngine.setConnectionTimeout(10000); + updateEngine.setPollQueueTime(25); + + css.info("Set up the Solr search engine; URL = '" + solrServerUrlString + "'."); } catch (Exception e) { css.fatal("Could not set up the Solr search engine", e); } @@ -66,13 +72,14 @@ public class SolrSearchEngine implements SearchEngine { @Override public void shutdown(Application application) { - server.shutdown(); + queryEngine.shutdown(); + updateEngine.shutdown(); } @Override public void ping() throws SearchEngineException { try { - server.ping(); + queryEngine.ping(); } catch (SolrServerException | IOException e) { throw appropriateException("Solr server did not respond to ping.", e); @@ -93,7 +100,7 @@ public class SolrSearchEngine implements SearchEngine { public void add(Collection docs) throws SearchEngineException { try { - server.add(SolrConversionUtils.convertToSolrInputDocuments(docs)); + updateEngine.add(SolrConversionUtils.convertToSolrInputDocuments(docs), 100); } catch (SolrServerException | IOException e) { throw appropriateException("Solr server failed to add documents " + docs, e); @@ -103,7 +110,8 @@ public class SolrSearchEngine implements SearchEngine { @Override public void commit() throws SearchEngineException { try { - server.commit(); + updateEngine.commit(); + updateEngine.optimize(); } catch (SolrServerException | IOException e) { throw appropriateException("Failed to commit to Solr server.", e); } @@ -112,7 +120,8 @@ public class SolrSearchEngine implements SearchEngine { @Override public void commit(boolean wait) throws SearchEngineException { try { - server.commit(wait, wait); + updateEngine.commit(wait, wait); + updateEngine.optimize(wait, wait); } catch (SolrServerException | IOException e) { throw appropriateException("Failed to commit to Solr server.", e); } @@ -126,7 +135,7 @@ public class SolrSearchEngine implements SearchEngine { @Override public void deleteById(Collection ids) throws SearchEngineException { try { - server.deleteById(new ArrayList<>(ids)); + updateEngine.deleteById(new ArrayList<>(ids), 100); } catch (SolrServerException | IOException e) { throw appropriateException( "Solr server failed to delete documents: " + ids, e); @@ -136,7 +145,7 @@ public class SolrSearchEngine implements SearchEngine { @Override public void deleteByQuery(String query) throws SearchEngineException { try { - server.deleteByQuery(query); + updateEngine.deleteByQuery(query, 100); } catch (SolrServerException | IOException e) { throw appropriateException( "Solr server failed to delete documents: " + query, e); @@ -159,7 +168,7 @@ public class SolrSearchEngine implements SearchEngine { public SearchResponse query(SearchQuery query) throws SearchEngineException { try { SolrQuery solrQuery = SolrConversionUtils.convertToSolrQuery(query); - QueryResponse response = server.query(solrQuery); + QueryResponse response = queryEngine.query(solrQuery); return SolrConversionUtils.convertToSearchResponse(response); } catch (SolrServerException e) { throw appropriateException( diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java index 0a029c53a..375cbadcf 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java @@ -112,6 +112,7 @@ public class RebuildIndexTask implements Task { if (!isInterrupted()) { deleteOutdatedDocuments(); } + finalizeIndexing(); } status = buildStatus(REBUILDING, getDocumentCount()); @@ -135,11 +136,22 @@ public class RebuildIndexTask implements Task { UpdateUrisTask.runNow(uris, excluders, modifiers, indDao, listeners, pool); } + private void finalizeIndexing() { + try { + searchEngine.commit(); + } catch (SearchEngineNotRespondingException e) { + log.warn("Failed to finalize search index: " + + "the search engine is not responding."); + } catch (SearchEngineException e) { + log.warn("Failed to finalize " + + "from the search index", e); + } + } + private void deleteOutdatedDocuments() { String query = "indexedTime:[ * TO " + requestedAt.getTime() + " ]"; try { searchEngine.deleteByQuery(query); - searchEngine.commit(); } catch (SearchEngineNotRespondingException e) { log.warn("Failed to delete outdated documents from the search index: " + "the search engine is not responding."); diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java index bdfd44dbf..8e9a7c363 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java @@ -153,8 +153,6 @@ public class UpdateUrisTask implements Task { } pool.waitUntilIdle(); - commitChanges(); - excluders.stopIndexing(); modifiers.stopIndexing(); listeners.fireEvent(new Event(STOP_URIS, status.getSearchIndexerStatus())); @@ -221,9 +219,6 @@ public class UpdateUrisTask implements Task { private void fireEvent(Event event) { listeners.fireEvent(event); - if (event.getType() == PROGRESS || event.getType() == STOP_URIS) { - commitChanges(); - } } private void commitChanges() { diff --git a/home/src/main/resources/solr/conf/solrconfig.xml b/home/src/main/resources/solr/conf/solrconfig.xml index ca36b0a83..86bc84080 100644 --- a/home/src/main/resources/solr/conf/solrconfig.xml +++ b/home/src/main/resources/solr/conf/solrconfig.xml @@ -35,7 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 4.7 + 4.10.4 - +