NIHVIVO-3454 Refine SolrSmokeTest. Produce better messages for Socket Timeout, Unknown host, or Connection Refused (wrong port). On non-fatal errors (status 400 or 404), try three times before giving up.

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java

@@ -4,8 +4,11 @@ package edu.cornell.mannlib.vitro.webapp.servlet.setup;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.ConnectException;
 import java.net.MalformedURLException;
+import java.net.SocketTimeoutException;
 import java.net.URL;
+import java.net.UnknownHostException;
 
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
@@ -14,6 +17,8 @@ import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.HttpException;
 import org.apache.commons.httpclient.HttpStatus;
 import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
@@ -29,6 +34,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
  * If we can't connect to Solr, add a Warning item to the StartupStatus.
  */
 public class SolrSmokeTest implements ServletContextListener {
+	private static final Log log = LogFactory.getLog(SolrSmokeTest.class);
 
 	@Override
 	public void contextInitialized(ServletContextEvent sce) {
@@ -64,9 +70,16 @@ public class SolrSmokeTest implements ServletContextListener {
 	}
 
 	private static class SolrSmokeTestThread extends VitroBackgroundThread {
+		/* Use this code instead of an exception to track socket timeout. */
+		private static final int SOCKET_TIMEOUT_STATUS = -500; 
+		
+		private static final long SLEEP_INTERVAL = 10000; // 10 seconds
 		private final SolrSmokeTest listener;
 		private final URL solrUrl;
 		private final StartupStatus ss;
+		private final HttpClient httpClient = new HttpClient();
+
+		private int statusCode;
 
 		public SolrSmokeTestThread(SolrSmokeTest listener, URL solrUrl,
 				StartupStatus ss) {
@@ -76,36 +89,91 @@ public class SolrSmokeTest implements ServletContextListener {
 			this.ss = ss;
 		}
 
+		/**
+		 * Try to connect until we suceed, until we detect an unrecoverable
+		 * error, or until we have failed 3 times.
+		 */
 		@Override
 		public void run() {
-			HttpClient client = new HttpClient();
-			GetMethod method = new GetMethod(solrUrl.toExternalForm());
 			try {
-				int statusCode = client.executeMethod(method);
+				tryToConnect();
-				InputStream stream = method.getResponseBodyAsStream();
+				if (!isDone()) {
-				stream.close();
+					sleep();
+					tryToConnect();
+					if (!isDone()) {
+						sleep();
+						tryToConnect();
+					}
+				}
 
 				if (statusCode == HttpStatus.SC_OK) {
 					reportSuccess();
 				} else if (statusCode == HttpStatus.SC_FORBIDDEN) {
 					warnForbidden();
+				} else if (statusCode == SOCKET_TIMEOUT_STATUS) {
+					warnSocketTimeout();
 				} else {
-					warnBadHttpStatus(statusCode);
+					warnBadHttpStatus();
 				}
 			} catch (HttpException e) {
 				warnProtocolViolation(e);
+			} catch (UnknownHostException e) {
+				warnUnknownHost(e);
+			} catch (ConnectException e) {
+				warnConnectionRefused(e);
 			} catch (IOException e) {
 				warnTransportError(e);
+			}
+		}
+
+		private void tryToConnect() throws IOException {
+			GetMethod method = new GetMethod(solrUrl.toExternalForm());
+			try {
+				log.debug("Trying to connect to Solr");
+				statusCode = httpClient.executeMethod(method);
+				log.debug("HTTP status was " + statusCode);
+
+				// clear the buffer.
+				InputStream stream = method.getResponseBodyAsStream();
+				stream.close();
+			} catch (SocketTimeoutException e) {
+				// Catch the exception so we can retry this.
+				// Save the status so we know why we failed.
+				statusCode = SOCKET_TIMEOUT_STATUS;
 			} finally {
 				method.releaseConnection();
 			}
 		}
 
+		/**
+		 * Stop trying to connect if we succeed, or if we receive an error that
+		 * won't change on retry.
+		 */
+		private boolean isDone() {
+			return (statusCode == HttpStatus.SC_OK)
+					|| (statusCode == HttpStatus.SC_FORBIDDEN);
+		}
+
+		private void sleep() {
+			try {
+				Thread.sleep(SLEEP_INTERVAL);
+			} catch (InterruptedException e) {
+				e.printStackTrace(); // Should never happen
+			}
+		}
+
 		private void reportSuccess() {
 			ss.info(listener,
 					"Successfully connected to the Solr search engine.");
 		}
 
+		private void warnSocketTimeout() {
+			ss.warning(listener, "Can't connect to the Solr search engine. "
+					+ "The socket connection has repeatedly timed out. "
+					+ "Check the value of vitro.local.solr.url in "
+					+ "deploy.properties. Is Solr responding at that URL?");
+		}
+
 		private void warnForbidden() {
 			ss.warning(listener, "Can't connect to the Solr search engine. "
 					+ "The Solr server will not accept connections from this "
@@ -115,9 +183,9 @@ public class SolrSmokeTest implements ServletContextListener {
 					+ "does it authorize access from this IP address?");
 		}
 
-		private void warnBadHttpStatus(int status) {
+		private void warnBadHttpStatus() {
 			ss.warning(listener, "Can't connect to the Solr search engine. "
-					+ "The Solr server returned a status code of " + status
+					+ "The Solr server returned a status code of " + statusCode
 					+ ". Check the value of vitro.local.solr.url in "
 					+ "deploy.properties.");
 		}
@@ -127,6 +195,21 @@ public class SolrSmokeTest implements ServletContextListener {
 					+ "Detected a protocol violation: " + e.getMessage(), e);
 		}
 
+		private void warnUnknownHost(UnknownHostException e) {
+			ss.warning(listener, "Can't connect to the Solr search engine. '"
+					+ e.getMessage() + "' is an unknown host."
+					+ "Check the value of vitro.local.solr.url in "
+					+ "deploy.properties.", e);
+		}
+
+		private void warnConnectionRefused(ConnectException e) {
+			ss.warning(listener, "Can't connect to the Solr search engine. "
+					+ "The host refused the connection. "
+					+ "Is it possible that the port number is incorrect? "
+					+ "Check the value of vitro.local.solr.url in "
+					+ "deploy.properties.", e);
+		}
+
 		private void warnTransportError(IOException e) {
 			ss.warning(listener, "Can't connect to the Solr search engine. "
 					+ "Detected a transport error: " + e.getMessage(), e);