VIVO-869 Improve the display

Show exclusions count as separate from deletions count.
When there is an error on the Ajax call, display the error status and stop asking for refreshes.
When a rebuild is requested, redirect to just status, so refreshing the page would not request another rebuild.

webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java

@@ -102,19 +102,25 @@ public class SearchIndexerStatus {
 	}
 
 	public static class UriCounts extends Counts {
+		private final int excluded;
 		private final int deleted;
 		private final int updated;
 		private final int remaining;
 		private final int total;
 
-		public UriCounts(int deleted, int updated, int remaining, int total) {
+		public UriCounts(int excluded, int deleted, int updated, int remaining, int total) {
 			super(Type.URI_COUNTS);
+			this.excluded = excluded;
 			this.deleted = deleted;
 			this.updated = updated;
 			this.remaining = remaining;
 			this.total = total;
 		}
 
+		public int getExcluded() {
+			return excluded;
+		}
+		
 		public int getDeleted() {
 			return deleted;
 		}
@@ -133,7 +139,7 @@ public class SearchIndexerStatus {
 
 		@Override
 		public String toString() {
-			return "[deleted=" + deleted + ", updated=" + updated
+			return "[excluded=" + excluded + ", deleted=" + deleted + ", updated=" + updated
 					+ ", remaining=" + remaining + ", total=" + total + "]";
 		}
 	}

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java

@@ -16,10 +16,13 @@ import org.apache.commons.logging.LogFactory;
 
 import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
 import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
+import edu.cornell.mannlib.vitro.webapp.auth.policy.PolicyHelper;
+import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest;
 import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.RequestedAction;
 import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
+import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.RedirectResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
 import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer;
@@ -99,10 +102,6 @@ public class IndexController extends FreemarkerHttpServlet {
 	@Override
 	public void doGet(HttpServletRequest req, HttpServletResponse resp)
 			throws IOException, ServletException {
-		if (!isAuthorizedToDisplayPage(req, resp, REQUIRED_ACTIONS)) {
-			return;
-		}
-
 		switch (RequestType.fromRequest(req)) {
 		case STATUS:
 			showStatus(req, resp);
@@ -118,12 +117,17 @@ public class IndexController extends FreemarkerHttpServlet {
 		return "Rebuild Search Index";
 	}
 
+	@Override
+	protected AuthorizationRequest requiredActions(VitroRequest vreq) {
+		return REQUIRED_ACTIONS;
+	}
+
 	@Override
 	protected ResponseValues processRequest(VitroRequest vreq) {
 		switch (RequestType.fromRequest(vreq)) {
 		case REBUILD:
 			requestRebuild();
-			return showDisplay();
+			return new RedirectResponseValues(PAGE_URL);
 		default:
 			return showDisplay();
 		}
@@ -138,6 +142,12 @@ public class IndexController extends FreemarkerHttpServlet {
 
 	private void showStatus(HttpServletRequest req, HttpServletResponse resp)
 			throws IOException {
+		if (!PolicyHelper.isAuthorizedForActions(req, REQUIRED_ACTIONS)) {
+			resp.setStatus(HttpServletResponse.SC_FORBIDDEN);
+			resp.getWriter().write("You are not authorized to access this page.");
+			return;
+		}
+		
 		try {
 			Map<String, Object> body = new HashMap<>();
 			body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true"));

webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java

@@ -28,7 +28,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatu
 public class IndexHistory implements SearchIndexer.Listener {
 	private static final Log log = LogFactory.getLog(IndexHistory.class);
 
-	private final static int MAX_EVENTS = 10;
+	private final static int MAX_EVENTS = 20;
 
 	private final Deque<Event> events = new LinkedList<>();
 
@@ -84,6 +84,7 @@ public class IndexHistory implements SearchIndexer.Listener {
 	}
 
 	private void addCounts(UriCounts counts, Map<String, Object> map) {
+		map.put("excluded", counts.getExcluded());
 		map.put("updated", counts.getUpdated());
 		map.put("deleted", counts.getDeleted());
 		map.put("remaining", counts.getRemaining());

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java

@@ -69,8 +69,7 @@ public class UpdateUrisTask implements Task {
 		this.status = new Status(uris.size(), 500, listeners);
 
 		this.searchEngine = ApplicationUtils.instance().getSearchEngine();
-		
+
-		
 	}
 
 	@Override
@@ -86,8 +85,10 @@ public class UpdateUrisTask implements Task {
 				break;
 			} else {
 				Individual ind = getIndividual(uri);
-				if (ind == null || isExcluded(ind)) {
+				if (ind == null) {
 					deleteDocument(uri);
+				} else if (isExcluded(ind)) {
+					excludeDocument(uri);
 				} else {
 					updateDocument(ind);
 				}
@@ -133,6 +134,17 @@ public class UpdateUrisTask implements Task {
 		}
 	}
 
+	/** An exclusion is just a delete for different reasons. */
+	private void excludeDocument(String uri) {
+		try {
+			searchEngine.deleteById(SearchIndexerUtils.getIdForUri(uri));
+			status.incrementExclusions();
+			log.debug("excluded '" + uri + "' from search index.");
+		} catch (Exception e) {
+			log.warn("Failed to exclude '" + uri + "' from search index", e);
+		}
+	}
+
 	private void updateDocument(Individual ind) {
 		Runnable workUnit = new UpdateDocumentWorkUnit(ind, modifiers);
 		pool.submit(workUnit, this);
@@ -165,6 +177,7 @@ public class UpdateUrisTask implements Task {
 		private final ListenerList listeners;
 		private int updated = 0;
 		private int deleted = 0;
+		private int excluded = 0;
 		private Date since = new Date();
 
 		public Status(int total, int progressInterval, ListenerList listeners) {
@@ -184,6 +197,11 @@ public class UpdateUrisTask implements Task {
 			since = new Date();
 		}
 
+		public synchronized void incrementExclusions() {
+			excluded++;
+			since = new Date();
+		}
+
 		private void maybeFireProgressEvent() {
 			if (updated > 0 && updated % progressInterval == 0) {
 				listeners.fireEvent(new Event(PROGRESS,
@@ -192,9 +210,9 @@ public class UpdateUrisTask implements Task {
 		}
 
 		public synchronized SearchIndexerStatus getSearchIndexerStatus() {
-			int remaining = total - updated - deleted;
+			int remaining = total - updated - deleted - excluded;
 			return new SearchIndexerStatus(PROCESSING_URIS, since,
-					new UriCounts(deleted, updated, remaining, total));
+					new UriCounts(excluded, deleted, updated, remaining, total));
 		}
 
 	}

webapp/web/js/search/searchIndex.js

@@ -9,8 +9,12 @@ function updateSearchIndexerStatus() {
         url: searchIndexerStatusUrl,
         dataType: "html",
         complete: function(xhr, status) {
-        	updatePanelContents(xhr.responseText);
+            if (xhr.status == 200) {
-        	setTimeout(updateSearchIndexerStatus,5000);
+                updatePanelContents(xhr.responseText);
+                setTimeout(updateSearchIndexerStatus,5000);
+            } else {
+                displayErrorMessage(xhr.status + " " + xhr.statusText);
+            }
         }
     });
 }
@@ -19,4 +23,9 @@ function updatePanelContents(contents) {
 	document.getElementById("searchIndexerStatus").innerHTML = contents;
 }
 
+function displayErrorMessage(message) {
+	document.getElementById("searchIndexerError").innerHTML = "<h3>" + message + "</h3>";
+}
+
+
 $(document).ready(updateSearchIndexerStatus());

webapp/web/templates/freemarker/body/admin/searchIndex.ftl

@@ -7,6 +7,8 @@
 
 <h2>${i18n().search_index_status}</h2>
 
+<div id="searchIndexerError" />
+
 <div id="searchIndexerStatus">
 	Search Indexer Status
 </div>

webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl

@@ -64,7 +64,7 @@
 
 <#macro showIndexerCounts countsType, counts>
    <#if countsType == "URI_COUNTS">
-       Updated: ${counts.updated}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total}
+       Updated: ${counts.updated}, excluded: ${counts.excluded}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total}
    <#elseif countsType == "STATEMENT_COUNTS">
        Processed: ${counts.processed}, remaining: ${counts.remaining}, total: ${counts.total}
    <#elseif countsType == "REBUILD_COUNTS">