From 83a5523ace522d1a99dfacff3a637301cfbb7bf1 Mon Sep 17 00:00:00 2001 From: Jim Blake Date: Tue, 20 Jan 2015 14:06:11 -0500 Subject: [PATCH] VIVO-869 Improve the display Show exclusions count as separate from deletions count. When there is an error on the Ajax call, display the error status and stop asking for refreshes. When a rebuild is requested, redirect to just status, so refreshing the page would not request another rebuild. --- .../searchIndexer/SearchIndexerStatus.java | 10 +++++-- .../search/controller/IndexController.java | 20 +++++++++---- .../search/controller/IndexHistory.java | 3 +- .../searchindex/tasks/UpdateUrisTask.java | 28 +++++++++++++++---- webapp/web/js/search/searchIndex.js | 13 +++++++-- .../freemarker/body/admin/searchIndex.ftl | 2 ++ .../body/admin/searchIndexStatus.ftl | 2 +- 7 files changed, 62 insertions(+), 16 deletions(-) diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java index c4d9848c6..89a7a5309 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/modules/searchIndexer/SearchIndexerStatus.java @@ -102,19 +102,25 @@ public class SearchIndexerStatus { } public static class UriCounts extends Counts { + private final int excluded; private final int deleted; private final int updated; private final int remaining; private final int total; - public UriCounts(int deleted, int updated, int remaining, int total) { + public UriCounts(int excluded, int deleted, int updated, int remaining, int total) { super(Type.URI_COUNTS); + this.excluded = excluded; this.deleted = deleted; this.updated = updated; this.remaining = remaining; this.total = total; } + public int getExcluded() { + return excluded; + } + public int getDeleted() { return deleted; } @@ -133,7 +139,7 @@ public class SearchIndexerStatus { @Override public String toString() { - return "[deleted=" + deleted + ", updated=" + updated + return "[excluded=" + excluded + ", deleted=" + deleted + ", updated=" + updated + ", remaining=" + remaining + ", total=" + total + "]"; } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java index 43f3e3676..f22cfcf6a 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java @@ -16,10 +16,13 @@ import org.apache.commons.logging.LogFactory; import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; +import edu.cornell.mannlib.vitro.webapp.auth.policy.PolicyHelper; +import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.AuthorizationRequest; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.RequestedAction; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.RedirectResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer; @@ -99,10 +102,6 @@ public class IndexController extends FreemarkerHttpServlet { @Override public void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException, ServletException { - if (!isAuthorizedToDisplayPage(req, resp, REQUIRED_ACTIONS)) { - return; - } - switch (RequestType.fromRequest(req)) { case STATUS: showStatus(req, resp); @@ -118,12 +117,17 @@ public class IndexController extends FreemarkerHttpServlet { return "Rebuild Search Index"; } + @Override + protected AuthorizationRequest requiredActions(VitroRequest vreq) { + return REQUIRED_ACTIONS; + } + @Override protected ResponseValues processRequest(VitroRequest vreq) { switch (RequestType.fromRequest(vreq)) { case REBUILD: requestRebuild(); - return showDisplay(); + return new RedirectResponseValues(PAGE_URL); default: return showDisplay(); } @@ -138,6 +142,12 @@ public class IndexController extends FreemarkerHttpServlet { private void showStatus(HttpServletRequest req, HttpServletResponse resp) throws IOException { + if (!PolicyHelper.isAuthorizedForActions(req, REQUIRED_ACTIONS)) { + resp.setStatus(HttpServletResponse.SC_FORBIDDEN); + resp.getWriter().write("You are not authorized to access this page."); + return; + } + try { Map body = new HashMap<>(); body.put("statusUrl", UrlBuilder.getUrl(PAGE_URL, "status", "true")); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java index 0bbac9364..75e18b962 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexHistory.java @@ -28,7 +28,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatu public class IndexHistory implements SearchIndexer.Listener { private static final Log log = LogFactory.getLog(IndexHistory.class); - private final static int MAX_EVENTS = 10; + private final static int MAX_EVENTS = 20; private final Deque events = new LinkedList<>(); @@ -84,6 +84,7 @@ public class IndexHistory implements SearchIndexer.Listener { } private void addCounts(UriCounts counts, Map map) { + map.put("excluded", counts.getExcluded()); map.put("updated", counts.getUpdated()); map.put("deleted", counts.getDeleted()); map.put("remaining", counts.getRemaining()); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java index d84f40799..3deb8c6ee 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java @@ -69,8 +69,7 @@ public class UpdateUrisTask implements Task { this.status = new Status(uris.size(), 500, listeners); this.searchEngine = ApplicationUtils.instance().getSearchEngine(); - - + } @Override @@ -86,8 +85,10 @@ public class UpdateUrisTask implements Task { break; } else { Individual ind = getIndividual(uri); - if (ind == null || isExcluded(ind)) { + if (ind == null) { deleteDocument(uri); + } else if (isExcluded(ind)) { + excludeDocument(uri); } else { updateDocument(ind); } @@ -133,6 +134,17 @@ public class UpdateUrisTask implements Task { } } + /** An exclusion is just a delete for different reasons. */ + private void excludeDocument(String uri) { + try { + searchEngine.deleteById(SearchIndexerUtils.getIdForUri(uri)); + status.incrementExclusions(); + log.debug("excluded '" + uri + "' from search index."); + } catch (Exception e) { + log.warn("Failed to exclude '" + uri + "' from search index", e); + } + } + private void updateDocument(Individual ind) { Runnable workUnit = new UpdateDocumentWorkUnit(ind, modifiers); pool.submit(workUnit, this); @@ -165,6 +177,7 @@ public class UpdateUrisTask implements Task { private final ListenerList listeners; private int updated = 0; private int deleted = 0; + private int excluded = 0; private Date since = new Date(); public Status(int total, int progressInterval, ListenerList listeners) { @@ -184,6 +197,11 @@ public class UpdateUrisTask implements Task { since = new Date(); } + public synchronized void incrementExclusions() { + excluded++; + since = new Date(); + } + private void maybeFireProgressEvent() { if (updated > 0 && updated % progressInterval == 0) { listeners.fireEvent(new Event(PROGRESS, @@ -192,9 +210,9 @@ public class UpdateUrisTask implements Task { } public synchronized SearchIndexerStatus getSearchIndexerStatus() { - int remaining = total - updated - deleted; + int remaining = total - updated - deleted - excluded; return new SearchIndexerStatus(PROCESSING_URIS, since, - new UriCounts(deleted, updated, remaining, total)); + new UriCounts(excluded, deleted, updated, remaining, total)); } } diff --git a/webapp/web/js/search/searchIndex.js b/webapp/web/js/search/searchIndex.js index 560136fa6..fca116ba6 100644 --- a/webapp/web/js/search/searchIndex.js +++ b/webapp/web/js/search/searchIndex.js @@ -9,8 +9,12 @@ function updateSearchIndexerStatus() { url: searchIndexerStatusUrl, dataType: "html", complete: function(xhr, status) { - updatePanelContents(xhr.responseText); - setTimeout(updateSearchIndexerStatus,5000); + if (xhr.status == 200) { + updatePanelContents(xhr.responseText); + setTimeout(updateSearchIndexerStatus,5000); + } else { + displayErrorMessage(xhr.status + " " + xhr.statusText); + } } }); } @@ -19,4 +23,9 @@ function updatePanelContents(contents) { document.getElementById("searchIndexerStatus").innerHTML = contents; } +function displayErrorMessage(message) { + document.getElementById("searchIndexerError").innerHTML = "

" + message + "

"; +} + + $(document).ready(updateSearchIndexerStatus()); diff --git a/webapp/web/templates/freemarker/body/admin/searchIndex.ftl b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl index 76fae4191..cd9330034 100644 --- a/webapp/web/templates/freemarker/body/admin/searchIndex.ftl +++ b/webapp/web/templates/freemarker/body/admin/searchIndex.ftl @@ -7,6 +7,8 @@

${i18n().search_index_status}

+
+
Search Indexer Status
diff --git a/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl b/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl index d4e97c3f0..804cad8dc 100644 --- a/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl +++ b/webapp/web/templates/freemarker/body/admin/searchIndexStatus.ftl @@ -64,7 +64,7 @@ <#macro showIndexerCounts countsType, counts> <#if countsType == "URI_COUNTS"> - Updated: ${counts.updated}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total} + Updated: ${counts.updated}, excluded: ${counts.excluded}, deleted: ${counts.deleted}, remaining: ${counts.remaining}, total: ${counts.total} <#elseif countsType == "STATEMENT_COUNTS"> Processed: ${counts.processed}, remaining: ${counts.remaining}, total: ${counts.total} <#elseif countsType == "REBUILD_COUNTS">