Defer tasks by passing passing in a reference for obtaining the config at runtime, rather than at creation, allowing tasks to be deferred during startup.

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerImpl.java

@@ -269,7 +269,7 @@ public class SearchIndexerImpl implements SearchIndexer {
             return;
         }
 
-		scheduler.scheduleTask(new UpdateStatementsTask.Deferrable(changes));
+		scheduler.scheduleTask(new UpdateStatementsTask(new IndexerConfigImpl(this), changes));
 		log.debug("Scheduled updates for " + changes.size() + " statements.");
 	}
 
@@ -287,7 +287,7 @@ public class SearchIndexerImpl implements SearchIndexer {
             return;
         }
 
-		scheduler.scheduleTask(new UpdateUrisTask.Deferrable(uris));
+		scheduler.scheduleTask(new UpdateUrisTask(new IndexerConfigImpl(this), uris));
 		log.debug("Scheduled updates for " + uris.size() + " uris.");
 	}
 
@@ -301,7 +301,7 @@ public class SearchIndexerImpl implements SearchIndexer {
             return;
         }
 
-		scheduler.scheduleTask(new RebuildIndexTask.Deferrable());
+		scheduler.scheduleTask(new RebuildIndexTask(new IndexerConfigImpl(this)));
 		log.debug("Scheduled a full rebuild.");
 	}
 
@@ -397,7 +397,7 @@ public class SearchIndexerImpl implements SearchIndexer {
 	 */
 	private static class Scheduler {
 		private final TaskQueue taskQueue;
-		private final List<DeferrableTask> deferredQueue;
+		private final List<Task> deferredQueue;
         private final SearchIndexerImpl indexer;
 		private volatile boolean started;
 		private volatile boolean paused;
@@ -405,7 +405,7 @@ public class SearchIndexerImpl implements SearchIndexer {
 		public Scheduler(SearchIndexerImpl indexer, TaskQueue taskQueue) {
             this.indexer = indexer;
 			this.taskQueue = taskQueue;
-			this.deferredQueue = new ArrayList<DeferrableTask>();
+			this.deferredQueue = new ArrayList<Task>();
 		}
 
 		public boolean isStarted() {
@@ -416,20 +416,13 @@ public class SearchIndexerImpl implements SearchIndexer {
 			return paused;
 		}
 
-        public synchronized void scheduleTask(DeferrableTask task) {
+		public synchronized void scheduleTask(Task task) {
             if (paused || !started) {
                 deferredQueue.add(task);
                 log.debug("added task to deferred queue: " + task);
             } else {
-                taskQueue.scheduleTask(task.makeRunnable(indexer.createFindersList(), indexer.createExcludersList(), indexer.createModifiersList(), indexer.wadf.getIndividualDao(), indexer.listeners, indexer.pool));
-            }
-        }
-		public synchronized void scheduleTask(Task task) {
-			if (started && !paused) {
 				taskQueue.scheduleTask(task);
 				log.debug("added task to task queue: " + task);
-			} else {
-                log.debug("indexer not running, task ignored: " + task);
             }
 		}
 
@@ -452,8 +445,8 @@ public class SearchIndexerImpl implements SearchIndexer {
 		}
 
 		private void processDeferredTasks() {
-			for (DeferrableTask task : deferredQueue) {
+			for (Task task : deferredQueue) {
-                taskQueue.scheduleTask(task.makeRunnable(indexer.createFindersList(), indexer.createExcludersList(), indexer.createModifiersList(), indexer.wadf.getIndividualDao(), indexer.listeners, indexer.pool));
+                taskQueue.scheduleTask(task);
 				log.debug("moved task from deferred queue to task queue: " + task);
 			}
 
@@ -561,10 +554,34 @@ public class SearchIndexerImpl implements SearchIndexer {
 		}
 	}
 
-    public static interface DeferrableTask {
+    /**
-        public Task makeRunnable(IndexingUriFinderList uriFinders, SearchIndexExcluderList excluders,
+     * Interface for tasks to access the Indexer config
-                                 DocumentModifierList modifiers, IndividualDao indDao,
+     */
-                                 ListenerList listeners, WorkerThreadPool pool);
+    public static interface IndexerConfig {
+        public IndexingUriFinderList uriFinderList();
+        public SearchIndexExcluderList excluderList();
+        public DocumentModifierList documentModifierList();
+        public IndividualDao individualDao();
+        public ListenerList listenerList();
+        public WorkerThreadPool workerThreadPool();
+    }
+
+    /**
+     * Implementation of IndexerConfig
+     * Defers access to the configuration until the task is running, so a Task
+     * created and deferred before the indexer is started will not cause a NullPointerException
+     */
+    private static class IndexerConfigImpl implements IndexerConfig {
+        private final SearchIndexerImpl sii;
+
+        public IndexerConfigImpl(SearchIndexerImpl sii) { this.sii = sii; }
+
+        public IndexingUriFinderList uriFinderList()       { return sii.createFindersList(); }
+        public SearchIndexExcluderList excluderList()      { return sii.createExcludersList(); }
+        public DocumentModifierList documentModifierList() { return sii.createModifiersList(); }
+        public IndividualDao individualDao()               { return sii.wadf.getIndividualDao(); }
+        public ListenerList listenerList()                 { return sii.listeners; }
+        public WorkerThreadPool workerThreadPool()         { return sii.pool; }
     }
 
 	public static interface Task extends Runnable {
@@ -671,4 +688,28 @@ public class SearchIndexerImpl implements SearchIndexer {
 		}
 
 	}
+
+    private static class StatementList {
+        List<Statement> changes;
+
+        public StatementList() {
+            changes = new ArrayList<Statement>();
+        }
+
+        public synchronized void addStatement(Statement stmt) {
+            changes.add(stmt);
+        }
+
+        public synchronized List<Statement> getStatements() {
+            try {
+                return new ArrayList<>(changes);
+            } finally {
+                changes.clear();
+            }
+        }
+
+        public synchronized int size() {
+            return changes.size();
+        }
+    };
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/RebuildIndexTask.java

@@ -10,8 +10,6 @@ import java.text.SimpleDateFormat;
 import java.util.Collection;
 import java.util.Date;
 
-import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl;
-import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.IndexingUriFinderList;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -24,7 +22,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Even
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.RebuildCounts;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State;
-import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.DeferrableTask;
+import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.IndexerConfig;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.ListenerList;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.Task;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.WorkerThreadPool;
@@ -40,6 +38,41 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExclud
  */
 public class RebuildIndexTask implements Task {
 	private static final Log log = LogFactory.getLog(RebuildIndexTask.class);
+	private final Date requestedAt;
+
+    private final IndexerConfig config;
+    private RebuildIndexTaskImpl impl;
+
+    public RebuildIndexTask(IndexerConfig config) {
+        this.config = config;
+        this.requestedAt = new Date();
+    }
+
+	@Override
+	public void run() {
+        impl = new RebuildIndexTaskImpl(config, requestedAt);
+        impl.run();
+	}
+
+	@Override
+	public SearchIndexerStatus getStatus() {
+        return impl == null ? null : impl.getStatus();
+	}
+
+	@Override
+	public void notifyWorkUnitCompletion(Runnable workUnit) {
+        if (impl != null) {
+            impl.notifyWorkUnitCompletion(workUnit);
+        }
+	}
+
+	@Override
+	public String toString() {
+		return "RebuildIndexTask[requestedAt=" + new SimpleDateFormat().format(requestedAt) + "]";
+	}
+
+    private static class RebuildIndexTaskImpl implements Task {
+        private final IndexerConfig config;
 
         private final IndividualDao indDao;
         private final SearchIndexExcluderList excluders;
@@ -53,27 +86,17 @@ public class RebuildIndexTask implements Task {
 
         private volatile SearchIndexerStatus status;
 
-    public static class Deferrable implements DeferrableTask {
+        public RebuildIndexTaskImpl(IndexerConfig config, Date requestedAt) {
-        public Deferrable() {}
+            this.config = config;
-
+            this.excluders = config.excluderList();
-        @Override
+            this.modifiers = config.documentModifierList();
-        public Task makeRunnable(IndexingUriFinderList uriFinders, SearchIndexExcluderList excluders, DocumentModifierList modifiers, IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
+            this.indDao = config.individualDao();
-            return new RebuildIndexTask(excluders, modifiers, indDao, listeners, pool);
+            this.listeners = config.listenerList();
-        }
+            this.pool = config.workerThreadPool();
-    }
-
-	public RebuildIndexTask(SearchIndexExcluderList excluders,
-			DocumentModifierList modifiers, IndividualDao indDao,
-			ListenerList listeners, WorkerThreadPool pool) {
-		this.excluders = excluders;
-		this.modifiers = modifiers;
-		this.indDao = indDao;
-		this.listeners = listeners;
-		this.pool = pool;
 
             this.searchEngine = ApplicationUtils.instance().getSearchEngine();
 
-		this.requestedAt = new Date();
+            this.requestedAt = requestedAt;
             this.documentsBefore = getDocumentCount();
             this.status = buildStatus(REBUILDING, 0);
         }
@@ -109,8 +132,7 @@ public class RebuildIndexTask implements Task {
         }
 
         private void updateTheUris(Collection<String> uris) {
-		new UpdateUrisTask(uris, excluders, modifiers, indDao, listeners, pool)
+            UpdateUrisTask.runNow(uris, excluders, modifiers, indDao, listeners, pool);
-				.run();
         }
 
         private void deleteOutdatedDocuments() {
@@ -161,5 +183,5 @@ public class RebuildIndexTask implements Task {
             return "RebuildIndexTask[requestedAt="
                     + new SimpleDateFormat().format(requestedAt) + "]";
         }
-
+    }
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java

@@ -24,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.StatementCounts;
-import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.DeferrableTask;
+import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.IndexerConfig;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.ListenerList;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.Task;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.WorkerThreadPool;
@@ -51,9 +51,36 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.IndexingUriFinderLi
  * Set to remove duplicates, and then process the URIs in the set.
  */
 public class UpdateStatementsTask implements Task {
-	private static final Log log = LogFactory
+    private static final Log log = LogFactory.getLog(UpdateStatementsTask.class);
-			.getLog(UpdateStatementsTask.class);
 
+    private final IndexerConfig config;
+    private UpdateStatementsTaskImpl impl;
+
+    private List<Statement> changes;
+
+    public UpdateStatementsTask(IndexerConfig config, List<Statement> changes) {
+        this.config = config;
+        this.changes = new ArrayList<>(changes);
+    }
+
+    @Override
+    public void run() {
+        impl = new UpdateStatementsTaskImpl(config, changes);
+        impl.run();
+    }
+    @Override
+    public SearchIndexerStatus getStatus() {
+        return impl == null ? null : impl.getStatus();
+    }
+
+    @Override
+    public void notifyWorkUnitCompletion(Runnable workUnit) {
+        if (impl != null) {
+            impl.notifyWorkUnitCompletion(workUnit);
+        }
+    }
+
+    private static class UpdateStatementsTaskImpl implements Task {
         private final List<Statement> changes;
         private final IndexingUriFinderList uriFinders;
         private final SearchIndexExcluderList excluders;
@@ -65,28 +92,14 @@ public class UpdateStatementsTask implements Task {
         private final Set<String> uris;
         private final Status status;
 
-    public static class Deferrable implements DeferrableTask {
+        public UpdateStatementsTaskImpl(IndexerConfig config, List<Statement> changes) {
-        List<Statement> changes;
+            this.changes = changes;
-
+            this.uriFinders = config.uriFinderList();
-        public Deferrable(List<Statement> changes) { this.changes = changes; }
+            this.excluders = config.excluderList();
-
+            this.modifiers = config.documentModifierList();
-        @Override
+            this.indDao = config.individualDao();
-        public Task makeRunnable(IndexingUriFinderList uriFinders, SearchIndexExcluderList excluders, DocumentModifierList modifiers, IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
+            this.listeners = config.listenerList();
-            return new UpdateStatementsTask(changes, uriFinders, excluders, modifiers, indDao, listeners, pool);
+            this.pool = config.workerThreadPool();
-        }
-    }
-
-    public UpdateStatementsTask(List<Statement> changes,
-			IndexingUriFinderList uriFinders,
-			SearchIndexExcluderList excluders, DocumentModifierList modifiers,
-			IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
-		this.changes = new ArrayList<>(changes);
-		this.uriFinders = uriFinders;
-		this.excluders = excluders;
-		this.modifiers = modifiers;
-		this.indDao = indDao;
-		this.listeners = listeners;
-		this.pool = pool;
 
             this.uris = Collections.synchronizedSet(new HashSet<String>());
 
@@ -141,8 +154,7 @@ public class UpdateStatementsTask implements Task {
         }
 
         private void updateTheUris() {
-		new UpdateUrisTask(uris, excluders, modifiers, indDao, listeners, pool)
+            UpdateUrisTask.runNow(uris, excluders, modifiers, indDao, listeners, pool);
-				.run();
         }
 
         @Override
@@ -203,5 +215,5 @@ public class UpdateStatementsTask implements Task {
             }
 
         }
-
+    }
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java

@@ -7,14 +7,8 @@ import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndex
 import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Event.Type.STOP_URIS;
 import static edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.State.PROCESSING_URIS;
 
-import java.util.Collection;
+import java.util.*;
-import java.util.Date;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
 
-import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl;
-import edu.cornell.mannlib.vitro.webapp.searchindex.indexing.IndexingUriFinderList;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -29,6 +23,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexer.Even
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerStatus.UriCounts;
 import edu.cornell.mannlib.vitro.webapp.modules.searchIndexer.SearchIndexerUtils;
+import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.IndexerConfig;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.ListenerList;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.Task;
 import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerImpl.WorkerThreadPool;
@@ -53,7 +48,46 @@ import edu.cornell.mannlib.vitro.webapp.searchindex.exclusions.SearchIndexExclud
 public class UpdateUrisTask implements Task {
     private static final Log log = LogFactory.getLog(UpdateUrisTask.class);
 
-	private final Set<String> uris;
+    private final IndexerConfig config;
+    private UpdateUrisTaskImpl impl;
+
+    private final Collection<String> uris;
+    private Date since = new Date();
+
+    public UpdateUrisTask(IndexerConfig config, Collection<String> uris) {
+        this.config = config;
+        this.uris = new HashSet<>(uris);
+    }
+
+    static void runNow(Collection<String> uris, SearchIndexExcluderList excluders, DocumentModifierList modifiers, IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
+        UpdateUrisTaskImpl impl = new UpdateUrisTaskImpl(uris, excluders, modifiers, indDao, listeners, pool);
+        impl.run();
+    }
+
+    @Override
+    public void run() {
+        impl = new UpdateUrisTaskImpl(config, uris);
+        impl.run();
+    }
+
+    @Override
+    public SearchIndexerStatus getStatus() {
+        if (impl != null) {
+            return impl.getStatus();
+        }
+
+        return new SearchIndexerStatus(PROCESSING_URIS, since, new UriCounts(0, 0, 0, uris.size(), uris.size()));
+    }
+
+    @Override
+    public void notifyWorkUnitCompletion(Runnable workUnit) {
+        if (impl != null) {
+            impl.notifyWorkUnitCompletion(workUnit);
+        }
+    }
+
+    private static class UpdateUrisTaskImpl implements Task {
+        private final Collection<String> uris;
         private final IndividualDao indDao;
         private final SearchIndexExcluderList excluders;
         private final DocumentModifierList modifiers;
@@ -63,36 +97,34 @@ public class UpdateUrisTask implements Task {
         private final Status status;
         private final SearchEngine searchEngine;
 
-    public static class Deferrable implements SearchIndexerImpl.DeferrableTask {
+        public UpdateUrisTaskImpl(IndexerConfig config, Collection<String> uris) {
-        Collection<String> uris;
+            this.excluders = config.excluderList();
-        public Deferrable(Collection<String> uris) { this.uris = uris; }
+            this.modifiers = config.documentModifierList();
+            this.indDao = config.individualDao();
+            this.listeners = config.listenerList();
+            this.pool = config.workerThreadPool();
 
-        @Override
+            this.uris = uris;
-        public Task makeRunnable(IndexingUriFinderList uriFinders, SearchIndexExcluderList excluders, DocumentModifierList modifiers, IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
+            this.status = new Status(this, uris.size(), 500);
-            return new UpdateUrisTask(uris, excluders, modifiers, indDao, listeners, pool);
+
-        }
+            this.searchEngine = ApplicationUtils.instance().getSearchEngine();
         }
 
-    public UpdateUrisTask(Collection<String> uris,
+        public UpdateUrisTaskImpl(Collection<String> uris, SearchIndexExcluderList excluders, DocumentModifierList modifiers, IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
-			SearchIndexExcluderList excluders, DocumentModifierList modifiers,
+            this.uris = uris;
-			IndividualDao indDao, ListenerList listeners, WorkerThreadPool pool) {
-		this.uris = new HashSet<>(uris);
             this.excluders = excluders;
             this.modifiers = modifiers;
             this.indDao = indDao;
             this.listeners = listeners;
             this.pool = pool;
-
             this.status = new Status(this, uris.size(), 500);
 
             this.searchEngine = ApplicationUtils.instance().getSearchEngine();
-
         }
 
         @Override
         public void run() {
-		listeners.fireEvent(new Event(START_URIS, status
+            listeners.fireEvent(new Event(START_URIS, status.getSearchIndexerStatus()));
-				.getSearchIndexerStatus()));
             excluders.startIndexing();
             modifiers.startIndexing();
 
@@ -119,8 +151,7 @@ public class UpdateUrisTask implements Task {
 
             excluders.stopIndexing();
             modifiers.stopIndexing();
-		listeners.fireEvent(new Event(STOP_URIS, status
+            listeners.fireEvent(new Event(STOP_URIS, status.getSearchIndexerStatus()));
-				.getSearchIndexerStatus()));
         }
 
         private boolean isInterrupted() {
@@ -144,7 +175,9 @@ public class UpdateUrisTask implements Task {
             return excluders.isExcluded(ind);
         }
 
-	/** A delete is fast enough to be done synchronously. */
+        /**
+         * A delete is fast enough to be done synchronously.
+         */
         private void deleteDocument(String uri) {
             try {
                 searchEngine.deleteById(SearchIndexerUtils.getIdForUri(uri));
@@ -158,7 +191,9 @@ public class UpdateUrisTask implements Task {
             }
         }
 
-	/** An exclusion is just a delete for different reasons. */
+        /**
+         * An exclusion is just a delete for different reasons.
+         */
         private void excludeDocument(String uri) {
             try {
                 searchEngine.deleteById(SearchIndexerUtils.getIdForUri(uri));
@@ -205,16 +240,12 @@ public class UpdateUrisTask implements Task {
             return status.getSearchIndexerStatus();
         }
 
-	// ----------------------------------------------------------------------
-	// helper classes
-	// ----------------------------------------------------------------------
-
         /**
          * A thread-safe collection of status information. All methods are
          * synchronized.
          */
         private static class Status {
-		private final UpdateUrisTask parent;
+            private final UpdateUrisTaskImpl parent;
             private final int total;
             private final int progressInterval;
             private int updated = 0;
@@ -222,7 +253,7 @@ public class UpdateUrisTask implements Task {
             private int excluded = 0;
             private Date since = new Date();
 
-		public Status(UpdateUrisTask parent, int total, int progressInterval) {
+            public Status(UpdateUrisTaskImpl parent, int total, int progressInterval) {
                 this.parent = parent;
                 this.total = total;
                 this.progressInterval = progressInterval;
@@ -255,9 +286,12 @@ public class UpdateUrisTask implements Task {
                 return new SearchIndexerStatus(PROCESSING_URIS, since,
                         new UriCounts(excluded, deleted, updated, remaining, total));
             }
-
+        }
     }
 
+    // ----------------------------------------------------------------------
+    // helper classes
+    // ----------------------------------------------------------------------
     /**
      * This will be first in the list of SearchIndexExcluders.
      */
@@ -275,6 +309,5 @@ public class UpdateUrisTask implements Task {
         public String toString() {
             return "Internal: ExcludeIfNoVClasses";
         }
-
    }
 }