litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve thread-safety in IndexWorkerThread counter. Add a "to-do" count. Make both counts publicly accessible.

Browse source
This commit is contained in:
j2blake 2011-11-25 18:48:21 +00:00
parent 5bd41ef88a
commit 89f753a728
2 changed files with 33 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -339,8 +339,6 @@ public class IndexBuilder extends VitroBackgroundThread {
if( numberOfThreads > MAX_THREADS ) if( numberOfThreads > MAX_THREADS )
numberOfThreads = MAX_THREADS; numberOfThreads = MAX_THREADS;
IndexWorkerThread.setStartTime(System.currentTimeMillis());
//make lists of work URIs for workers //make lists of work URIs for workers
List<List<String>> workLists = makeWorkerUriLists(updateUris, numberOfThreads); List<List<String>> workLists = makeWorkerUriLists(updateUris, numberOfThreads);
@ -351,6 +349,9 @@ public class IndexBuilder extends VitroBackgroundThread {
workers.add( new IndexWorkerThread(indexer, i, workToDo) ); workers.add( new IndexWorkerThread(indexer, i, workToDo) );
} }
// reset the counters so we can monitor the progress
IndexWorkerThread.resetCounters(System.currentTimeMillis(), figureWorkLoad(workLists));
log.debug("Starting the building and indexing of documents in worker threads"); log.debug("Starting the building and indexing of documents in worker threads");
// starting worker threads // starting worker threads
for(int i =0; i < numberOfThreads; i++){ for(int i =0; i < numberOfThreads; i++){
@ -371,8 +372,6 @@ public class IndexBuilder extends VitroBackgroundThread {
return; return;
} }
} }
IndexWorkerThread.resetCount();
} }
/* maybe ObjectSourceIface should be replaced with just an iterator. */ /* maybe ObjectSourceIface should be replaced with just an iterator. */
@ -423,6 +422,14 @@ public class IndexBuilder extends VitroBackgroundThread {
return work; return work;
} }
private long figureWorkLoad(List<List<String>> workLists) {
long load = 0;
for (List<String> list: workLists) {
load += list.size();
}
return load;
}
private static class UriLists { private static class UriLists {
private final List<String> updatedUris = new ArrayList<String>(); private final List<String> updatedUris = new ArrayList<String>();
private final List<String> deletedUris = new ArrayList<String>(); private final List<String> deletedUris = new ArrayList<String>();

27
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java
View file

@ -3,6 +3,7 @@
package edu.cornell.mannlib.vitro.webapp.search.indexing; package edu.cornell.mannlib.vitro.webapp.search.indexing;
import java.util.Iterator; import java.util.Iterator;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
@ -21,7 +22,8 @@ class IndexWorkerThread extends Thread{
protected boolean stopRequested = false; protected boolean stopRequested = false;
private Log log = LogFactory.getLog(IndexWorkerThread.class); private Log log = LogFactory.getLog(IndexWorkerThread.class);
private static long count=0; private static AtomicLong countCompleted= new AtomicLong();
private static AtomicLong countToIndex= new AtomicLong();
private static long starttime = 0; private static long starttime = 0;
public IndexWorkerThread(IndexerIface indexer, int threadNum , Iterator<Individual> individualsToIndex){ public IndexWorkerThread(IndexerIface indexer, int threadNum , Iterator<Individual> individualsToIndex){
@ -71,14 +73,12 @@ class IndexWorkerThread extends Thread{
} }
synchronized(this){ long countNow = countCompleted.incrementAndGet();
count++;
if( log.isInfoEnabled() ){ if( log.isInfoEnabled() ){
if( (count % 100 ) == 0 && count > 0 ){ if( (countNow % 100 ) == 0 ){
long dt = (System.currentTimeMillis() - starttime); long dt = (System.currentTimeMillis() - starttime);
log.info("individuals indexed: " + count + " in " + dt + " msec " + log.info("individuals indexed: " + countNow + " in " + dt + " msec " +
" time per individual = " + (dt / count) + " msec" ); " time per individual = " + (dt / countNow) + " msec" );
}
} }
} }
}catch(Throwable th){ }catch(Throwable th){
@ -89,12 +89,17 @@ class IndexWorkerThread extends Thread{
} }
} }
public static void resetCount(){ public static void resetCounters(long time, long workload) {
count = 0; IndexWorkerThread.starttime = time;
IndexWorkerThread.countToIndex.set(workload);
IndexWorkerThread.countCompleted.set(0);
} }
public static void setStartTime(long startTime){ public static long getCount() {
starttime = startTime; return countCompleted.get();
} }
public static long getCountToIndex() {
return countToIndex.get();
}
} }