From 4b2d55aa7580ee715ecdcd5098ff1eb3a501d054 Mon Sep 17 00:00:00 2001 From: anupsawant Date: Fri, 17 Jun 2011 15:34:20 +0000 Subject: [PATCH] updates to queries --- webapp/config/default.log4j.properties | 4 +- .../webapp/search/indexing/IndexBuilder.java | 10 +- .../search/indexing/IndexWorkerThread.java | 78 ++++++++++++++ .../search/solr/CalculateParameters.java | 101 +++++++++++++++--- .../vitro/webapp/search/solr/SolrIndexer.java | 6 +- .../vitro/webapp/search/solr/SolrSetup.java | 3 +- 6 files changed, 180 insertions(+), 22 deletions(-) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java diff --git a/webapp/config/default.log4j.properties b/webapp/config/default.log4j.properties index b7ddfdda5..9fc388165 100644 --- a/webapp/config/default.log4j.properties +++ b/webapp/config/default.log4j.properties @@ -39,7 +39,9 @@ log4j.logger.edu.cornell.mannlib.vitro.webapp.dao.jena.RDBGraphGenerator=WARN #log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.IndividualToSolrDocument=DEBUG #log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.CalculateParameters=DEBUG #log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.ContextNodeFields=DEBUG - +log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder=INFO +log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder=DEBUG +log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexThread=DEBUG # suppress odd warnings from libraries log4j.logger.org.openjena.riot=FATAL log4j.logger.org.directwebremoting=FATAL \ No newline at end of file diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java index d75961336..c95c837d1 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java @@ -22,6 +22,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary; import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.search.beans.ObjectSourceIface; + /** * The IndexBuilder is used to rebuild or update a search index. * There should only be one IndexBuilder in a vitro web application. @@ -282,6 +283,8 @@ public class IndexBuilder extends Thread { * @throws AbortIndexing */ private void indexForSource(Iterator individuals , boolean newDocs) throws AbortIndexing{ + + long starttime = System.currentTimeMillis(); long count = 0; while(individuals.hasNext()){ @@ -290,7 +293,8 @@ public class IndexBuilder extends Thread { Individual ind = null; try{ - ind = individuals.next(); + ind = individuals.next(); + indexer.index(ind, newDocs); }catch(Throwable ex){ if( stopRequested || log == null){//log might be null if system is shutting down. @@ -387,5 +391,7 @@ public class IndexBuilder extends Thread { private class AbortIndexing extends Exception { // Just a vanilla exception - } + } + } + diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java new file mode 100644 index 000000000..5a032cc7c --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java @@ -0,0 +1,78 @@ +package edu.cornell.mannlib.vitro.webapp.search.indexing; + +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Queue; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import edu.cornell.mannlib.vitro.webapp.beans.Individual; + + + +class IndexWorkerThread implements Runnable{ + + private IndexerIface indexer; + private static Log log = LogFactory.getLog(IndexWorkerThread.class); + private Queue indQueue = new LinkedList(); + + public IndexWorkerThread(IndexerIface indexer){ + + this.indexer = indexer; + } + + public void addToQueue(Individual ind, boolean newDocs){ + + } + + public void shutdown() { + + + } + + + public void run(){ + + //check for work + //if work found, + // translate + // send to server + //sleep (1000) + + } + + /*protected void indexInd() throws AbortIndexing{ + long starttime = System.currentTimeMillis(); + long count = 0; + Iterator individuals = firstList.iterator(); + while(individuals.hasNext()){ + if( stopRequested ) + throw new AbortIndexing(); + + Individual ind = null; + try{ + ind = individuals.next(); + indexer.index(ind, newDocs); + }catch(Throwable ex){ + if( stopRequested || log == null){//log might be null if system is shutting down. + throw new AbortIndexing(); + } + String uri = ind!=null?ind.getURI():"null"; + log.warn("Error indexing individual from separate thread" + uri + " " + ex.getMessage()); + } + count++; + if( log.isDebugEnabled() ){ + if( (count % 100 ) == 0 && count > 0 ){ + long dt = (System.currentTimeMillis() - starttime); + log.debug("individuals indexed from seperate thread: " + count + " in " + dt + " msec " + + " time pre individual from seperate thread = " + (dt / count) + " msec" ); + } + } + } + } + private class AbortIndexing extends Exception { + // Just a vanilla exception + } */ +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/CalculateParameters.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/CalculateParameters.java index dd0544af2..e8a504527 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/CalculateParameters.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/CalculateParameters.java @@ -14,6 +14,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.SolrInputField; +import com.hp.hpl.jena.query.Dataset; import com.hp.hpl.jena.query.Query; import com.hp.hpl.jena.query.QueryExecution; import com.hp.hpl.jena.query.QueryExecutionFactory; @@ -37,8 +38,8 @@ import edu.cornell.mannlib.vitro.webapp.search.VitroTermNames; public class CalculateParameters implements DocumentModifier { - Model fullModel; - int totalInd; + private Dataset dataset; + public static int totalInd=1; public static Map betaMap = new Hashtable(); private float phi; private static final String prefix = "prefix owl: " @@ -50,6 +51,17 @@ public class CalculateParameters implements DocumentModifier { + " prefix localNav: " + " prefix bibo: "; + private static final String betaQuery = prefix + " SELECT count(distinct ?inLinks) " + + " WHERE { " + + " ?uri rdf:type owl:Thing . " + + " ?inLinks ?prop ?uri . " + + " } "; + + private static final String totalCountQuery = prefix + " SELECT count(distinct ?ind) " + + " WHERE { " + + " ?ind rdf:type owl:Thing . " + + " } "; + private static Log log = LogFactory.getLog(CalculateParameters.class); private static final String[] fieldsToAddBetaTo = { @@ -64,20 +76,40 @@ public class CalculateParameters implements DocumentModifier { VitroTermNames.ALLTEXTUNSTEMMED, }; - public CalculateParameters(OntModel fullModel){ - this.fullModel=fullModel; - this.totalInd = fullModel.listIndividuals().toList().size(); + public CalculateParameters(Dataset dataset){ + this.dataset =dataset; + new Thread(new TotalInd(this.dataset,totalCountQuery)).start(); } public float calculateBeta(String uri){ - float beta=0; - RDFNode node = (Resource) fullModel.getResource(uri); - StmtIterator stmtItr = fullModel.listStatements((Resource)null, (Property)null,node); - int Conn = stmtItr.toList().size(); - beta = (float)Conn/totalInd; - beta *= 100; - beta += 1; - return beta; + float beta=0; + int Conn=0; + Query query; + QuerySolutionMap initialBinding = new QuerySolutionMap(); + QuerySolution soln = null; + Resource uriResource = ResourceFactory.createResource(uri); + initialBinding.add("uri", uriResource); + dataset.getLock().enterCriticalSection(Lock.READ); + + try{ + query = QueryFactory.create(betaQuery,Syntax.syntaxARQ); + QueryExecution qexec = QueryExecutionFactory.create(query,dataset,initialBinding); + ResultSet results = qexec.execSelect(); + List resultVars = results.getResultVars(); + if(resultVars!=null && resultVars.size()!=0){ + soln = results.next(); + Conn = Integer.parseInt(soln.getLiteral(resultVars.get(0)).getLexicalForm()); + } + }catch(Throwable t){ + log.error(t,t); + }finally{ + dataset.getLock().leaveCriticalSection(); + } + + beta = (float)Conn/totalInd; + beta *= 100; + beta += 1; + return beta; } public float calculatePhi(StringBuffer adjNodes){ @@ -185,7 +217,7 @@ public class CalculateParameters implements DocumentModifier { Iterator queryItr = queryList.iterator(); - fullModel.enterCriticalSection(Lock.READ); + dataset.getLock().enterCriticalSection(Lock.READ); Resource adjacentIndividual = null; RDFNode coauthor = null; try{ @@ -194,7 +226,7 @@ public class CalculateParameters implements DocumentModifier { queryItr.next(); // we don't want first query to execute if the ind is not a person. } query = QueryFactory.create(queryItr.next(),Syntax.syntaxARQ); - QueryExecution qexec = QueryExecutionFactory.create(query,fullModel,initialBinding); + QueryExecution qexec = QueryExecutionFactory.create(query,dataset,initialBinding); try{ ResultSet results = qexec.execSelect(); while(results.hasNext()){ @@ -235,7 +267,7 @@ public class CalculateParameters implements DocumentModifier { catch(Throwable t){ log.error(t,t); }finally{ - fullModel.leaveCriticalSection(); + dataset.getLock().leaveCriticalSection(); adjacentNodes = null; adjacentNodesConcat = null; coauthorBuff = null; @@ -283,3 +315,40 @@ public class CalculateParameters implements DocumentModifier { } } + +class TotalInd implements Runnable{ + private Dataset dataset; + private String totalCountQuery; + private static Log log = LogFactory.getLog(TotalInd.class); + + public TotalInd(Dataset dataset,String totalCountQuery){ + this.dataset = dataset; + this.totalCountQuery = totalCountQuery; + + } + public void run(){ + int totalInd=0; + Query query; + QuerySolution soln = null; + dataset.getLock().enterCriticalSection(Lock.READ); + + try{ + query = QueryFactory.create(totalCountQuery,Syntax.syntaxARQ); + QueryExecution qexec = QueryExecutionFactory.create(query,dataset); + ResultSet results = qexec.execSelect(); + List resultVars = results.getResultVars(); + + if(resultVars!=null && resultVars.size()!=0){ + soln = results.next(); + totalInd = Integer.parseInt(soln.getLiteral(resultVars.get(0)).getLexicalForm()); + } + CalculateParameters.totalInd = totalInd; + log.info("Total number of individuals in the system are : " + CalculateParameters.totalInd); + }catch(Throwable t){ + log.error(t,t); + }finally{ + dataset.getLock().leaveCriticalSection(); + } + + } +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java index c0fd9fc27..33ba1fa6d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java @@ -125,8 +125,10 @@ public class SolrIndexer implements IndexerIface { } catch(IOException e){ log.error("Could not commit to solr server", e); }finally{ - CalculateParameters.betaMap.clear(); - CalculateParameters.betaMap = null; + if(CalculateParameters.betaMap!=null){ + CalculateParameters.betaMap.clear(); + CalculateParameters.betaMap = null; + } } try { diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java index a2243d372..7464fc49f 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java @@ -76,7 +76,8 @@ public class SolrSetup implements javax.servlet.ServletContextListener{ List modifiers = new ArrayList(); // modifiers.add(new CalculateParameters(ModelContext.getJenaOntModel(context))); - modifiers.add(new ContextNodeFields( dataset )); + modifiers.add(new CalculateParameters(dataset)); + modifiers.add(new ContextNodeFields(dataset)); IndividualToSolrDocument indToSolrDoc = new IndividualToSolrDocument( new ProhibitedFromSearch(DisplayVocabulary.PRIMARY_LUCENE_INDEX_URI, displayOntModel),