litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updates to queries

Browse source
This commit is contained in:
anupsawant 2011-06-17 15:34:20 +00:00
parent 107b5c6745
commit 4b2d55aa75
6 changed files with 180 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

4
webapp/config/default.log4j.properties
View file

@ -39,7 +39,9 @@ log4j.logger.edu.cornell.mannlib.vitro.webapp.dao.jena.RDBGraphGenerator=WARN
#log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.IndividualToSolrDocument=DEBUG
#log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.CalculateParameters=DEBUG
#log4j.logger.edu.cornell.mannlib.vitro.webapp.search.solr.ContextNodeFields=DEBUG
log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder=INFO
log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder=DEBUG
log4j.logger.edu.cornell.mannlib.vitro.webapp.search.indexing.IndexThread=DEBUG
# suppress odd warnings from libraries
log4j.logger.org.openjena.riot=FATAL
log4j.logger.org.directwebremoting=FATAL

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -22,6 +22,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.search.beans.ObjectSourceIface;
/**
* The IndexBuilder is used to rebuild or update a search index.
* There should only be one IndexBuilder in a vitro web application.
@ -282,6 +283,8 @@ public class IndexBuilder extends Thread {
* @throws AbortIndexing
*/
private void indexForSource(Iterator<Individual> individuals , boolean newDocs) throws AbortIndexing{
long starttime = System.currentTimeMillis();
long count = 0;
while(individuals.hasNext()){
@ -291,6 +294,7 @@ public class IndexBuilder extends Thread {
Individual ind = null;
try{
ind = individuals.next();
indexer.index(ind, newDocs);
}catch(Throwable ex){
if( stopRequested || log == null){//log might be null if system is shutting down.
@ -388,4 +392,6 @@ public class IndexBuilder extends Thread {
private class AbortIndexing extends Exception {
// Just a vanilla exception
}
}

78
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java Normal file
View file

@ -0,0 +1,78 @@
package edu.cornell.mannlib.vitro.webapp.search.indexing;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
class IndexWorkerThread implements Runnable{
private IndexerIface indexer;
private static Log log = LogFactory.getLog(IndexWorkerThread.class);
private Queue<Individual> indQueue = new LinkedList<Individual>();
public IndexWorkerThread(IndexerIface indexer){
this.indexer = indexer;
}
public void addToQueue(Individual ind, boolean newDocs){
}
public void shutdown() {
}
public void run(){
//check for work
//if work found,
// translate
// send to server
//sleep (1000)
}
/*protected void indexInd() throws AbortIndexing{
long starttime = System.currentTimeMillis();
long count = 0;
Iterator<Individual> individuals = firstList.iterator();
while(individuals.hasNext()){
if( stopRequested )
throw new AbortIndexing();
Individual ind = null;
try{
ind = individuals.next();
indexer.index(ind, newDocs);
}catch(Throwable ex){
if( stopRequested || log == null){//log might be null if system is shutting down.
throw new AbortIndexing();
}
String uri = ind!=null?ind.getURI():"null";
log.warn("Error indexing individual from separate thread" + uri + " " + ex.getMessage());
}
count++;
if( log.isDebugEnabled() ){
if( (count % 100 ) == 0 && count > 0 ){
long dt = (System.currentTimeMillis() - starttime);
log.debug("individuals indexed from seperate thread: " + count + " in " + dt + " msec " +
" time pre individual from seperate thread = " + (dt / count) + " msec" );
}
}
}
}
private class AbortIndexing extends Exception {
// Just a vanilla exception
} */
}

91
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/CalculateParameters.java
View file

@ -14,6 +14,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
@ -37,8 +38,8 @@ import edu.cornell.mannlib.vitro.webapp.search.VitroTermNames;
public class CalculateParameters implements DocumentModifier {
Model fullModel;
int totalInd;
private Dataset dataset;
public static int totalInd=1;
public static Map<String,Float> betaMap = new Hashtable<String,Float>();
private float phi;
private static final String prefix = "prefix owl: <http://www.w3.org/2002/07/owl#> "
@ -50,6 +51,17 @@ public class CalculateParameters implements DocumentModifier {
+ " prefix localNav: <http://vitro.mannlib.cornell.edu/ns/localnav#> "
+ " prefix bibo: <http://purl.org/ontology/bibo/> ";
private static final String betaQuery = prefix + " SELECT count(distinct ?inLinks) " +
" WHERE { " +
" ?uri rdf:type owl:Thing . " +
" ?inLinks ?prop ?uri . " +
" } ";
private static final String totalCountQuery = prefix + " SELECT count(distinct ?ind) " +
" WHERE { " +
" ?ind rdf:type owl:Thing . " +
" } ";
private static Log log = LogFactory.getLog(CalculateParameters.class);
private static final String[] fieldsToAddBetaTo = {
@ -64,16 +76,36 @@ public class CalculateParameters implements DocumentModifier {
VitroTermNames.ALLTEXTUNSTEMMED,
};
public CalculateParameters(OntModel fullModel){
this.fullModel=fullModel;
this.totalInd = fullModel.listIndividuals().toList().size();
public CalculateParameters(Dataset dataset){
this.dataset =dataset;
new Thread(new TotalInd(this.dataset,totalCountQuery)).start();
}
public float calculateBeta(String uri){
float beta=0;
RDFNode node = (Resource) fullModel.getResource(uri);
StmtIterator stmtItr = fullModel.listStatements((Resource)null, (Property)null,node);
int Conn = stmtItr.toList().size();
int Conn=0;
Query query;
QuerySolutionMap initialBinding = new QuerySolutionMap();
QuerySolution soln = null;
Resource uriResource = ResourceFactory.createResource(uri);
initialBinding.add("uri", uriResource);
dataset.getLock().enterCriticalSection(Lock.READ);
try{
query = QueryFactory.create(betaQuery,Syntax.syntaxARQ);
QueryExecution qexec = QueryExecutionFactory.create(query,dataset,initialBinding);
ResultSet results = qexec.execSelect();
List<String> resultVars = results.getResultVars();
if(resultVars!=null && resultVars.size()!=0){
soln = results.next();
Conn = Integer.parseInt(soln.getLiteral(resultVars.get(0)).getLexicalForm());
}
}catch(Throwable t){
log.error(t,t);
}finally{
dataset.getLock().leaveCriticalSection();
}
beta = (float)Conn/totalInd;
beta *= 100;
beta += 1;
@ -185,7 +217,7 @@ public class CalculateParameters implements DocumentModifier {
Iterator<String> queryItr = queryList.iterator();
fullModel.enterCriticalSection(Lock.READ);
dataset.getLock().enterCriticalSection(Lock.READ);
Resource adjacentIndividual = null;
RDFNode coauthor = null;
try{
@ -194,7 +226,7 @@ public class CalculateParameters implements DocumentModifier {
queryItr.next(); // we don't want first query to execute if the ind is not a person.
}
query = QueryFactory.create(queryItr.next(),Syntax.syntaxARQ);
QueryExecution qexec = QueryExecutionFactory.create(query,fullModel,initialBinding);
QueryExecution qexec = QueryExecutionFactory.create(query,dataset,initialBinding);
try{
ResultSet results = qexec.execSelect();
while(results.hasNext()){
@ -235,7 +267,7 @@ public class CalculateParameters implements DocumentModifier {
catch(Throwable t){
log.error(t,t);
}finally{
fullModel.leaveCriticalSection();
dataset.getLock().leaveCriticalSection();
adjacentNodes = null;
adjacentNodesConcat = null;
coauthorBuff = null;
@ -283,3 +315,40 @@ public class CalculateParameters implements DocumentModifier {
}
}
class TotalInd implements Runnable{
private Dataset dataset;
private String totalCountQuery;
private static Log log = LogFactory.getLog(TotalInd.class);
public TotalInd(Dataset dataset,String totalCountQuery){
this.dataset = dataset;
this.totalCountQuery = totalCountQuery;
}
public void run(){
int totalInd=0;
Query query;
QuerySolution soln = null;
dataset.getLock().enterCriticalSection(Lock.READ);
try{
query = QueryFactory.create(totalCountQuery,Syntax.syntaxARQ);
QueryExecution qexec = QueryExecutionFactory.create(query,dataset);
ResultSet results = qexec.execSelect();
List<String> resultVars = results.getResultVars();
if(resultVars!=null && resultVars.size()!=0){
soln = results.next();
totalInd = Integer.parseInt(soln.getLiteral(resultVars.get(0)).getLexicalForm());
}
CalculateParameters.totalInd = totalInd;
log.info("Total number of individuals in the system are : " + CalculateParameters.totalInd);
}catch(Throwable t){
log.error(t,t);
}finally{
dataset.getLock().leaveCriticalSection();
}
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java
View file

@ -125,9 +125,11 @@ public class SolrIndexer implements IndexerIface {
} catch(IOException e){
log.error("Could not commit to solr server", e);
}finally{
if(CalculateParameters.betaMap!=null){
CalculateParameters.betaMap.clear();
CalculateParameters.betaMap = null;
}
}
try {
server.optimize();

1
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java
View file

@ -76,6 +76,7 @@ public class SolrSetup implements javax.servlet.ServletContextListener{
List<DocumentModifier> modifiers = new ArrayList<DocumentModifier>();
// modifiers.add(new CalculateParameters(ModelContext.getJenaOntModel(context)));
modifiers.add(new CalculateParameters(dataset));
modifiers.add(new ContextNodeFields(dataset));
IndividualToSolrDocument indToSolrDoc = new IndividualToSolrDocument(