litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[VIVO-1031] Use streaming result set for co- visualisations

Browse source
This commit is contained in:
grahamtriggs 2015-10-19 15:51:57 +01:00
parent 208f103629
commit 8fc40fcf98
2 changed files with 457 additions and 485 deletions
Download patch file Download diff file Expand all files Collapse all files

164
src/edu/cornell/mannlib/vitro/webapp/visualization/coauthorship/CoAuthorshipQueryRunner.java
View file

@ -16,6 +16,8 @@ import java.util.concurrent.ConcurrentHashMap;
import com.hp.hpl.jena.query.ResultSetFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import net.sf.jga.algorithms.Unique;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.jena.iri.IRI;
@ -23,7 +25,6 @@ import org.apache.jena.iri.IRIFactory;
import org.apache.jena.iri.Violation;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.RDFNode;
@ -58,10 +59,6 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
private Log log;
private UniqueIDGenerator nodeIDGenerator;
private UniqueIDGenerator edgeIDGenerator;
public CoAuthorshipQueryRunner(String egoURI,
RDFService rdfService, Log log) {
@ -69,18 +66,13 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
this.rdfService = rdfService;
this.log = log;
this.nodeIDGenerator = new UniqueIDGenerator();
this.edgeIDGenerator = new UniqueIDGenerator();
}
private CollaborationData createQueryResult(ResultSet resultSet) {
private static class QueryResultConsumer extends ResultSetConsumer {
Set<Collaborator> nodes = new HashSet<Collaborator>();
Map<String, Activity> biboDocumentURLToVO = new HashMap<String, Activity>();
Map<String, Set<Collaborator>> biboDocumentURLToCoAuthors =
new HashMap<String, Set<Collaborator>>();
Map<String, Set<Collaborator>> biboDocumentURLToCoAuthors = new HashMap<String, Set<Collaborator>>();
Map<String, Collaborator> nodeURLToVO = new HashMap<String, Collaborator>();
Map<String, Collaboration> edgeUniqueIdentifierToVO = new HashMap<String, Collaboration>();
@ -88,13 +80,15 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
Set<Collaboration> edges = new HashSet<Collaboration>();
while (resultSet.hasNext()) {
QuerySolution solution = resultSet.nextSolution();
private UniqueIDGenerator nodeIDGenerator = new UniqueIDGenerator();
private UniqueIDGenerator edgeIDGenerator = new UniqueIDGenerator();
@Override
protected void processQuerySolution(QuerySolution qs) {
/*
* We only want to create only ONE ego node.
* */
RDFNode egoAuthorURLNode = solution.get(QueryFieldLabels.AUTHOR_URL);
RDFNode egoAuthorURLNode = qs.get(QueryFieldLabels.AUTHOR_URL);
if (nodeURLToVO.containsKey(egoAuthorURLNode.toString())) {
egoNode = nodeURLToVO.get(egoAuthorURLNode.toString());
@ -105,19 +99,19 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
nodes.add(egoNode);
nodeURLToVO.put(egoAuthorURLNode.toString(), egoNode);
RDFNode authorLabelNode = solution.get(QueryFieldLabels.AUTHOR_LABEL);
RDFNode authorLabelNode = qs.get(QueryFieldLabels.AUTHOR_LABEL);
if (authorLabelNode != null) {
egoNode.setCollaboratorName(authorLabelNode.toString());
}
}
RDFNode documentNode = solution.get(QueryFieldLabels.DOCUMENT_URL);
RDFNode documentNode = qs.get(QueryFieldLabels.DOCUMENT_URL);
Activity biboDocument;
if (biboDocumentURLToVO.containsKey(documentNode.toString())) {
biboDocument = biboDocumentURLToVO.get(documentNode.toString());
} else {
biboDocument = createDocumentVO(solution, documentNode.toString());
biboDocument = createDocumentVO(qs, documentNode.toString());
biboDocumentURLToVO.put(documentNode.toString(), biboDocument);
}
@ -128,14 +122,14 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
* we do not want a co-author node or Collaboration if the publication has only one
* author and that happens to be the ego.
* */
if (solution.get(QueryFieldLabels.AUTHOR_URL).toString().equalsIgnoreCase(
solution.get(QueryFieldLabels.CO_AUTHOR_URL).toString())) {
continue;
if (qs.get(QueryFieldLabels.AUTHOR_URL).toString().equalsIgnoreCase(
qs.get(QueryFieldLabels.CO_AUTHOR_URL).toString())) {
return;
}
Collaborator coAuthorNode;
RDFNode coAuthorURLNode = solution.get(QueryFieldLabels.CO_AUTHOR_URL);
RDFNode coAuthorURLNode = qs.get(QueryFieldLabels.CO_AUTHOR_URL);
if (nodeURLToVO.containsKey(coAuthorURLNode.toString())) {
coAuthorNode = nodeURLToVO.get(coAuthorURLNode.toString());
@ -146,7 +140,7 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
nodes.add(coAuthorNode);
nodeURLToVO.put(coAuthorURLNode.toString(), coAuthorNode);
RDFNode coAuthorLabelNode = solution.get(QueryFieldLabels.CO_AUTHOR_LABEL);
RDFNode coAuthorLabelNode = qs.get(QueryFieldLabels.CO_AUTHOR_LABEL);
if (coAuthorLabelNode != null) {
coAuthorNode.setCollaboratorName(coAuthorLabelNode.toString());
}
@ -186,12 +180,10 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
coAuthorNode.getCollaboratorID()),
egoCoAuthorEdge);
}
}
@Override
protected void endProcessing() {
/*
* This method takes out all the authors & edges between authors that belong to documents
* that have more than 100 authors. We conjecture that these papers do not provide much
@ -222,51 +214,33 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
biboDocumentURLToCoAuthors,
edges,
edgeUniqueIdentifierToVO);
}
public CollaborationData getCollaborationData() {
return new CoAuthorshipData(egoNode, nodes, edges);
}
private void removeLowQualityNodesAndEdges(
Set<Collaborator> nodes,
Map<String, Activity> biboDocumentURLToVO,
Map<String, Set<Collaborator>> biboDocumentURLToCoAuthors,
Set<Collaboration> edges) {
private Activity createDocumentVO(QuerySolution solution, String documentURL) {
Set<Collaborator> nodesToBeRemoved = new HashSet<Collaborator>();
for (Map.Entry<String, Set<Collaborator>> currentBiboDocumentEntry
: biboDocumentURLToCoAuthors.entrySet()) {
Activity biboDocument = new Activity(documentURL);
if (currentBiboDocumentEntry.getValue().size() > MAX_AUTHORS_PER_PAPER_ALLOWED) {
Activity currentBiboDocument = biboDocumentURLToVO
.get(currentBiboDocumentEntry.getKey());
Set<Collaboration> edgesToBeRemoved = new HashSet<Collaboration>();
for (Collaboration currentEdge : edges) {
Set<Activity> currentCollaboratorDocuments =
currentEdge.getCollaborationActivities();
if (currentCollaboratorDocuments.contains(currentBiboDocument)) {
currentCollaboratorDocuments.remove(currentBiboDocument);
if (currentCollaboratorDocuments.isEmpty()) {
edgesToBeRemoved.add(currentEdge);
}
}
RDFNode publicationDateNode = solution.get(QueryFieldLabels.DOCUMENT_PUBLICATION_DATE);
if (publicationDateNode != null) {
biboDocument.setActivityDate(publicationDateNode.toString());
}
edges.removeAll(edgesToBeRemoved);
return biboDocument;
}
for (Collaborator currentCoAuthor : currentBiboDocumentEntry.getValue()) {
currentCoAuthor.getCollaboratorActivities().remove(currentBiboDocument);
if (currentCoAuthor.getCollaboratorActivities().isEmpty()) {
nodesToBeRemoved.add(currentCoAuthor);
private String getEdgeUniqueIdentifier(int nodeID1, int nodeID2) {
String separator = "*";
if (nodeID1 < nodeID2) {
return nodeID1 + separator + nodeID2;
} else {
return nodeID2 + separator + nodeID1;
}
}
}
}
nodes.removeAll(nodesToBeRemoved);
}
private void createCoAuthorEdges(
@ -334,7 +308,6 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
}
edges.addAll(newlyAddedEdges);
}
}
}
@ -351,28 +324,49 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
}
private String getEdgeUniqueIdentifier(int nodeID1, int nodeID2) {
private void removeLowQualityNodesAndEdges(
Set<Collaborator> nodes,
Map<String, Activity> biboDocumentURLToVO,
Map<String, Set<Collaborator>> biboDocumentURLToCoAuthors,
Set<Collaboration> edges) {
String separator = "*";
Set<Collaborator> nodesToBeRemoved = new HashSet<Collaborator>();
for (Map.Entry<String, Set<Collaborator>> currentBiboDocumentEntry
: biboDocumentURLToCoAuthors.entrySet()) {
if (nodeID1 < nodeID2) {
return nodeID1 + separator + nodeID2;
} else {
return nodeID2 + separator + nodeID1;
if (currentBiboDocumentEntry.getValue().size() > MAX_AUTHORS_PER_PAPER_ALLOWED) {
Activity currentBiboDocument = biboDocumentURLToVO
.get(currentBiboDocumentEntry.getKey());
Set<Collaboration> edgesToBeRemoved = new HashSet<Collaboration>();
for (Collaboration currentEdge : edges) {
Set<Activity> currentCollaboratorDocuments =
currentEdge.getCollaborationActivities();
if (currentCollaboratorDocuments.contains(currentBiboDocument)) {
currentCollaboratorDocuments.remove(currentBiboDocument);
if (currentCollaboratorDocuments.isEmpty()) {
edgesToBeRemoved.add(currentEdge);
}
}
}
edges.removeAll(edgesToBeRemoved);
for (Collaborator currentCoAuthor : currentBiboDocumentEntry.getValue()) {
currentCoAuthor.getCollaboratorActivities().remove(currentBiboDocument);
if (currentCoAuthor.getCollaboratorActivities().isEmpty()) {
nodesToBeRemoved.add(currentCoAuthor);
}
}
}
}
nodes.removeAll(nodesToBeRemoved);
}
private Activity createDocumentVO(QuerySolution solution, String documentURL) {
Activity biboDocument = new Activity(documentURL);
RDFNode publicationDateNode = solution.get(QueryFieldLabels.DOCUMENT_PUBLICATION_DATE);
if (publicationDateNode != null) {
biboDocument.setActivityDate(publicationDateNode.toString());
}
return biboDocument;
/* END QUERY RUNNER */
}
private String generateEgoCoAuthorshipSparqlQuery(String queryURI) {
@ -455,19 +449,13 @@ public class CoAuthorshipQueryRunner implements QueryRunner<CollaborationData> {
throw new MalformedQueryParametersException("URI parameter is either null or empty.");
}
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(generateEgoCoAuthorshipSparqlQuery(this.egoURI), RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
data = createQueryResult(rs);
QueryResultConsumer consumer = new QueryResultConsumer();
rdfService.sparqlSelectQuery(generateEgoCoAuthorshipSparqlQuery(this.egoURI), consumer);
data = consumer.getCollaborationData();
} catch (RDFServiceException e) {
log.error("Unable to execute query", e);
throw new RuntimeException(e);
} finally {
if (is != null) {
try { is.close(); } catch (Throwable t) { }
}
}
CollaborationDataCacheEntry newEntry = new CollaborationDataCacheEntry();

120
src/edu/cornell/mannlib/vitro/webapp/visualization/coprincipalinvestigator/CoPIGrantCountQueryRunner.java
View file

@ -2,7 +2,6 @@
package edu.cornell.mannlib.vitro.webapp.visualization.coprincipalinvestigator;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@ -13,9 +12,9 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import com.hp.hpl.jena.query.ResultSetFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -23,14 +22,8 @@ import org.apache.jena.iri.IRI;
import org.apache.jena.iri.IRIFactory;
import org.apache.jena.iri.Violation;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.RDFNode;
import edu.cornell.mannlib.vitro.webapp.visualization.collaborationutils.CoInvestigationData;
@ -60,10 +53,6 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
private Log log = LogFactory.getLog(CoPIGrantCountQueryRunner.class.getName());
private UniqueIDGenerator nodeIDGenerator;
private UniqueIDGenerator edgeIDGenerator;
private long before, after;
private static final String SPARQL_QUERY_COMMON_OPTIONAL_BLOCK_FOR_ROLE_DATE_TIME = ""
@ -95,9 +84,6 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
this.egoURI = egoURI;
this.rdfService = rdfService;
// this.log = log;
this.nodeIDGenerator = new UniqueIDGenerator();
this.edgeIDGenerator = new UniqueIDGenerator();
}
private String generateEgoCoPIquery(String queryURI) {
@ -294,19 +280,13 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
before = System.currentTimeMillis();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(generateEgoCoPIquery(this.egoURI), RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
data = createQueryResult(rs);
QueryResultConsumer consumer = new QueryResultConsumer();
rdfService.sparqlSelectQuery(generateEgoCoPIquery(this.egoURI), consumer);
data = consumer.getData();
} catch (RDFServiceException e) {
log.error("Unable to execute query", e);
throw new RuntimeException(e);
} finally {
if (is != null) {
try { is.close(); } catch (Throwable t) { }
}
}
after = System.currentTimeMillis();
@ -356,34 +336,7 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
}
}
private Collaboration getExistingEdge(
Collaborator collaboratingNode1,
Collaborator collaboratingNode2,
Map<String, Collaboration> edgeUniqueIdentifierToVO) {
String edgeUniqueIdentifier = getEdgeUniqueIdentifier(
collaboratingNode1.getCollaboratorID(),
collaboratingNode2.getCollaboratorID());
return edgeUniqueIdentifierToVO.get(edgeUniqueIdentifier);
}
private String getEdgeUniqueIdentifier(int nodeID1, int nodeID2) {
String separator = "*";
if (nodeID1 < nodeID2) {
return nodeID1 + separator + nodeID2;
} else {
return nodeID2 + separator + nodeID1;
}
}
private CollaborationData createQueryResult(ResultSet resultSet) {
private static class QueryResultConsumer extends ResultSetConsumer {
Set<Collaborator> nodes = new HashSet<Collaborator>();
Map<String, Activity> grantURLToVO = new HashMap<String, Activity>();
@ -395,15 +348,17 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
Set<Collaboration> edges = new HashSet<Collaboration>();
before = System.currentTimeMillis();
private UniqueIDGenerator nodeIDGenerator = new UniqueIDGenerator();
private UniqueIDGenerator edgeIDGenerator = new UniqueIDGenerator();
while (resultSet.hasNext()) {
QuerySolution solution = resultSet.nextSolution();
private Log log = LogFactory.getLog(CoPIGrantCountQueryRunner.class.getName());
@Override
protected void processQuerySolution(QuerySolution qs) {
/*
* We only want to create only ONE ego node.
* */
RDFNode egoPIURLNode = solution.get(QueryFieldLabels.PI_URL);
RDFNode egoPIURLNode = qs.get(QueryFieldLabels.PI_URL);
if (nodeURLToVO.containsKey(egoPIURLNode.toString())) {
egoNode = nodeURLToVO.get(egoPIURLNode.toString());
@ -415,20 +370,20 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
nodeURLToVO.put(egoPIURLNode.toString(), egoNode);
RDFNode authorLabelNode = solution.get(QueryFieldLabels.PI_LABEL);
RDFNode authorLabelNode = qs.get(QueryFieldLabels.PI_LABEL);
if (authorLabelNode != null) {
egoNode.setCollaboratorName(authorLabelNode.toString());
}
}
log.debug("PI: " + egoNode.getIndividualLabel());
RDFNode grantNode = solution.get(QueryFieldLabels.GRANT_URL);
RDFNode grantNode = qs.get(QueryFieldLabels.GRANT_URL);
Activity grant;
if (grantURLToVO.containsKey(grantNode.toString())) {
grant = grantURLToVO.get(grantNode.toString());
} else {
grant = createGrantVO(solution, grantNode.toString());
grant = createGrantVO(qs, grantNode.toString());
grantURLToVO.put(grantNode.toString(), grant);
}
@ -440,14 +395,14 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
* we do not want a co-pi node or edge if the grant has only one
* pi and that happens to be the ego.
* */
if (solution.get(QueryFieldLabels.PI_URL).toString().equalsIgnoreCase(
solution.get(QueryFieldLabels.CO_PI_URL).toString())) {
continue;
if (qs.get(QueryFieldLabels.PI_URL).toString().equalsIgnoreCase(
qs.get(QueryFieldLabels.CO_PI_URL).toString())) {
return;
}
Collaborator coPINode;
RDFNode coPIURLNode = solution.get(QueryFieldLabels.CO_PI_URL);
RDFNode coPIURLNode = qs.get(QueryFieldLabels.CO_PI_URL);
if (nodeURLToVO.containsKey(coPIURLNode.toString())) {
coPINode = nodeURLToVO.get(coPIURLNode.toString());
@ -458,7 +413,7 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
nodes.add(coPINode);
nodeURLToVO.put(coPIURLNode.toString(), coPINode);
RDFNode coPILabelNode = solution.get(QueryFieldLabels.CO_PI_LABEL);
RDFNode coPILabelNode = qs.get(QueryFieldLabels.CO_PI_LABEL);
if (coPILabelNode != null) {
coPINode.setCollaboratorName(coPILabelNode.toString());
}
@ -497,9 +452,12 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
coPINode.getCollaboratorID()),
egoCoPIEdge);
}
}
@Override
protected void endProcessing() {
super.endProcessing();
/*
* This method takes out all the PIs & edges between PIs that belong to grants
* that have more than 100 PIs. We conjecture that these grants do not provide much
@ -528,11 +486,10 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
grantURLToCoPIs,
edges,
edgeUniqueIdentifierToVO);
}
after = System.currentTimeMillis();
log.debug("Time taken to iterate through the ResultSet of SELECT queries is in ms: "
+ (after - before));
public CollaborationData getData() {
return new CoInvestigationData(egoNode, nodes, edges);
}
@ -674,4 +631,31 @@ public class CoPIGrantCountQueryRunner implements QueryRunner<CollaborationData>
return grant;
}
private Collaboration getExistingEdge(
Collaborator collaboratingNode1,
Collaborator collaboratingNode2,
Map<String, Collaboration> edgeUniqueIdentifierToVO) {
String edgeUniqueIdentifier = getEdgeUniqueIdentifier(
collaboratingNode1.getCollaboratorID(),
collaboratingNode2.getCollaboratorID());
return edgeUniqueIdentifierToVO.get(edgeUniqueIdentifier);
}
private String getEdgeUniqueIdentifier(int nodeID1, int nodeID2) {
String separator = "*";
if (nodeID1 < nodeID2) {
return nodeID1 + separator + nodeID2;
} else {
return nodeID2 + separator + nodeID1;
}
}
/** END QUERY RESULT CONSUMER **/
}
}