litvinovg/vivo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[VIVO-1031] Improve performance of Temporal graphs (along lines of Map of Science). Improve SPARQL used in Map of Science to halve execution time.

Browse source
This commit is contained in:
grahamtriggs 2015-10-17 21:06:54 +01:00
parent 2519914833
commit 34ead82676
8 changed files with 1018 additions and 737 deletions
Download patch file Download diff file Expand all files Collapse all files

361
src/edu/cornell/mannlib/vitro/webapp/visualization/mapofscience/MapOfScienceVisualizationRequestHandler.java
View file

@ -9,9 +9,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
@ -20,6 +17,9 @@ import com.hp.hpl.jena.rdf.model.Resource;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.visualization.constants.QueryConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.CachingRDFServiceExecutor;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.OrgUtils;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.VisualizationCaches;
import mapping.ScienceMapping;
import mapping.ScienceMappingResult;
@ -40,9 +40,7 @@ import edu.cornell.mannlib.vitro.webapp.visualization.constants.MapOfScienceCons
import edu.cornell.mannlib.vitro.webapp.visualization.constants.VisConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.exceptions.MalformedQueryParametersException;
import edu.cornell.mannlib.vitro.webapp.visualization.temporalgraph.OrganizationUtilityFunctions;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.Activity;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.GenericQueryMap;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.MapOfScienceActivity;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.json.MapOfScience;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.UtilityFunctions;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.VisualizationRequestHandler;
@ -93,14 +91,13 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
return prepareStandaloneMarkupResponse(vitroRequest, entityURI);
}
private Map<String, String> getSubjectPersonEntityAndGenerateDataResponse(
VitroRequest vitroRequest, String subjectEntityURI, String entityLabel, VisConstants.DataVisMode dataOuputFormat)
throws MalformedQueryParametersException {
RDFService rdfService = vitroRequest.getRDFService();
Map<String, Set<String>> personToPublicationMap = cachedPersonToPublication.get(rdfService);
Map<String, Set<String>> personToPublicationMap = VisualizationCaches.cachedPersonToPublication.get(rdfService);
Map<String, String> publicationToJournalMap = cachedPublicationToJournal.get(rdfService);
if (!personToPublicationMap.containsKey(subjectEntityURI)) {
@ -163,25 +160,13 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
}
}
private Set<String> addOrgAndAllSubOrgs(Set<String> allSubOrgs, String org, Map<String, Set<String>> subOrgMap) {
if (allSubOrgs.add(org)) {
if (subOrgMap.containsKey(org)) {
for (String subOrg : subOrgMap.get(org)) {
addOrgAndAllSubOrgs(allSubOrgs, subOrg, subOrgMap);
}
}
}
return allSubOrgs;
}
private Map<String, String> getSubjectEntityAndGenerateDataResponse(
VitroRequest vitroRequest, String subjectEntityURI, String entityLabel, VisConstants.DataVisMode dataOuputFormat)
throws MalformedQueryParametersException {
RDFService rdfService = vitroRequest.getRDFService();
Map<String, String> orgLabelMap = cachedOrganizationLabels.get(rdfService);
Map<String, String> orgLabelMap = VisualizationCaches.cachedOrganizationLabels.get(rdfService);
if (orgLabelMap.get(subjectEntityURI) == null) {
if (VisConstants.DataVisMode.JSON.equals(dataOuputFormat)) {
@ -191,9 +176,9 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
}
}
Map<String, Set<String>> subOrgMap = cachedOrganizationSubOrgs.get(rdfService);
Map<String, Set<String>> organisationToPeopleMap = cachedOrganisationToPeopleMap.get(rdfService);
Map<String, Set<String>> personToPublicationMap = cachedPersonToPublication.get(rdfService);
Map<String, Set<String>> subOrgMap = VisualizationCaches.cachedOrganizationSubOrgs.get(rdfService);
Map<String, Set<String>> organisationToPeopleMap = VisualizationCaches.cachedOrganisationToPeopleMap.get(rdfService);
Map<String, Set<String>> personToPublicationMap = VisualizationCaches.cachedPersonToPublication.get(rdfService);
Map<String, String> publicationToJournalMap = cachedPublicationToJournal.get(rdfService);
Set<String> orgPublications = new HashSet<String>();
@ -201,44 +186,15 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
Map<String, Set<String>> subOrgPublicationsMap = new HashMap<String, Set<String>>();
if (subOrgMap.containsKey(subjectEntityURI)) {
for (String topSubOrg : subOrgMap.get(subjectEntityURI)) {
Set<String> subOrgPublications = new HashSet<String>();
Set<String> subOrgPublicationsPeople = new HashSet<String>();
Set<String> fullSubOrgs = addOrgAndAllSubOrgs(new HashSet<String>(), topSubOrg, subOrgMap);
for (String subOrg : fullSubOrgs) {
Set<String> peopleInSubOrg = organisationToPeopleMap.get(subOrg);
if (peopleInSubOrg != null) {
for (String person : peopleInSubOrg) {
if (personToPublicationMap.containsKey(person)) {
if (subOrgPublicationsPeople.add(person)) {
subOrgPublications.addAll(personToPublicationMap.get(person));
if (orgPublicationsPeople.add(person)) {
orgPublications.addAll(personToPublicationMap.get(person));
}
}
}
}
}
}
subOrgPublicationsMap.put(topSubOrg, subOrgPublications);
}
}
Set<String> people = organisationToPeopleMap.get(subjectEntityURI);
if (people != null) {
for (String person : people) {
if (personToPublicationMap.containsKey(person)) {
if (orgPublicationsPeople.add(person)) {
orgPublications.addAll(personToPublicationMap.get(person));
}
}
}
}
OrgUtils.getObjectMappingsForOrgAnSubOrgs(
subjectEntityURI,
orgPublications,
orgPublicationsPeople,
subOrgPublicationsMap,
subOrgMap,
organisationToPeopleMap,
personToPublicationMap
);
if (orgPublications.isEmpty()) {
if (VisConstants.DataVisMode.JSON.equals(dataOuputFormat)) {
@ -584,191 +540,11 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
return null;
}
private static CachingRDFServiceExecutor<Map<String, String>> cachedOrganizationLabels =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?org ?orgLabel\n" +
"WHERE\n" +
"{\n" +
" ?org a foaf:Organization .\n" +
" ?org rdfs:label ?orgLabel .\n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("org").getURI();
String orgLabel = qs.getLiteral("orgLabel").getString();
map.put(org, orgLabel);
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
private static CachingRDFServiceExecutor<Map<String, Set<String>>> cachedOrganizationSubOrgs =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?org ?subOrg\n" +
"WHERE\n" +
"{\n" +
" ?org a foaf:Organization .\n" +
" ?org <http://purl.obolibrary.org/obo/BFO_0000051> ?subOrg .\n" +
"}\n";
Map<String, Set<String>> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("org").getURI();
String subOrg = qs.getResource("subOrg").getURI();
Set<String> subOrgs = map.get(org);
if (subOrgs == null) {
subOrgs = new HashSet<String>();
subOrgs.add(subOrg);
map.put(org, subOrgs);
} else {
subOrgs.add(subOrg);
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
private static CachingRDFServiceExecutor<Map<String, Set<String>>> cachedOrganisationToPeopleMap =
new CachingRDFServiceExecutor<Map<String, Set<String>>>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
public Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?organisation ?person\n" +
"WHERE\n" +
"{\n" +
" ?organisation a foaf:Organization .\n" +
" ?organisation core:relatedBy ?position .\n" +
" ?position core:relates ?person .\n" +
" ?person a foaf:Person .\n" +
"}\n";
// TODO Critical section?
Map<String, Set<String>> orgToPeopleMap = new HashMap<String, Set<String>>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("organisation").getURI();
String person = qs.getResource("person").getURI();
Set<String> people = orgToPeopleMap.get(org);
if (people == null) {
people = new HashSet<String>();
people.add(person);
orgToPeopleMap.put(org, people);
} else {
people.add(person);
}
}
} finally {
silentlyClose(is);
}
return orgToPeopleMap;
}
}
);
private static CachingRDFServiceExecutor<Map<String, Set<String>>> cachedPersonToPublication =
new CachingRDFServiceExecutor<Map<String, Set<String>>>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
public Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?person ?document\n" +
"WHERE\n" +
"{\n" +
" ?person a foaf:Person .\n" +
" ?person core:relatedBy ?authorship .\n" +
" ?authorship core:relates ?document .\n" +
" ?document a bibo:Document .\n" +
"}\n";
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
Resource person = qs.getResource("person");
Resource document = qs.getResource("document");
if (person != null && document != null) {
String personURI = person.getURI();
Set<String> documents = map.get(personURI);
if (documents == null) {
documents = new HashSet<String>();
documents.add(document.getURI());
map.put(personURI, documents);
} else {
documents.add(document.getURI());
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
private static CachingRDFServiceExecutor<Map<String, String>> cachedPublicationToJournal =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
Map<String, String> callWithService(RDFService rdfService) throws Exception {
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?document ?journalLabel\n" +
"WHERE\n" +
@ -830,107 +606,6 @@ public class MapOfScienceVisualizationRequestHandler implements VisualizationReq
return total;
}
}
private static class CachingRDFServiceExecutor<T> {
private T cachedResults;
private long lastCacheTime;
private RDFServiceCallable<T> resultBuilder;
private FutureTask<T> backgroundTask = null;
CachingRDFServiceExecutor(RDFServiceCallable<T> resultBuilder) {
this.resultBuilder = resultBuilder;
}
synchronized T get(RDFService rdfService) {
if (cachedResults != null) {
if (!resultBuilder.invalidateCache(System.currentTimeMillis() - lastCacheTime)) {
return cachedResults;
}
}
try {
if (backgroundTask == null) {
resultBuilder.setRDFService(rdfService);
backgroundTask = new FutureTask<T>(resultBuilder);
Thread thread = new Thread(backgroundTask);
thread.setDaemon(true);
thread.start();
if (cachedResults == null || resultBuilder.executionTime < 2000) {
completeBackgroundTask();
}
} else if (backgroundTask.isDone()) {
completeBackgroundTask();
}
} catch (InterruptedException e) {
abortBackgroundTask();
} catch (ExecutionException e) {
abortBackgroundTask();
throw new RuntimeException("Background RDF thread through an exception", e.getCause());
}
return cachedResults;
}
private void abortBackgroundTask() {
if (backgroundTask != null) {
backgroundTask.cancel(true);
backgroundTask = null;
}
}
private void completeBackgroundTask() throws InterruptedException, ExecutionException {
if (backgroundTask != null) {
cachedResults = backgroundTask.get();
lastCacheTime = System.currentTimeMillis();
backgroundTask = null;
}
}
static abstract class RDFServiceCallable<T> implements Callable<T> {
private RDFService rdfService;
private long executionTime = -1;
final void setRDFService(RDFService rdfService) {
this.rdfService = rdfService;
}
@Override
final public T call() throws Exception {
long start = System.currentTimeMillis();
T val = callWithService(rdfService);
executionTime = System.currentTimeMillis() - start;
return val;
}
abstract T callWithService(RDFService rdfService) throws Exception;
boolean invalidateCache(long timeCached) {
if (executionTime > -1) {
/*
Determine validity as a function of the time it takes to execute the query.
Query exec time | Keep cache for
-----------------+-----------------
10 seconds | 20 minutes
30 seconds | 1 hour
1 minute | 2 hours
5 minutes | 10 hours
Multiplier of the last execution time is 120.
At most, keep a cache for one day (24 * 60 * 60 * 1000 = 86400000)
*/
return timeCached > Math.min(executionTime * 120, 86400000);
}
return false;
}
}
}
private static void silentlyClose(InputStream is) {
try {
if (is != null) {

308
src/edu/cornell/mannlib/vitro/webapp/visualization/temporalgraph/TemporalGrantVisualizationRequestHandler.java
View file

@ -3,12 +3,18 @@
package edu.cornell.mannlib.vitro.webapp.visualization.temporalgraph;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.CounterUtils;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.OrgUtils;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.VisualizationCaches;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.Individual;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@ -43,21 +49,17 @@ public class TemporalGrantVisualizationRequestHandler implements
VitroRequest vitroRequest, Log log, Dataset dataset)
throws MalformedQueryParametersException {
String entityURI = vitroRequest
.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
return generateStandardVisualizationForGrantTemporalVis(vitroRequest,
log, dataset, entityURI);
String entityURI = vitroRequest.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
return generateStandardVisualizationForGrantTemporalVis(vitroRequest, log, dataset, entityURI);
}
private ResponseValues generateStandardVisualizationForGrantTemporalVis(
VitroRequest vitroRequest, Log log, Dataset dataset,
String entityURI) throws MalformedQueryParametersException {
if (StringUtils.isBlank(entityURI)) {
entityURI = OrganizationUtilityFunctions
.getStaffProvidedOrComputedHighestLevelOrganization(
entityURI = OrganizationUtilityFunctions.getStaffProvidedOrComputedHighestLevelOrganization(
log,
dataset,
vitroRequest);
@ -73,19 +75,16 @@ public class TemporalGrantVisualizationRequestHandler implements
Map<String, String> parameters, VitroRequest vitroRequest, Log log,
Dataset dataset) throws MalformedQueryParametersException {
return generateStandardVisualizationForGrantTemporalVis(
vitroRequest, log, dataset, parameters.get(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY));
}
@Override
public Map<String, String> generateDataVisualization(
VitroRequest vitroRequest, Log log, Dataset dataset)
public Map<String, String> generateDataVisualization(VitroRequest vitroRequest, Log log, Dataset dataset)
throws MalformedQueryParametersException {
String entityURI = vitroRequest
.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
String entityURI = vitroRequest.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
VisConstants.DataVisMode currentDataMode = VisConstants.DataVisMode.CSV;
@ -120,77 +119,146 @@ public class TemporalGrantVisualizationRequestHandler implements
}
private Map<String, String> prepareDataErrorResponse() {
String outputFileName = "no-organization_grants-per-year.csv";
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_NAME_KEY,
outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_NAME_KEY, outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, "");
return fileData;
}
@Override
public Object generateAjaxVisualization(VitroRequest vitroRequest, Log log,
Dataset dataset) throws MalformedQueryParametersException {
public Object generateAjaxVisualization(VitroRequest vitroRequest, Log log, Dataset dataset)
throws MalformedQueryParametersException {
throw new UnsupportedOperationException("Entity Grant Count "
+ "does not provide Ajax response.");
throw new UnsupportedOperationException("Entity Grant Count does not provide Ajax response.");
}
private Map<String, String> getSubjectEntityAndGenerateDataResponse(
VitroRequest vitroRequest, Log log, Dataset dataset,
String subjectEntityURI, VisConstants.DataVisMode visMode)
throws MalformedQueryParametersException {
Entity organizationEntity = SelectOnModelUtilities
.getSubjectOrganizationHierarchy(dataset, subjectEntityURI);
if (organizationEntity.getSubEntities() == null) {
RDFService rdfService = vitroRequest.getRDFService();
Map<String, String> orgLabelMap = VisualizationCaches.cachedOrganizationLabels.get(rdfService);
Map<String, String> personLabelMap = VisualizationCaches.cachedPersonLabels.get(rdfService);
if (orgLabelMap.get(subjectEntityURI) == null) {
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataErrorResponse();
} else {
return prepareDataErrorResponse();
}
}
Map<String, Activity> grantURIForAssociatedPeopleToVO = new HashMap<String, Activity>();
Map<String, Activity> allGrantURIToVO = new HashMap<String, Activity>();
allGrantURIToVO = SelectOnModelUtilities.getGrantsForAllSubOrganizations(dataset, organizationEntity);
Entity organizationWithAssociatedPeople = SelectOnModelUtilities
.getSubjectOrganizationAssociatedPeople(dataset, subjectEntityURI);
if (organizationWithAssociatedPeople.getSubEntities() != null) {
grantURIForAssociatedPeopleToVO = SelectOnModelUtilities
.getGrantsForAssociatedPeople(dataset, organizationWithAssociatedPeople.getSubEntities());
organizationEntity = OrganizationUtilityFunctions.mergeEntityIfShareSameURI(
organizationEntity,
organizationWithAssociatedPeople);
}
if (allGrantURIToVO.isEmpty() && grantURIForAssociatedPeopleToVO.isEmpty()) {
Map<String, Set<String>> subOrgMap = VisualizationCaches.cachedOrganizationSubOrgs.get(rdfService);
Map<String, Set<String>> organisationToPeopleMap = VisualizationCaches.cachedOrganisationToPeopleMap.get(rdfService);
Map<String, String> orgMostSpecificLabelMap = VisualizationCaches.cachedOrganizationToMostSpecificLabel.get(rdfService);
Map<String, String> personMostSpecificLabelMap = VisualizationCaches.cachedPersonToMostSpecificLabel.get(rdfService);
Map<String, Set<String>> personToGrantMap = VisualizationCaches.cachedPersonToGrant.get(rdfService);
Map<String, String> grantToYearMap = VisualizationCaches.cachedGrantToYear.get(rdfService);
Set<String> orgGrants = new HashSet<String>();
Set<String> orgGrantsPeople = new HashSet<String>();
Map<String, Set<String>> subOrgPublicationsMap = new HashMap<String, Set<String>>();
OrgUtils.getObjectMappingsForOrgAnSubOrgs(
subjectEntityURI,
orgGrants,
orgGrantsPeople,
subOrgPublicationsMap,
subOrgMap,
organisationToPeopleMap,
personToGrantMap
);
if (orgGrants.isEmpty()) {
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataErrorResponse();
} else {
return prepareDataErrorResponse();
}
} else {
} else {
Map<String, String> fileData = new HashMap<String, String>();
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataResponse(vitroRequest, organizationEntity);
Gson json = new Gson();
Set subEntitiesJson = new HashSet();
// For each suborganisation
for (String subOrg : subOrgPublicationsMap.keySet()) {
JsonObject entityJson = new JsonObject(orgLabelMap.get(subOrg));
List<List<Integer>> yearPubCounts = CounterUtils.getObjectCountByYear(subOrgPublicationsMap.get(subOrg), grantToYearMap);
String type = orgMostSpecificLabelMap.get(subOrg);
entityJson.setYearToActivityCount(yearPubCounts);
entityJson.setOrganizationTypes(Arrays.asList(type == null ? "Organization" : type));
entityJson.setEntityURI(subOrg);
entityJson.setVisMode("ORGANIZATION");
subEntitiesJson.add(entityJson);
}
// For each person
for (String person : orgGrantsPeople) {
JsonObject entityJson = new JsonObject(personLabelMap.get(person));
List<List<Integer>> yearPubCounts = CounterUtils.getObjectCountByYear(personToGrantMap.get(person), grantToYearMap);
String type = personMostSpecificLabelMap.get(person);
entityJson.setYearToActivityCount(yearPubCounts);
entityJson.setOrganizationTypes(Arrays.asList(type == null ? "Person" : type));
entityJson.setEntityURI(person);
entityJson.setVisMode("PERSON");
subEntitiesJson.add(entityJson);
}
SubjectEntityJSON subjectEntityJSON = new SubjectEntityJSON(
orgLabelMap.get(subjectEntityURI),
subjectEntityURI,
OrgUtils.getParentURIsToLabel(subjectEntityURI, subOrgMap, orgLabelMap));
subEntitiesJson.add(subjectEntityJSON);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, json.toJson(subEntitiesJson));
} else {
return prepareDataResponse(organizationEntity);
String entityLabel = orgLabelMap.get(subjectEntityURI);
if (StringUtils.isBlank(entityLabel)) {
entityLabel = "no-organization";
}
StringBuilder csvFileContent = new StringBuilder();
csvFileContent.append("Entity Name, Grant Count, Entity Type\n");
for (String subOrg : subOrgPublicationsMap.keySet()) {
csvFileContent.append(StringEscapeUtils.escapeCsv(orgLabelMap.get(subOrg)));
csvFileContent.append(", ");
csvFileContent.append(subOrgPublicationsMap.get(subOrg).size());
csvFileContent.append(", ");
csvFileContent.append("Organization");
csvFileContent.append("\n");
}
String outputFileName = UtilityFunctions.slugify(entityLabel) + "_grants-per-year" + ".csv";
fileData.put(DataVisualizationController.FILE_NAME_KEY, outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, csvFileContent.toString());
}
return fileData;
}
}
@ -205,56 +273,6 @@ public class TemporalGrantVisualizationRequestHandler implements
return fileData;
}
private Map<String, String> prepareStandaloneDataResponse(
VitroRequest vitroRequest,
Entity entity) {
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY,
writeGrantsOverTimeJSON(vitroRequest,
entity));
return fileData;
}
/**
* Provides response when json file containing the grant count over the
* years is requested.
*
* @param entity
* @param subentities
* @param subOrganizationTypesResult
*/
private Map<String, String> prepareDataResponse(Entity entity) {
String entityLabel = entity.getEntityLabel();
/*
* To make sure that null/empty records for entity names do not cause any mischief.
* */
if (StringUtils.isBlank(entityLabel)) {
entityLabel = "no-organization";
}
String outputFileName = UtilityFunctions.slugify(entityLabel)
+ "_grants-per-year" + ".csv";
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_NAME_KEY,
outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY,
getEntityGrantsPerYearCSVContent(entity));
return fileData;
}
private TemplateResponseValues prepareStandaloneMarkupResponse(VitroRequest vreq,
String entityURI) {
@ -273,90 +291,6 @@ public class TemporalGrantVisualizationRequestHandler implements
return new TemplateResponseValues(standaloneTemplate, body);
}
/**
* Function to generate a json file for year <-> grant count mapping.
* @param vreq
* @param subentities
* @param subOrganizationTypesResult
*/
private String writeGrantsOverTimeJSON(VitroRequest vreq,
Entity subjectEntity) {
Gson json = new Gson();
Set jsonifiedResponse = new HashSet();
for (SubEntity subentity : subjectEntity.getSubEntities()) {
JsonObject entityJson = new JsonObject(
subentity.getIndividualLabel());
List<List<Integer>> yearGrantCount = new ArrayList<List<Integer>>();
for (Map.Entry<String, Integer> grantEntry : UtilityFunctions
.getYearToActivityCount(subentity.getActivities())
.entrySet()) {
List<Integer> currentGrantYear = new ArrayList<Integer>();
if (grantEntry.getKey().equals(
VOConstants.DEFAULT_GRANT_YEAR)) {
currentGrantYear.add(-1);
} else {
currentGrantYear.add(Integer.parseInt(grantEntry.getKey()));
}
currentGrantYear.add(grantEntry.getValue());
yearGrantCount.add(currentGrantYear);
}
entityJson.setYearToActivityCount(yearGrantCount);
entityJson.setOrganizationTypes(subentity.getEntityTypeLabels());
entityJson.setEntityURI(subentity.getIndividualURI());
entityJson.setLastCachedAtDateTime(subentity.getLastCachedAtDateTime());
if (subentity.getEntityClass().equals(VOConstants.EntityClassType.PERSON)) {
entityJson.setVisMode("PERSON");
} else if (subentity.getEntityClass().equals(VOConstants.EntityClassType.ORGANIZATION)) {
entityJson.setVisMode("ORGANIZATION");
}
jsonifiedResponse.add(entityJson);
}
SubjectEntityJSON subjectEntityJSON = new SubjectEntityJSON(subjectEntity.getEntityLabel(),
subjectEntity.getEntityURI(),
subjectEntity.getParents());
jsonifiedResponse.add(subjectEntityJSON);
return json.toJson(jsonifiedResponse);
}
private String getEntityGrantsPerYearCSVContent(Entity entity) {
StringBuilder csvFileContent = new StringBuilder();
csvFileContent.append("Entity Name, Grant Count, Entity Type\n");
for (SubEntity subEntity : entity.getSubEntities()) {
csvFileContent.append(StringEscapeUtils.escapeCsv(subEntity.getIndividualLabel()));
csvFileContent.append(", ");
csvFileContent.append(subEntity.getActivities().size());
csvFileContent.append(", ");
String allTypes = StringUtils.join(subEntity.getEntityTypeLabels(), "; ");
csvFileContent.append(StringEscapeUtils.escapeCsv(allTypes));
csvFileContent.append("\n");
}
return csvFileContent.toString();
}
@Override
public AuthorizationRequest getRequiredPrivileges() {
return null;

331
src/edu/cornell/mannlib/vitro/webapp/visualization/temporalgraph/TemporalPublicationVisualizationRequestHandler.java
View file

@ -2,13 +2,17 @@
package edu.cornell.mannlib.vitro.webapp.visualization.temporalgraph;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.CounterUtils;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.OrgUtils;
import edu.cornell.mannlib.vitro.webapp.visualization.utilities.VisualizationCaches;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@ -22,15 +26,10 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Res
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.visualization.DataVisualizationController;
import edu.cornell.mannlib.vitro.webapp.controller.visualization.VisualizationFrameworkConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.constants.VOConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.constants.VisConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.exceptions.MalformedQueryParametersException;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.Activity;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.Entity;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.SubEntity;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.json.JsonObject;
import edu.cornell.mannlib.vitro.webapp.visualization.valueobjects.json.SubjectEntityJSON;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.SelectOnModelUtilities;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.UtilityFunctions;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.VisualizationRequestHandler;
@ -38,16 +37,11 @@ public class TemporalPublicationVisualizationRequestHandler implements
VisualizationRequestHandler {
@Override
public ResponseValues generateStandardVisualization(
VitroRequest vitroRequest, Log log, Dataset dataset)
public ResponseValues generateStandardVisualization(VitroRequest vitroRequest, Log log, Dataset dataset)
throws MalformedQueryParametersException {
String entityURI = vitroRequest
.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
return generateStandardVisualizationForPublicationTemporalVis(
vitroRequest, log, dataset, entityURI);
String entityURI = vitroRequest.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
return generateStandardVisualizationForPublicationTemporalVis(vitroRequest, log, dataset, entityURI);
}
@Override
@ -55,10 +49,8 @@ public class TemporalPublicationVisualizationRequestHandler implements
Map<String, String> parameters, VitroRequest vitroRequest, Log log,
Dataset dataset) throws MalformedQueryParametersException {
return generateStandardVisualizationForPublicationTemporalVis(
vitroRequest, log, dataset, parameters.get(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY));
}
private ResponseValues generateStandardVisualizationForPublicationTemporalVis(
@ -66,13 +58,11 @@ public class TemporalPublicationVisualizationRequestHandler implements
String entityURI) throws MalformedQueryParametersException {
if (StringUtils.isBlank(entityURI)) {
entityURI = OrganizationUtilityFunctions
.getStaffProvidedOrComputedHighestLevelOrganization(
log,
dataset,
vitroRequest);
}
@ -83,121 +73,150 @@ public class TemporalPublicationVisualizationRequestHandler implements
VitroRequest vitroRequest, Log log, Dataset dataset,
String subjectEntityURI, VisConstants.DataVisMode visMode)
throws MalformedQueryParametersException {
Entity organizationEntity = SelectOnModelUtilities
.getSubjectOrganizationHierarchy(dataset, subjectEntityURI);
if (organizationEntity.getSubEntities() == null) {
RDFService rdfService = vitroRequest.getRDFService();
Map<String, String> orgLabelMap = VisualizationCaches.cachedOrganizationLabels.get(rdfService);
Map<String, String> personLabelMap = VisualizationCaches.cachedPersonLabels.get(rdfService);
if (orgLabelMap.get(subjectEntityURI) == null) {
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataErrorResponse();
} else {
return prepareDataErrorResponse();
}
}
Map<String, Activity> documentURIForAssociatedPeopleTOVO = new HashMap<String, Activity>();
Map<String, Activity> allDocumentURIToVOs = new HashMap<String, Activity>();
allDocumentURIToVOs = SelectOnModelUtilities.getPublicationsForAllSubOrganizations(dataset, organizationEntity);
Entity organizationWithAssociatedPeople = SelectOnModelUtilities
.getSubjectOrganizationAssociatedPeople(dataset, subjectEntityURI);
if (organizationWithAssociatedPeople.getSubEntities() != null) {
documentURIForAssociatedPeopleTOVO = SelectOnModelUtilities
.getPublicationsForAssociatedPeople(dataset, organizationWithAssociatedPeople.getSubEntities());
organizationEntity = OrganizationUtilityFunctions.mergeEntityIfShareSameURI(
organizationEntity,
organizationWithAssociatedPeople);
}
if (allDocumentURIToVOs.isEmpty() && documentURIForAssociatedPeopleTOVO.isEmpty()) {
Map<String, Set<String>> subOrgMap = VisualizationCaches.cachedOrganizationSubOrgs.get(rdfService);
Map<String, String> orgMostSpecificLabelMap = VisualizationCaches.cachedOrganizationToMostSpecificLabel.get(rdfService);
Map<String, String> personMostSpecificLabelMap = VisualizationCaches.cachedPersonToMostSpecificLabel.get(rdfService);
Map<String, Set<String>> organisationToPeopleMap = VisualizationCaches.cachedOrganisationToPeopleMap.get(rdfService);
Map<String, Set<String>> personToPublicationMap = VisualizationCaches.cachedPersonToPublication.get(rdfService);
Map<String, String> publicationToYearMap = VisualizationCaches.cachedPublicationToYear.get(rdfService);
Set<String> orgPublications = new HashSet<String>();
Set<String> orgPublicationsPeople = new HashSet<String>();
Map<String, Set<String>> subOrgPublicationsMap = new HashMap<String, Set<String>>();
OrgUtils.getObjectMappingsForOrgAnSubOrgs(
subjectEntityURI,
orgPublications,
orgPublicationsPeople,
subOrgPublicationsMap,
subOrgMap,
organisationToPeopleMap,
personToPublicationMap
);
if (orgPublications.isEmpty()) {
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataErrorResponse();
} else {
return prepareDataErrorResponse();
}
} else {
} else {
Map<String, String> fileData = new HashMap<String, String>();
if (VisConstants.DataVisMode.JSON.equals(visMode)) {
return prepareStandaloneDataResponse(vitroRequest, organizationEntity);
Gson json = new Gson();
Set subEntitiesJson = new HashSet();
// For each suborganisation
for (String subOrg : subOrgPublicationsMap.keySet()) {
JsonObject entityJson = new JsonObject(orgLabelMap.get(subOrg));
List<List<Integer>> yearPubCounts = CounterUtils.getObjectCountByYear(subOrgPublicationsMap.get(subOrg), publicationToYearMap);
String type = orgMostSpecificLabelMap.get(subOrg);
entityJson.setYearToActivityCount(yearPubCounts);
entityJson.setOrganizationTypes(Arrays.asList(type == null ? "Organization" : type));
entityJson.setEntityURI(subOrg);
entityJson.setVisMode("ORGANIZATION");
subEntitiesJson.add(entityJson);
}
// For each person
for (String person : orgPublicationsPeople) {
JsonObject entityJson = new JsonObject(personLabelMap.get(person));
List<List<Integer>> yearPubCounts = CounterUtils.getObjectCountByYear(personToPublicationMap.get(person), publicationToYearMap);
String type = personMostSpecificLabelMap.get(person);
entityJson.setYearToActivityCount(yearPubCounts);
entityJson.setOrganizationTypes(Arrays.asList(type == null ? "Person" : type));
entityJson.setEntityURI(person);
entityJson.setVisMode("PERSON");
subEntitiesJson.add(entityJson);
}
SubjectEntityJSON subjectEntityJSON = new SubjectEntityJSON(
orgLabelMap.get(subjectEntityURI),
subjectEntityURI,
OrgUtils.getParentURIsToLabel(subjectEntityURI, subOrgMap, orgLabelMap));
subEntitiesJson.add(subjectEntityJSON);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, json.toJson(subEntitiesJson));
} else {
return prepareDataResponse(organizationEntity);
String entityLabel = orgLabelMap.get(subjectEntityURI);
if (StringUtils.isBlank(entityLabel)) {
entityLabel = "no-organization";
}
StringBuilder csvFileContent = new StringBuilder();
csvFileContent.append("Entity Name, Publication Count, Entity Type\n");
for (String subOrg : subOrgPublicationsMap.keySet()) {
csvFileContent.append(StringEscapeUtils.escapeCsv(orgLabelMap.get(subOrg)));
csvFileContent.append(", ");
csvFileContent.append(subOrgPublicationsMap.get(subOrg).size());
csvFileContent.append(", ");
csvFileContent.append("Organization");
csvFileContent.append("\n");
}
String outputFileName = UtilityFunctions.slugify(entityLabel) + "_publications-per-year" + ".csv";
fileData.put(DataVisualizationController.FILE_NAME_KEY, outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, csvFileContent.toString());
}
return fileData;
}
}
/**
* Provides response when json file containing the publication count over the
* years is requested.
*
* @param entity
* @param subentities
* @param subOrganizationTypesResult
*/
private Map<String, String> prepareDataResponse(Entity entity) {
String entityLabel = entity.getEntityLabel();
/*
* To make sure that null/empty records for entity names do not cause any mischief.
* */
if (StringUtils.isBlank(entityLabel)) {
entityLabel = "no-organization";
}
String outputFileName = UtilityFunctions.slugify(entityLabel)
+ "_publications-per-year" + ".csv";
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_NAME_KEY,
outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY,
getEntityPublicationsPerYearCSVContent(entity));
return fileData;
}
private Map<String, String> prepareDataErrorResponse() {
String outputFileName = "no-organization_publications-per-year.csv";
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_NAME_KEY,
outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, "");
fileData.put(DataVisualizationController.FILE_NAME_KEY, outputFileName);
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, "");
return fileData;
}
private Map<String, String> prepareStandaloneDataErrorResponse() {
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY,
"{\"error\" : \"No Publications for this Organization found in VIVO.\"}");
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY, "application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY, "{\"error\" : \"No Publications for this Organization found in VIVO.\"}");
return fileData;
}
@Override
public Map<String, String> generateDataVisualization(
VitroRequest vitroRequest, Log log, Dataset dataset)
public Map<String, String> generateDataVisualization(VitroRequest vitroRequest, Log log, Dataset dataset)
throws MalformedQueryParametersException {
String entityURI = vitroRequest
.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
String entityURI = vitroRequest.getParameter(VisualizationFrameworkConstants.INDIVIDUAL_URI_KEY);
VisConstants.DataVisMode currentDataMode = VisConstants.DataVisMode.CSV;
@ -233,33 +252,13 @@ public class TemporalPublicationVisualizationRequestHandler implements
@Override
public Object generateAjaxVisualization(VitroRequest vitroRequest, Log log,
Dataset dataset) throws MalformedQueryParametersException {
public Object generateAjaxVisualization(VitroRequest vitroRequest, Log log, Dataset dataset) throws MalformedQueryParametersException {
throw new UnsupportedOperationException("Entity Pub Count does not provide Ajax Response.");
}
private Map<String, String> prepareStandaloneDataResponse(
VitroRequest vitroRequest,
Entity entity) {
Map<String, String> fileData = new HashMap<String, String>();
fileData.put(DataVisualizationController.FILE_CONTENT_TYPE_KEY,
"application/octet-stream");
fileData.put(DataVisualizationController.FILE_CONTENT_KEY,
writePublicationsOverTimeJSON(vitroRequest,
entity));
return fileData;
}
private TemplateResponseValues prepareStandaloneMarkupResponse(VitroRequest vreq,
String entityURI) {
private TemplateResponseValues prepareStandaloneMarkupResponse(VitroRequest vreq, String entityURI) {
String standaloneTemplate = "entityComparisonOnPublicationsStandalone.ftl";
String organizationLabel = OrganizationUtilityFunctions
.getEntityLabelFromDAO(vreq,
entityURI);
String organizationLabel = OrganizationUtilityFunctions.getEntityLabelFromDAO(vreq, entityURI);
Map<String, Object> body = new HashMap<String, Object>();
body.put("title", organizationLabel + " - Temporal Graph Visualization");
@ -271,91 +270,9 @@ public class TemporalPublicationVisualizationRequestHandler implements
return new TemplateResponseValues(standaloneTemplate, body);
}
/**
* Function to generate a json file for year <-> publication count mapping.
* @param vreq
* @param subentities
* @param subOrganizationTypesResult
*/
private String writePublicationsOverTimeJSON(VitroRequest vreq,
Entity subjectEntity) {
Gson json = new Gson();
Set subEntitiesJson = new HashSet();
for (SubEntity subentity : subjectEntity.getSubEntities()) {
JsonObject entityJson = new JsonObject(
subentity.getIndividualLabel());
List<List<Integer>> yearPubCount = new ArrayList<List<Integer>>();
for (Map.Entry<String, Integer> pubEntry : UtilityFunctions
.getYearToActivityCount(subentity.getActivities())
.entrySet()) {
List<Integer> currentPubYear = new ArrayList<Integer>();
if (pubEntry.getKey().equals(VOConstants.DEFAULT_PUBLICATION_YEAR)) {
currentPubYear.add(-1);
} else {
currentPubYear.add(Integer.parseInt(pubEntry.getKey()));
}
currentPubYear.add(pubEntry.getValue());
yearPubCount.add(currentPubYear);
}
entityJson.setYearToActivityCount(yearPubCount);
entityJson.setOrganizationTypes(subentity.getEntityTypeLabels());
entityJson.setEntityURI(subentity.getIndividualURI());
entityJson.setLastCachedAtDateTime(subentity.getLastCachedAtDateTime());
if (subentity.getEntityClass().equals(VOConstants.EntityClassType.PERSON)) {
entityJson.setVisMode("PERSON");
} else if (subentity.getEntityClass().equals(VOConstants.EntityClassType.ORGANIZATION)) {
entityJson.setVisMode("ORGANIZATION");
}
subEntitiesJson.add(entityJson);
}
SubjectEntityJSON subjectEntityJSON = new SubjectEntityJSON(subjectEntity.getEntityLabel(),
subjectEntity.getEntityURI(),
subjectEntity.getParents());
subEntitiesJson.add(subjectEntityJSON);
return json.toJson(subEntitiesJson);
}
private String getEntityPublicationsPerYearCSVContent(Entity entity) {
StringBuilder csvFileContent = new StringBuilder();
csvFileContent.append("Entity Name, Publication Count, Entity Type\n");
for (SubEntity subEntity : entity.getSubEntities()) {
csvFileContent.append(StringEscapeUtils.escapeCsv(subEntity.getIndividualLabel()));
csvFileContent.append(", ");
csvFileContent.append(subEntity.getActivities().size());
csvFileContent.append(", ");
String allTypes = StringUtils.join(subEntity.getEntityTypeLabels(), "; ");
csvFileContent.append(StringEscapeUtils.escapeCsv(allTypes));
csvFileContent.append("\n");
}
return csvFileContent.toString();
}
@Override
public AuthorizationRequest getRequiredPrivileges() {
// TODO Auto-generated method stub
return null;
}
}
}

108
src/edu/cornell/mannlib/vitro/webapp/visualization/utilities/CachingRDFServiceExecutor.java Normal file
View file

@ -0,0 +1,108 @@
package edu.cornell.mannlib.vitro.webapp.visualization.utilities;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
public class CachingRDFServiceExecutor<T> {
private T cachedResults;
private long lastCacheTime;
private RDFServiceCallable<T> resultBuilder;
private FutureTask<T> backgroundTask = null;
public CachingRDFServiceExecutor(RDFServiceCallable<T> resultBuilder) {
this.resultBuilder = resultBuilder;
}
public synchronized T get(RDFService rdfService) {
if (cachedResults != null) {
if (!resultBuilder.invalidateCache(System.currentTimeMillis() - lastCacheTime)) {
return cachedResults;
}
}
try {
if (backgroundTask == null) {
resultBuilder.setRDFService(rdfService);
backgroundTask = new FutureTask<T>(resultBuilder);
Thread thread = new Thread(backgroundTask);
thread.setDaemon(true);
thread.start();
if (cachedResults == null || resultBuilder.executionTime < 2000) {
completeBackgroundTask();
}
} else if (backgroundTask.isDone()) {
completeBackgroundTask();
}
} catch (InterruptedException e) {
abortBackgroundTask();
} catch (ExecutionException e) {
abortBackgroundTask();
throw new RuntimeException("Background RDF thread through an exception", e.getCause());
}
return cachedResults;
}
private void abortBackgroundTask() {
if (backgroundTask != null) {
backgroundTask.cancel(true);
backgroundTask = null;
}
}
private void completeBackgroundTask() throws InterruptedException, ExecutionException {
if (backgroundTask != null) {
cachedResults = backgroundTask.get();
lastCacheTime = System.currentTimeMillis();
backgroundTask = null;
}
}
public static abstract class RDFServiceCallable<T> implements Callable<T> {
private RDFService rdfService;
private long executionTime = -1;
final void setRDFService(RDFService rdfService) {
this.rdfService = rdfService;
}
@Override
final public T call() throws Exception {
long start = System.currentTimeMillis();
T val = callWithService(rdfService);
executionTime = System.currentTimeMillis() - start;
return val;
}
protected abstract T callWithService(RDFService rdfService) throws Exception;
boolean invalidateCache(long timeCached) {
if (executionTime > -1) {
/*
Determine validity as a function of the time it takes to execute the query.
Query exec time | Keep cache for
-----------------+-----------------
10 seconds | 20 minutes
30 seconds | 1 hour
1 minute | 2 hours
5 minutes | 10 hours
Multiplier of the last execution time is 120.
At most, keep a cache for one day (24 * 60 * 60 * 1000 = 86400000)
*/
return timeCached > Math.min(executionTime * 120, 86400000);
}
return false;
}
}
}

47
src/edu/cornell/mannlib/vitro/webapp/visualization/utilities/CounterUtils.java Normal file
View file

@ -0,0 +1,47 @@
package edu.cornell.mannlib.vitro.webapp.visualization.utilities;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class CounterUtils {
public static final List<List<Integer>> getObjectCountByYear(Set<String> objects, Map<String, String> objectToYearMap) {
List<List<Integer>> yearCounts = new ArrayList<>();
if (objects != null) {
int[] counts = new int[Calendar.getInstance().get(Calendar.YEAR) + 1000];
for (String publication : objects) {
int year = 0;
try {
year = Integer.parseInt(objectToYearMap.get(publication), 10);
} catch (Throwable t) {
}
if (year > counts.length - 1) {
year = 0;
}
counts[year]++;
}
for (int i = 1; i < counts.length; i++) {
if (counts[i] > 0) {
List<Integer> currentYear = new ArrayList<Integer>();
currentYear.add(i);
currentYear.add(counts[i]);
yearCounts.add(currentYear);
}
}
if (counts[0] > 0) {
List<Integer> currentYear = new ArrayList<Integer>();
currentYear.add(-1);
currentYear.add(counts[0]);
yearCounts.add(currentYear);
}
}
return yearCounts;
}
}

86
src/edu/cornell/mannlib/vitro/webapp/visualization/utilities/OrgUtils.java Normal file
View file

@ -0,0 +1,86 @@
package edu.cornell.mannlib.vitro.webapp.visualization.utilities;
import org.apache.commons.lang.StringUtils;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
public class OrgUtils {
public static Map<String, String> getParentURIsToLabel(String org, Map<String, Set<String>> subOrgMap, Map<String, String> orgLabelMap) {
Map<String, String> parentURIsToLabel = new TreeMap<>();
if (!StringUtils.isEmpty(org)) {
for (Map.Entry<String, Set<String>> orgMapEntry : subOrgMap.entrySet()) {
Set<String> subOrgs = orgMapEntry.getValue();
if (subOrgs != null && subOrgs.contains(org) && orgLabelMap.containsKey(orgMapEntry.getKey())) {
parentURIsToLabel.put(orgMapEntry.getKey(), orgLabelMap.get(orgMapEntry.getKey()));
}
}
}
return parentURIsToLabel;
}
public static void getObjectMappingsForOrgAnSubOrgs(
String orgUri,
Set<String> orgObjects,
Set<String> orgObjectsIncludesPeople,
Map<String, Set<String>> subOrgObjectMap,
Map<String, Set<String>> subOrgMap,
Map<String, Set<String>> organisationToPeopleMap,
Map<String, Set<String>> personToObjectMap
) {
if (subOrgMap.containsKey(orgUri)) {
for (String topSubOrg : subOrgMap.get(orgUri)) {
Set<String> subOrgPublications = new HashSet<String>();
Set<String> subOrgPublicationsPeople = new HashSet<String>();
Set<String> fullSubOrgs = OrgUtils.orgAndAllSubOrgs(new HashSet<String>(), topSubOrg, subOrgMap);
for (String subOrg : fullSubOrgs) {
Set<String> peopleInSubOrg = organisationToPeopleMap.get(subOrg);
if (peopleInSubOrg != null) {
for (String person : peopleInSubOrg) {
if (personToObjectMap.containsKey(person)) {
if (subOrgPublicationsPeople.add(person)) {
subOrgPublications.addAll(personToObjectMap.get(person));
if (orgObjectsIncludesPeople.add(person)) {
orgObjects.addAll(personToObjectMap.get(person));
}
}
}
}
}
}
subOrgObjectMap.put(topSubOrg, subOrgPublications);
}
}
Set<String> people = organisationToPeopleMap.get(orgUri);
if (people != null) {
for (String person : people) {
if (personToObjectMap.containsKey(person)) {
if (orgObjectsIncludesPeople.add(person)) {
orgObjects.addAll(personToObjectMap.get(person));
}
}
}
}
}
private static Set<String> orgAndAllSubOrgs(Set<String> allSubOrgs, String org, Map<String, Set<String>> subOrgMap) {
if (allSubOrgs.add(org)) {
if (subOrgMap.containsKey(org)) {
for (String subOrg : subOrgMap.get(org)) {
orgAndAllSubOrgs(allSubOrgs, subOrg, subOrgMap);
}
}
}
return allSubOrgs;
}
}

507
src/edu/cornell/mannlib/vitro/webapp/visualization/utilities/VisualizationCaches.java Normal file
View file

@ -0,0 +1,507 @@
package edu.cornell.mannlib.vitro.webapp.visualization.utilities;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.ResultSetFactory;
import com.hp.hpl.jena.rdf.model.Resource;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.visualization.constants.QueryConstants;
import edu.cornell.mannlib.vitro.webapp.visualization.visutils.UtilityFunctions;
import org.joda.time.DateTime;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
final public class VisualizationCaches {
public static final CachingRDFServiceExecutor<Map<String, String>> cachedOrganizationLabels =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?org ?orgLabel\n" +
"WHERE\n" +
"{\n" +
" ?org a foaf:Organization .\n" +
" ?org rdfs:label ?orgLabel .\n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("org").getURI();
String orgLabel = qs.getLiteral("orgLabel").getString();
map.put(org, orgLabel);
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, Set<String>>> cachedOrganizationSubOrgs =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
protected Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?org ?subOrg\n" +
"WHERE\n" +
"{\n" +
" ?org a foaf:Organization .\n" +
" ?org <http://purl.obolibrary.org/obo/BFO_0000051> ?subOrg .\n" +
"}\n";
Map<String, Set<String>> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("org").getURI();
String subOrg = qs.getResource("subOrg").getURI();
Set<String> subOrgs = map.get(org);
if (subOrgs == null) {
subOrgs = new HashSet<String>();
subOrgs.add(subOrg);
map.put(org, subOrgs);
} else {
subOrgs.add(subOrg);
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedOrganizationToMostSpecificLabel =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?org ?typeLabel\n" +
"WHERE\n" +
"{\n" +
" ?org a foaf:Organization .\n" +
" ?org vitro:mostSpecificType ?type .\n" +
" ?type rdfs:label ?typeLabel .\n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("org").getURI();
String typeLabel = qs.getLiteral("typeLabel").getString();
map.put(org, String.valueOf(typeLabel));
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, Set<String>>> cachedOrganisationToPeopleMap =
new CachingRDFServiceExecutor<Map<String, Set<String>>>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
protected Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?organisation ?person\n" +
"WHERE\n" +
"{\n" +
" ?organisation a foaf:Organization .\n" +
" ?organisation core:relatedBy ?position .\n" +
" ?position core:relates ?person .\n" +
" ?person a foaf:Person .\n" +
"}\n";
// TODO Critical section?
Map<String, Set<String>> orgToPeopleMap = new HashMap<String, Set<String>>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String org = qs.getResource("organisation").getURI();
String person = qs.getResource("person").getURI();
Set<String> people = orgToPeopleMap.get(org);
if (people == null) {
people = new HashSet<String>();
people.add(person);
orgToPeopleMap.put(org, people);
} else {
people.add(person);
}
}
} finally {
silentlyClose(is);
}
return orgToPeopleMap;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedPersonLabels =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?person ?personLabel\n" +
"WHERE\n" +
"{\n" +
" ?person a foaf:Person .\n" +
" ?person rdfs:label ?personLabel .\n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String person = qs.getResource("person").getURI();
String personLabel = qs.getLiteral("personLabel").getString();
map.put(person, personLabel);
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, Set<String>>> cachedPersonToPublication =
new CachingRDFServiceExecutor<Map<String, Set<String>>>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
protected Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?person ?document\n" +
"WHERE\n" +
"{\n" +
" ?person a foaf:Person .\n" +
" ?person core:relatedBy ?authorship .\n" +
" ?authorship a core:Authorship .\n" +
" ?authorship core:relates ?document .\n" +
" ?document a bibo:Document .\n" +
"}\n";
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
Resource person = qs.getResource("person");
Resource document = qs.getResource("document");
if (person != null && document != null) {
String personURI = person.getURI();
Set<String> documents = map.get(personURI);
if (documents == null) {
documents = new HashSet<String>();
documents.add(document.getURI());
map.put(personURI, documents);
} else {
documents.add(document.getURI());
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedPublicationToYear =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?document ?publicationDate\n" +
"WHERE\n" +
"{\n" +
" ?document a bibo:Document .\n" +
" ?document core:dateTimeValue ?dateTimeValue . \n" +
" ?dateTimeValue core:dateTime ?publicationDate . \n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String document = qs.getResource("document").getURI();
String pubDate = qs.getLiteral("publicationDate").getString();
if (pubDate != null) {
DateTime validParsedDateTimeObject = UtilityFunctions
.getValidParsedDateTimeObject(pubDate);
if (validParsedDateTimeObject != null) {
map.put(document, String.valueOf(validParsedDateTimeObject.getYear()));
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, Set<String>>> cachedPersonToGrant =
new CachingRDFServiceExecutor<Map<String, Set<String>>>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, Set<String>>>() {
@Override
protected Map<String, Set<String>> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?person ?grant\n" +
"WHERE\n" +
"{\n" +
" ?person a foaf:Person .\n" +
" ?person <http://purl.obolibrary.org/obo/RO_0000053> ?role .\n" +
" { ?role a core:PrincipalInvestigatorRole . } UNION { ?role a core:CoPrincipalInvestigatorRole . } \n" +
" ?role core:relatedBy ?grant .\n" +
" ?grant a core:Grant .\n" +
"}\n";
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
Resource person = qs.getResource("person");
Resource grant = qs.getResource("grant");
if (person != null && grant != null) {
String personURI = person.getURI();
Set<String> documents = map.get(personURI);
if (documents == null) {
documents = new HashSet<String>();
documents.add(grant.getURI());
map.put(personURI, documents);
} else {
documents.add(grant.getURI());
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedGrantToYear =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?grant ?startDateTimeValue\n" +
"WHERE\n" +
"{\n" +
" ?grant a core:Grant .\n" +
" ?grant core:dateTimeInterval ?dateTimeIntervalValue . \n" +
" ?dateTimeIntervalValue core:start ?startDate . \n" +
" ?startDate core:dateTime ?startDateTimeValue . \n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String grant = qs.getResource("grant").getURI();
String startDate = qs.getLiteral("startDateTimeValue").getString();
if (startDate != null) {
DateTime validParsedDateTimeObject = UtilityFunctions
.getValidParsedDateTimeObject(startDate);
if (validParsedDateTimeObject != null) {
map.put(grant, String.valueOf(validParsedDateTimeObject.getYear()));
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedGrantToRoleYear =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?grant ?startDateTimeValue\n" +
"WHERE\n" +
"{\n" +
" ?grant a core:Grant .\n" +
" ?grant core:relates ?role .\n" +
" ?role core:dateTimeInterval ?dateTimeIntervalValue . \n" +
" ?dateTimeIntervalValue core:start ?startDate . \n" +
" ?startDate core:dateTime ?startDateTimeValue . \n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String grant = qs.getResource("grant").getURI();
String startDate = qs.getLiteral("startDateTimeValue").getString();
if (startDate != null) {
DateTime validParsedDateTimeObject = UtilityFunctions
.getValidParsedDateTimeObject(startDate);
if (validParsedDateTimeObject != null) {
map.put(grant, String.valueOf(validParsedDateTimeObject.getYear()));
}
}
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
public static final CachingRDFServiceExecutor<Map<String, String>> cachedPersonToMostSpecificLabel =
new CachingRDFServiceExecutor<>(
new CachingRDFServiceExecutor.RDFServiceCallable<Map<String, String>>() {
@Override
protected Map<String, String> callWithService(RDFService rdfService) throws Exception {
String query = QueryConstants.getSparqlPrefixQuery() +
"SELECT ?person ?typeLabel\n" +
"WHERE\n" +
"{\n" +
" ?person a foaf:Person .\n" +
" ?person vitro:mostSpecificType ?type .\n" +
" ?type rdfs:label ?typeLabel .\n" +
"}\n";
Map<String, String> map = new HashMap<>();
InputStream is = null;
ResultSet rs = null;
try {
is = rdfService.sparqlSelectQuery(query, RDFService.ResultFormat.JSON);
rs = ResultSetFactory.fromJSON(is);
while (rs.hasNext()) {
QuerySolution qs = rs.next();
String person = qs.getResource("person").getURI();
String typeLabel = qs.getLiteral("typeLabel").getString();
map.put(person, String.valueOf(typeLabel));
}
} finally {
silentlyClose(is);
}
return map;
}
}
);
private static void silentlyClose(InputStream is) {
try {
if (is != null) {
is.close();
}
} catch (Throwable t) {
}
}
}

7
src/edu/cornell/mannlib/vitro/webapp/visualization/valueobjects/json/SubjectEntityJSON.java
View file

@ -21,6 +21,13 @@ public class SubjectEntityJSON {
this.setParentURIToLabel(parentOrganizations);
}
public SubjectEntityJSON(String subjectEntityURI, String label,
Map<String, String> parentURIToLabel) {
this.subjectEntityURI = subjectEntityURI;
this.subjectEntityLabel = label;
this.parentURIToLabel = parentURIToLabel;
}
public String getSubjectEntityURI() {
return subjectEntityURI;
}