From 78491234db2199f83a05e8d0d30cd402caeba33d Mon Sep 17 00:00:00 2001 From: Jim Blake Date: Tue, 20 Jan 2015 10:25:55 -0500 Subject: [PATCH] VIVO-871 Improve performance when reading object properties. Also, improve display. --- .../jena/ObjectPropertyStatementDaoJena.java | 4 +- .../jena/ObjectPropertyStatementDaoSDB.java | 313 ++++++++++-------- .../tasks/UpdateStatementsTask.java | 2 +- .../searchindex/tasks/UpdateUrisTask.java | 2 +- .../body/admin/admin-showThreads.ftl | 23 +- 5 files changed, 196 insertions(+), 148 deletions(-) diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java index 34eeb6dd8..44bedbdd2 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java @@ -53,8 +53,8 @@ public class ObjectPropertyStatementDaoJena extends JenaBaseDao implements Objec private static final Log log = LogFactory.getLog(ObjectPropertyStatementDaoJena.class); - private DatasetWrapperFactory dwf; - private RDFService rdfService; + protected DatasetWrapperFactory dwf; + protected RDFService rdfService; public ObjectPropertyStatementDaoJena(RDFService rdfService, DatasetWrapperFactory dwf, diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java index 07565f011..30cf15db9 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java @@ -2,164 +2,205 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena; +import static edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat.N3; + import java.util.ArrayList; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import com.hp.hpl.jena.ontology.OntModel; -import com.hp.hpl.jena.ontology.OntModelSpec; -import com.hp.hpl.jena.query.Dataset; -import com.hp.hpl.jena.query.QueryExecution; -import com.hp.hpl.jena.query.QueryExecutionFactory; -import com.hp.hpl.jena.query.QueryFactory; import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.ModelFactory; -import com.hp.hpl.jena.rdf.model.Property; -import com.hp.hpl.jena.rdf.model.Resource; +import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.Statement; -import com.hp.hpl.jena.shared.Lock; -import com.hp.hpl.jena.util.iterator.ClosableIterator; +import com.hp.hpl.jena.vocabulary.RDF; import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty; import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement; import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatementImpl; import edu.cornell.mannlib.vitro.webapp.dao.ObjectPropertyStatementDao; -import edu.cornell.mannlib.vitro.webapp.dao.jena.IndividualSDB.IndividualNotFoundException; import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactorySDB.SDBDatasetMode; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; +import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; public class ObjectPropertyStatementDaoSDB extends ObjectPropertyStatementDaoJena implements ObjectPropertyStatementDao { + private static final Log log = LogFactory + .getLog(ObjectPropertyStatementDaoSDB.class); - private static final Log log = LogFactory.getLog(ObjectPropertyStatementDaoSDB.class); - - private DatasetWrapperFactory dwf; - private SDBDatasetMode datasetMode; - private WebappDaoFactorySDB wadf; - - public ObjectPropertyStatementDaoSDB( - RDFService rdfService, - DatasetWrapperFactory dwf, - SDBDatasetMode datasetMode, - WebappDaoFactorySDB wadf) { - super (rdfService, dwf, wadf); - this.dwf = dwf; - this.datasetMode = datasetMode; + // Get the types of the base entity. + private static final String SUBJECT_TYPE_QUERY = "" + + "PREFIX rdf: \n" + + "CONSTRUCT { \n" // + + " ?uri rdf:type ?type . \n" // + + "} WHERE { \n" // + + " ?uri rdf:type ?type . \n" // + + "} \n"; + + // Get the types of all objects of properties. + private static final String OBJECT_TYPE_QUERY = "" + + "PREFIX rdf: \n" + + "CONSTRUCT { \n" // + + " ?uri ?p ?o . \n" // + + " ?o rdf:type ?type . \n" // + + "} WHERE { \n" // + + " ?uri ?p ?o . \n" // + + " ?o rdf:type ?type . \n" // + + "} \n"; + + // Get the labels of all objects of properties. + private static final String OBJECT_LABEL_QUERY = "" + + "PREFIX rdfs: \n" + + "CONSTRUCT { \n" // + + " ?uri ?p ?o . \n" // + + " ?o rdfs:label ?label . \n" // + + "} WHERE { \n" // + + " ?uri ?p ?o . \n" // + + " ?o rdfs:label ?label . \n" // + + "} \n"; + + private final WebappDaoFactorySDB wadf; + private final SDBDatasetMode datasetMode; + + public ObjectPropertyStatementDaoSDB(RDFService rdfService, + DatasetWrapperFactory dwf, SDBDatasetMode datasetMode, + WebappDaoFactorySDB wadf) { + super(rdfService, dwf, wadf); this.wadf = wadf; + this.datasetMode = datasetMode; + } + + @Override + public Individual fillExistingObjectPropertyStatements(Individual entity) { + if (entity == null || entity.getURI() == null) + return entity; + else { + List objectPropertyStatements = new ArrayList<>(); + String subjectUri = entity.getURI(); + + Model m = getInfoForObjectsOfThisEntity(subjectUri); + + Set subjectTypes = getTypes(m, subjectUri); + for (ObjectPropertyPair pair : getRawObjectPropertyPairs(m, + subjectUri)) { + String predicateUri = pair.getPredicateUri(); + String objectUri = pair.getObjectUri(); + Set objectTypes = getTypes(m, objectUri); + + ObjectProperty prop = findRawProperty(predicateUri); + if (prop == null) { + continue; + } + + Individual object = new IndividualSDB(objectUri, dwf, + datasetMode, wadf, m); + objectPropertyStatements.add(createStatement(entity, prop, + object)); + } + entity.setObjectPropertyStatements(objectPropertyStatements); + return entity; + } + } + + /** + * Get the types of this entity. Get the related object and the predicates + * by which they are related. Get the types and labels of those related + * objects. + */ + private Model getInfoForObjectsOfThisEntity(String subjectUri) { + Model m = ModelFactory.createDefaultModel(); + try { + m.add(RDFServiceUtils.parseModel( + rdfService.sparqlConstructQuery( + substituteUri(subjectUri, SUBJECT_TYPE_QUERY), N3), + N3)); + m.add(RDFServiceUtils.parseModel( + rdfService.sparqlConstructQuery( + substituteUri(subjectUri, OBJECT_TYPE_QUERY), N3), + N3)); + m.add(RDFServiceUtils.parseModel( + rdfService.sparqlConstructQuery( + substituteUri(subjectUri, OBJECT_LABEL_QUERY), N3), + N3)); + } catch (RDFServiceException e) { + log.warn("Failed to fill object property statements for '" + + subjectUri + "'", e); + } + return m; + } + + private String substituteUri(String uri, String query) { + return query.replace("?uri", "<" + uri + "> "); + } + + private Set getTypes(Model m, String uri) { + Set typeUris = new HashSet<>(); + for (RDFNode typeNode : m.listObjectsOfProperty(m.createResource(uri), + RDF.type).toSet()) { + if (typeNode.isURIResource()) { + typeUris.add(typeNode.asResource().getURI()); + } + } + return typeUris; + } + + private List getRawObjectPropertyPairs(Model m, + String subjectUri) { + List list = new ArrayList<>(); + for (Statement stmt : m.listStatements(m.createResource(subjectUri), + null, (RDFNode) null).toList()) { + if (wadf.getNonuserNamespaces().contains( + stmt.getPredicate().getNameSpace())) { + continue; + } + if (!stmt.getObject().isURIResource()) { + continue; + } + list.add(new ObjectPropertyPair(stmt.getPredicate().getURI(), stmt + .getObject().asResource().getURI())); + } + return list; + } + + private ObjectProperty findRawProperty(String predicateUri) { + return wadf.getObjectPropertyDao().getObjectPropertyByURI(predicateUri); + } + + private ObjectPropertyStatement createStatement(Individual entity, + ObjectProperty prop, Individual object) { + ObjectPropertyStatementImpl ops = new ObjectPropertyStatementImpl(); + ops.setSubject(entity); + ops.setProperty(prop); + ops.setObject(object); + return ops; + } + + // ---------------------------------------------------------------------- + // Helper classes + // ---------------------------------------------------------------------- + + private static class ObjectPropertyPair { + private final String predicateUri; + private final String objectUri; + + public ObjectPropertyPair(String predicateUri, String objectUri) { + this.predicateUri = predicateUri; + this.objectUri = objectUri; + } + + public String getPredicateUri() { + return predicateUri; + } + + public String getObjectUri() { + return objectUri; + } + } - - @Override - public Individual fillExistingObjectPropertyStatements(Individual entity) { - if (entity.getURI() == null) - return entity; - else { - Map uriToObjectProperty = new HashMap(); - String query = "CONSTRUCT { \n" + - " <" + entity.getURI() + "> ?p ?o . \n" + -// " ?o a ?oType . \n" + -// " ?o <" + RDFS.label.getURI() + "> ?oLabel . \n" + -// " ?o <" + VitroVocabulary.MONIKER + "> ?oMoniker \n" + - "} WHERE { \n" + - " { <" + entity.getURI() + "> ?p ?o } \n" + -// " UNION { <" + entity.getURI() + "> ?p ?o . ?o a ?oType } \n" + -// " UNION { <" + entity.getURI() + "> ?p ?o . \n" + -// " ?o <" + RDFS.label.getURI() + "> ?oLabel } \n" + -// " UNION { <" + entity.getURI() + "> ?p ?o . \n " + -// " ?o <" + VitroVocabulary.MONIKER + "> ?oMoniker } \n" + - "}"; - long startTime = System.currentTimeMillis(); - Model m = null; - DatasetWrapper w = dwf.getDatasetWrapper(); - Dataset dataset = w.getDataset(); - dataset.getLock().enterCriticalSection(Lock.READ); - QueryExecution qexec = null; - try { - qexec = QueryExecutionFactory.create(QueryFactory.create(query), dataset); - m = qexec.execConstruct(); - } finally { - if(qexec != null) qexec.close(); - dataset.getLock().leaveCriticalSection(); - w.close(); - } - if (log.isDebugEnabled()) { - log.debug("Time (ms) to query for related individuals: " + (System.currentTimeMillis() - startTime)); - if (System.currentTimeMillis() - startTime > 1000) { - //log.debug(query); - log.debug("Results size (statements): " + m.size()); - } - } - - OntModel ontModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, m); - ontModel.enterCriticalSection(Lock.READ); - try { - Resource ind = ontModel.getResource(entity.getURI()); - List objPropertyStmtList = new ArrayList(); - ClosableIterator propIt = ind.listProperties(); - try { - while (propIt.hasNext()) { - Statement st = propIt.next(); - if (st.getObject().isResource() && !(NONUSER_NAMESPACES.contains(st.getPredicate().getNameSpace()))) { - try { - ObjectPropertyStatement objPropertyStmt = new ObjectPropertyStatementImpl(); - objPropertyStmt.setSubjectURI(entity.getURI()); - objPropertyStmt.setSubject(entity); - objPropertyStmt.setObjectURI(((Resource)st.getObject()).getURI()); - - objPropertyStmt.setPropertyURI(st.getPredicate().getURI()); - Property prop = st.getPredicate(); - if( uriToObjectProperty.containsKey(prop.getURI())){ - objPropertyStmt.setProperty(uriToObjectProperty.get(prop.getURI())); - }else{ - ObjectProperty p = getWebappDaoFactory().getObjectPropertyDao().getObjectPropertyByURI(prop.getURI()); - if( p != null ){ - uriToObjectProperty.put(prop.getURI(), p); - objPropertyStmt.setProperty(uriToObjectProperty.get(prop.getURI())); - }else{ - //if ObjectProperty not found in ontology, skip it - continue; - } - } - if (objPropertyStmt.getObjectURI() != null) { - //this might throw IndividualNotFoundException - Individual objInd = new IndividualSDB( - objPropertyStmt.getObjectURI(), - this.dwf, - datasetMode, - wadf); - objPropertyStmt.setObject(objInd); - } - - //only add statement to list if it has its values filled out - if ( (objPropertyStmt.getSubjectURI() != null) - && (objPropertyStmt.getPropertyURI() != null) - && (objPropertyStmt.getObject() != null) ) { - objPropertyStmtList.add(objPropertyStmt); - } - - } catch (IndividualNotFoundException t) { - log.debug(t,t); - continue; - } catch (Throwable t){ - log.error(t,t); - continue; - } - } - } - } finally { - propIt.close(); - } - entity.setObjectPropertyStatements(objPropertyStmtList); - } finally { - ontModel.leaveCriticalSection(); - } - return entity; - } - } - } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java index 505c58fcb..dcf489fa6 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java @@ -77,7 +77,7 @@ public class UpdateStatementsTask implements Task { this.uris = Collections.synchronizedSet(new HashSet()); - this.status = new Status(changes.size(), 200, listeners); + this.status = new Status(changes.size(), 500, listeners); } @Override diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java index 9eec3d2ff..d84f40799 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java @@ -66,7 +66,7 @@ public class UpdateUrisTask implements Task { this.listeners = listeners; this.pool = pool; - this.status = new Status(uris.size(), 200, listeners); + this.status = new Status(uris.size(), 500, listeners); this.searchEngine = ApplicationUtils.instance().getSearchEngine(); diff --git a/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl b/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl index 70043a9d5..cf6e61d0f 100644 --- a/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl +++ b/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl @@ -20,12 +20,19 @@ table.threadInfo th {

${i18n().background_threads}

- <#list threads as threadInfo> - - - - - -
${i18n().name}${threadInfo.name}
${i18n().work_level}${threadInfo.workLevel}
${i18n().since}${threadInfo.since}
${i18n().flags}${threadInfo.flags}
- + + + + + + + + <#list threads as threadInfo> + + + + + + +
${i18n().name}${i18n().work_level}${i18n().since}${i18n().flags}
${threadInfo.name}${threadInfo.workLevel}${threadInfo.since}${threadInfo.flags}
\ No newline at end of file