VIVO-871 Improve performance when reading object properties.

Also, improve display.
2015-01-20 10:25:55 -05:00 · 2015-01-20 10:25:55 -05:00 · 78491234db
commit 78491234db
parent 56a640fe44
5 changed files with 196 additions and 148 deletions
--- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java
+++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoJena.java
@ -53,8 +53,8 @@ public class ObjectPropertyStatementDaoJena extends JenaBaseDao implements Objec

    private static final Log log = LogFactory.getLog(ObjectPropertyStatementDaoJena.class);
    
-    private DatasetWrapperFactory dwf;
-    private RDFService rdfService;
+    protected DatasetWrapperFactory dwf;
+    protected RDFService rdfService;
    
    public ObjectPropertyStatementDaoJena(RDFService rdfService,
                                          DatasetWrapperFactory dwf,
--- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java
+++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyStatementDaoSDB.java
@ -2,164 +2,205 @@

 package edu.cornell.mannlib.vitro.webapp.dao.jena;

+import static edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat.N3;
+
 import java.util.ArrayList;
-import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
-import java.util.Map;
+import java.util.Set;

 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;

-import com.hp.hpl.jena.ontology.OntModel;
-import com.hp.hpl.jena.ontology.OntModelSpec;
-import com.hp.hpl.jena.query.Dataset;
-import com.hp.hpl.jena.query.QueryExecution;
-import com.hp.hpl.jena.query.QueryExecutionFactory;
-import com.hp.hpl.jena.query.QueryFactory;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
-import com.hp.hpl.jena.rdf.model.Property;
-import com.hp.hpl.jena.rdf.model.Resource;
+import com.hp.hpl.jena.rdf.model.RDFNode;
 import com.hp.hpl.jena.rdf.model.Statement;
-import com.hp.hpl.jena.shared.Lock;
-import com.hp.hpl.jena.util.iterator.ClosableIterator;
+import com.hp.hpl.jena.vocabulary.RDF;

 import edu.cornell.mannlib.vitro.webapp.beans.Individual;
 import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
 import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
 import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatementImpl;
 import edu.cornell.mannlib.vitro.webapp.dao.ObjectPropertyStatementDao;
-import edu.cornell.mannlib.vitro.webapp.dao.jena.IndividualSDB.IndividualNotFoundException;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactorySDB.SDBDatasetMode;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;

 public class ObjectPropertyStatementDaoSDB extends
 		ObjectPropertyStatementDaoJena implements ObjectPropertyStatementDao {
+	private static final Log log = LogFactory
+			.getLog(ObjectPropertyStatementDaoSDB.class);

-    private static final Log log = LogFactory.getLog(ObjectPropertyStatementDaoSDB.class);
+	// Get the types of the base entity.
+	private static final String SUBJECT_TYPE_QUERY = ""
+			+ "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> \n"
+			+ "CONSTRUCT { \n" //
+			+ "   ?uri rdf:type ?type . \n" //
+			+ "} WHERE { \n" //
+			+ "   ?uri rdf:type ?type . \n" //
+			+ "} \n";

-	private DatasetWrapperFactory dwf;
-	private SDBDatasetMode datasetMode;
-	private WebappDaoFactorySDB wadf;
+	// Get the types of all objects of properties.
+	private static final String OBJECT_TYPE_QUERY = ""
+			+ "PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> \n"
+			+ "CONSTRUCT { \n" //
+			+ "   ?uri ?p ?o . \n" //
+			+ "   ?o rdf:type ?type . \n" //
+			+ "} WHERE { \n" //
+			+ "   ?uri ?p ?o . \n" //
+			+ "   ?o rdf:type ?type . \n" //
+			+ "} \n";

-	public ObjectPropertyStatementDaoSDB(
-	            RDFService rdfService,
-	            DatasetWrapperFactory dwf, 
-	            SDBDatasetMode datasetMode,
+	// Get the labels of all objects of properties.
+	private static final String OBJECT_LABEL_QUERY = ""
+			+ "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> \n"
+			+ "CONSTRUCT { \n" //
+			+ "   ?uri ?p ?o . \n" //
+			+ "   ?o rdfs:label ?label . \n" //
+			+ "} WHERE { \n" //
+			+ "   ?uri ?p ?o . \n" //
+			+ "   ?o rdfs:label ?label . \n" //
+			+ "} \n";
+
+	private final WebappDaoFactorySDB wadf;
+	private final SDBDatasetMode datasetMode;
+
+	public ObjectPropertyStatementDaoSDB(RDFService rdfService,
+			DatasetWrapperFactory dwf, SDBDatasetMode datasetMode,
 			WebappDaoFactorySDB wadf) {
-		super (rdfService, dwf, wadf);
-		this.dwf = dwf;
-		this.datasetMode = datasetMode;
+		super(rdfService, dwf, wadf);
 		this.wadf = wadf;
+		this.datasetMode = datasetMode;
 	}

 	@Override
 	public Individual fillExistingObjectPropertyStatements(Individual entity) {
-        if (entity.getURI() == null)
+		if (entity == null || entity.getURI() == null)
 			return entity;
 		else {
-        	Map<String, ObjectProperty> uriToObjectProperty = new HashMap<String,ObjectProperty>();
-        	String query = "CONSTRUCT { \n" +
-        			       "   <" + entity.getURI() + "> ?p ?o . \n" +
-//        			       "   ?o a ?oType . \n" +
-//        			       "   ?o <" + RDFS.label.getURI() + "> ?oLabel .  \n" +
-//        			       "   ?o <" + VitroVocabulary.MONIKER + "> ?oMoniker  \n" +
-        			       "} WHERE { \n" +
-        			       "   { <" + entity.getURI() + "> ?p ?o } \n" +
-//        			       "   UNION { <" + entity.getURI() + "> ?p ?o . ?o a ?oType } \n" +
-//        			       "   UNION { <" + entity.getURI() + "> ?p ?o . \n" +
-//        			       "           ?o <" + RDFS.label.getURI() + "> ?oLabel } \n" +
-//        			       "   UNION { <" + entity.getURI() + "> ?p ?o . \n " +
-//        			       "           ?o <" + VitroVocabulary.MONIKER + "> ?oMoniker } \n" +
-        			       "}";
-        	long startTime = System.currentTimeMillis();
-        	Model m = null;
-        	DatasetWrapper w = dwf.getDatasetWrapper();
-        	Dataset dataset = w.getDataset();
-        	dataset.getLock().enterCriticalSection(Lock.READ);
-        	QueryExecution qexec = null;
-        	try {
-        		qexec = QueryExecutionFactory.create(QueryFactory.create(query), dataset);
-        		m = qexec.execConstruct();
-        	} finally {
-        	    if(qexec != null) qexec.close();
-        		dataset.getLock().leaveCriticalSection();
-        		w.close();
-        	}
-        	if (log.isDebugEnabled()) {
-	        	log.debug("Time (ms) to query for related individuals: " + (System.currentTimeMillis() - startTime));
-	        	if (System.currentTimeMillis() - startTime > 1000) {
-	        		//log.debug(query);
-	        		log.debug("Results size (statements): " + m.size());
-	        	}
-        	}
+			List<ObjectPropertyStatement> objectPropertyStatements = new ArrayList<>();
+			String subjectUri = entity.getURI();

-        	OntModel ontModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, m);
-        	ontModel.enterCriticalSection(Lock.READ);
-        	try {
-	            Resource ind = ontModel.getResource(entity.getURI());
-	            List<ObjectPropertyStatement> objPropertyStmtList = new ArrayList<ObjectPropertyStatement>();
-	            ClosableIterator<Statement> propIt = ind.listProperties();
-	            try {
-	                while (propIt.hasNext()) {
-	                    Statement st = propIt.next();
-	                    if (st.getObject().isResource() && !(NONUSER_NAMESPACES.contains(st.getPredicate().getNameSpace()))) {
-	                        try {
-	                            ObjectPropertyStatement objPropertyStmt = new ObjectPropertyStatementImpl();
-	                            objPropertyStmt.setSubjectURI(entity.getURI());
-	                            objPropertyStmt.setSubject(entity);
-	                            objPropertyStmt.setObjectURI(((Resource)st.getObject()).getURI());
+			Model m = getInfoForObjectsOfThisEntity(subjectUri);

-	                            objPropertyStmt.setPropertyURI(st.getPredicate().getURI());
-                                Property prop = st.getPredicate();
-                                if( uriToObjectProperty.containsKey(prop.getURI())){
-                                	objPropertyStmt.setProperty(uriToObjectProperty.get(prop.getURI()));
-                                }else{
-                                	ObjectProperty p = getWebappDaoFactory().getObjectPropertyDao().getObjectPropertyByURI(prop.getURI());
-                                	if( p != null ){
-                                		uriToObjectProperty.put(prop.getURI(), p);
-                                		objPropertyStmt.setProperty(uriToObjectProperty.get(prop.getURI()));
-                                	}else{
-                                		//if ObjectProperty not found in ontology, skip it
+			Set<String> subjectTypes = getTypes(m, subjectUri);
+			for (ObjectPropertyPair pair : getRawObjectPropertyPairs(m,
+					subjectUri)) {
+				String predicateUri = pair.getPredicateUri();
+				String objectUri = pair.getObjectUri();
+				Set<String> objectTypes = getTypes(m, objectUri);
+
+				ObjectProperty prop = findRawProperty(predicateUri);
+				if (prop == null) {
 					continue;
 				}
-                                }                                

-	                            if (objPropertyStmt.getObjectURI() != null) {
-	                                //this might throw IndividualNotFoundException
-                                    Individual objInd = new IndividualSDB(
-                                        objPropertyStmt.getObjectURI(), 
-                                        this.dwf, 
-                                        datasetMode,
-                                        wadf);
-                                    objPropertyStmt.setObject(objInd);	                                
-	                            }
-	                            
-	                            //only add statement to list if it has its values filled out
-	                            if (    (objPropertyStmt.getSubjectURI() != null) 
-	                                 && (objPropertyStmt.getPropertyURI() != null) 
-	                                 && (objPropertyStmt.getObject() != null) ) {
-	                                objPropertyStmtList.add(objPropertyStmt);                           
-	                            } 
-	                            
-	                        } catch (IndividualNotFoundException t) {
-	                            log.debug(t,t);
-	                            continue;
-	                        } catch (Throwable t){
-	                            log.error(t,t);
-                                continue;
-	                        }
-	                    }
-	                }
-	            } finally {
-	                propIt.close();
-	            }
-	            entity.setObjectPropertyStatements(objPropertyStmtList);
-        	} finally {
-        		ontModel.leaveCriticalSection();
+				Individual object = new IndividualSDB(objectUri, dwf,
+						datasetMode, wadf, m);
+				objectPropertyStatements.add(createStatement(entity, prop,
+						object));
 			}
+			entity.setObjectPropertyStatements(objectPropertyStatements);
 			return entity;
 		}
 	}

+	/**
+	 * Get the types of this entity. Get the related object and the predicates
+	 * by which they are related. Get the types and labels of those related
+	 * objects.
+	 */
+	private Model getInfoForObjectsOfThisEntity(String subjectUri) {
+		Model m = ModelFactory.createDefaultModel();
+		try {
+			m.add(RDFServiceUtils.parseModel(
+					rdfService.sparqlConstructQuery(
+							substituteUri(subjectUri, SUBJECT_TYPE_QUERY), N3),
+					N3));
+			m.add(RDFServiceUtils.parseModel(
+					rdfService.sparqlConstructQuery(
+							substituteUri(subjectUri, OBJECT_TYPE_QUERY), N3),
+					N3));
+			m.add(RDFServiceUtils.parseModel(
+					rdfService.sparqlConstructQuery(
+							substituteUri(subjectUri, OBJECT_LABEL_QUERY), N3),
+					N3));
+		} catch (RDFServiceException e) {
+			log.warn("Failed to fill object property statements for '"
+					+ subjectUri + "'", e);
+		}
+		return m;
+	}
+
+	private String substituteUri(String uri, String query) {
+		return query.replace("?uri", "<" + uri + "> ");
+	}
+
+	private Set<String> getTypes(Model m, String uri) {
+		Set<String> typeUris = new HashSet<>();
+		for (RDFNode typeNode : m.listObjectsOfProperty(m.createResource(uri),
+				RDF.type).toSet()) {
+			if (typeNode.isURIResource()) {
+				typeUris.add(typeNode.asResource().getURI());
+			}
+		}
+		return typeUris;
+	}
+
+	private List<ObjectPropertyPair> getRawObjectPropertyPairs(Model m,
+			String subjectUri) {
+		List<ObjectPropertyPair> list = new ArrayList<>();
+		for (Statement stmt : m.listStatements(m.createResource(subjectUri),
+				null, (RDFNode) null).toList()) {
+			if (wadf.getNonuserNamespaces().contains(
+					stmt.getPredicate().getNameSpace())) {
+				continue;
+			}
+			if (!stmt.getObject().isURIResource()) {
+				continue;
+			}
+			list.add(new ObjectPropertyPair(stmt.getPredicate().getURI(), stmt
+					.getObject().asResource().getURI()));
+		}
+		return list;
+	}
+
+	private ObjectProperty findRawProperty(String predicateUri) {
+		return wadf.getObjectPropertyDao().getObjectPropertyByURI(predicateUri);
+	}
+
+	private ObjectPropertyStatement createStatement(Individual entity,
+			ObjectProperty prop, Individual object) {
+		ObjectPropertyStatementImpl ops = new ObjectPropertyStatementImpl();
+		ops.setSubject(entity);
+		ops.setProperty(prop);
+		ops.setObject(object);
+		return ops;
+	}
+
+	// ----------------------------------------------------------------------
+	// Helper classes
+	// ----------------------------------------------------------------------
+
+	private static class ObjectPropertyPair {
+		private final String predicateUri;
+		private final String objectUri;
+
+		public ObjectPropertyPair(String predicateUri, String objectUri) {
+			this.predicateUri = predicateUri;
+			this.objectUri = objectUri;
+		}
+
+		public String getPredicateUri() {
+			return predicateUri;
+		}
+
+		public String getObjectUri() {
+			return objectUri;
+		}
+
+	}
+
 }
--- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java
+++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateStatementsTask.java
@ -77,7 +77,7 @@ public class UpdateStatementsTask implements Task {

 		this.uris = Collections.synchronizedSet(new HashSet<String>());

-		this.status = new Status(changes.size(), 200, listeners);
+		this.status = new Status(changes.size(), 500, listeners);
 	}

 	@Override
--- a/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java
+++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/tasks/UpdateUrisTask.java
@ -66,7 +66,7 @@ public class UpdateUrisTask implements Task {
 		this.listeners = listeners;
 		this.pool = pool;

-		this.status = new Status(uris.size(), 200, listeners);
+		this.status = new Status(uris.size(), 500, listeners);

 		this.searchEngine = ApplicationUtils.instance().getSearchEngine();
 		
--- a/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl
+++ b/webapp/web/templates/freemarker/body/admin/admin-showThreads.ftl
@ -20,12 +20,19 @@ table.threadInfo th {
 <h2>${i18n().background_threads}</h2>

 <section id="show-threads" role="region">
+    <table class="threadInfo" summary="Status of background threads.">
+        <tr>
+            <th>${i18n().name}</th>
+            <th>${i18n().work_level}</th>
+            <th>${i18n().since}</th>
+            <th>${i18n().flags}</th>
+        </tr>
        <#list threads as threadInfo>
-        <table class="threadInfo ${threadInfo.workLevel}" summary="Thread ${threadInfo.name}">
-            <tr><th>${i18n().name}</th><td>${threadInfo.name}</td></tr>
-            <tr><th>${i18n().work_level}</th><td>${threadInfo.workLevel}</td></tr>
-            <tr><th>${i18n().since}</th><td>${threadInfo.since}</td></tr>
-            <tr><th>${i18n().flags}</th><td>${threadInfo.flags}</td></tr>
-        </table>
+            <tr>
+                <td>${threadInfo.name}</td>
+                <td>${threadInfo.workLevel}</td>
+                <td>${threadInfo.since}</td>
+                <td>${threadInfo.flags}</td>
        </#list>
+    </table>
 </section>