various migration issues and VIVO-265 blank node deletion optimization

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/ABoxUpdater.java

@@ -420,7 +420,7 @@ public class ABoxUpdater {
 		Iterator<AtomicOntologyChange> propItr = changes.iterator();
 		while(propItr.hasNext()){
 			AtomicOntologyChange propChangeObj = propItr.next();
-			log.info("processing " + propChangeObj);
+			log.debug("processing " + propChangeObj);
 			try {
 			    if (propChangeObj.getAtomicChangeType() == null) {
 			        log.error("Missing change type; skipping " + propChangeObj);
@@ -428,19 +428,19 @@ public class ABoxUpdater {
 			    }
     			switch (propChangeObj.getAtomicChangeType()){
     			  case ADD: 
-    			   log.info("add");
+    			   log.debug("add");
     			   addProperty(propChangeObj);
     			   break;
     			case DELETE: 
-    			   log.info("delete");
+    			   log.debug("delete");
     			   deleteProperty(propChangeObj);
     			   break;
     			case RENAME: 
-    			   log.info("rename");
+    			   log.debug("rename");
     			   renameProperty(propChangeObj);
     			   break;
     			default: 
-    			   log.info("unknown");
+    			   log.debug("unknown");
     			   logger.logError("unexpected change type indicator: " + propChangeObj.getAtomicChangeType());
     			   break;
     		    }	

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java

@@ -96,7 +96,7 @@ public class KnowledgeBaseUpdater {
 		
         //process the TBox before the ABox
 		try {
-		    log.info("\tupdating tbox annotations");
+		    log.debug("\tupdating tbox annotations");
 	        updateTBoxAnnotations();
 		} catch (Exception e) {
 		    log.error(e,e);
@@ -396,13 +396,13 @@ public class KnowledgeBaseUpdater {
 			while(listItr.hasNext()) {
 				AtomicOntologyChange changeObj = listItr.next();
 				if (changeObj.getSourceURI() != null){
-			        log.info("triaging " + changeObj);
+			        log.debug("triaging " + changeObj);
 					if (oldTboxModel.getOntProperty(changeObj.getSourceURI()) != null){
 						 atomicPropertyChanges.add(changeObj);
-						 log.info("added to property changes");
+						 log.debug("added to property changes");
 					} else if (oldTboxModel.getOntClass(changeObj.getSourceURI()) != null) {
 						 atomicClassChanges.add(changeObj);
-						 log.info("added to class changes");
+						 log.debug("added to class changes");
 					} else if ("Prop".equals(changeObj.getNotes())) {
 						 atomicPropertyChanges.add(changeObj);
 					} else if ("Class".equals(changeObj.getNotes())) {

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/TBoxUpdater.java

@@ -454,20 +454,23 @@ public class TBoxUpdater {
     }
     
     public void renameProperty(AtomicOntologyChange changeObj) throws IOException {
+        Dataset dataset = new RDFServiceDataset(settings.getRDFService());
+        Model userAnnotationsModel = dataset.getNamedModel(
+                JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
         if(changeObj.getNotes() != null && changeObj.getNotes().startsWith("cc:")) {
-            mergePropertyAnnotationsToPropertyConfig(changeObj);
+            mergePropertyAnnotationsToPropertyConfig(changeObj, userAnnotationsModel);
         }        
-        
+        Resource renamedProperty = userAnnotationsModel.getResource(changeObj.getSourceURI());
+        userAnnotationsModel.removeAll(renamedProperty, null, (RDFNode) null);
+        userAnnotationsModel.removeAll(null, null, renamedProperty);
     }
     
-    private void mergePropertyAnnotationsToPropertyConfig(AtomicOntologyChange changeObj) throws IOException {
+    private void mergePropertyAnnotationsToPropertyConfig(AtomicOntologyChange changeObj,
+            Model userAnnotationsModel) throws IOException {
         String contextURI = VitroVocabulary.PROPERTY_CONFIG_DATA + changeObj.getNotes().substring(3);
         String oldPropertyURI = changeObj.getSourceURI();
         
         Model oldAnnotationsModel = settings.getOldTBoxAnnotationsModel();
-        Dataset dataset = new RDFServiceDataset(settings.getRDFService());
-        Model userAnnotationsModel = dataset.getNamedModel(
-                JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
         
         String propertyAnnotationsQuery = 
                 "PREFIX config: <" + VitroVocabulary.configURI + "> \n" +

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java

@@ -6,13 +6,16 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
+import com.hp.hpl.jena.graph.Node;
 import com.hp.hpl.jena.graph.Triple;
 import com.hp.hpl.jena.query.DataSource;
 import com.hp.hpl.jena.query.Dataset;
@@ -21,11 +24,9 @@ import com.hp.hpl.jena.query.Query;
 import com.hp.hpl.jena.query.QueryExecution;
 import com.hp.hpl.jena.query.QueryExecutionFactory;
 import com.hp.hpl.jena.query.QueryFactory;
-import com.hp.hpl.jena.query.QueryParseException;
 import com.hp.hpl.jena.query.QuerySolution;
 import com.hp.hpl.jena.query.ResultSet;
 import com.hp.hpl.jena.query.ResultSetFormatter;
-import com.hp.hpl.jena.query.Syntax;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
 import com.hp.hpl.jena.rdf.model.RDFNode;
@@ -129,7 +130,7 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
                                 Resource s2 = (Resource) n;
                                 // now run yet another describe query
                                 String smallerTree = makeDescribe(s2);
-                                log.info(smallerTree);
+                                log.debug(smallerTree);
                                 Query smallerTreeQuery = QueryFactory.create(smallerTree);
                                 QueryExecution qe3 = QueryExecutionFactory.create(
                                         smallerTreeQuery, tree);
@@ -173,19 +174,23 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         
         StringBuffer queryBuff = new StringBuffer();
         queryBuff.append("CONSTRUCT { \n");
-        addStatementPatterns(stmtIt, queryBuff, !WHERE_CLAUSE);
+        List<Statement> stmts = stmtIt.toList();
+        stmts = sort(stmts);
+        addStatementPatterns(stmts, queryBuff, !WHERE_CLAUSE);
         queryBuff.append("} WHERE { \n");
         if (graphURI != null) {
             queryBuff.append("    GRAPH <" + graphURI + "> { \n");
         }
         stmtIt = model.listStatements();
-        addStatementPatterns(stmtIt, queryBuff, WHERE_CLAUSE);
+        stmts = stmtIt.toList();
+        stmts = sort(stmts);
+        addStatementPatterns(stmts, queryBuff, WHERE_CLAUSE);
         if (graphURI != null) {
             queryBuff.append("    } \n");
         }
         queryBuff.append("} \n");
         
-        log.info(queryBuff.toString());
+        log.debug(queryBuff.toString());
         
         Query construct = QueryFactory.create(queryBuff.toString());
         // make a plain dataset to force the query to be run in a way that
@@ -209,11 +214,61 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         }       
     }
     
+    private List<Statement> sort(List<Statement> stmts) {
+        List<Statement> output = new ArrayList<Statement>();
+        int originalSize = stmts.size();
+        List <Statement> remaining = stmts;
+        ConcurrentLinkedQueue<Resource> subjQueue = new ConcurrentLinkedQueue<Resource>();
+        for(Statement stmt : remaining) {
+            if(stmt.getSubject().isURIResource()) {
+                subjQueue.add(stmt.getSubject());
+                break;
+            }
+        }
+        if (subjQueue.isEmpty()) {
+            throw new RuntimeException("No named subject in statement patterns");
+        }
+        while(remaining.size() > 0) {
+            if(subjQueue.isEmpty()) {
+                subjQueue.add(remaining.get(0).getSubject());
+            }
+            while(!subjQueue.isEmpty()) {
+                Resource subj = subjQueue.poll();
+                List<Statement> temp = new ArrayList<Statement>();
+                for (Statement stmt : remaining) {
+                    if(stmt.getSubject().equals(subj)) {
+                        output.add(stmt);
+                        if (stmt.getObject().isResource()) {
+                            subjQueue.add((Resource) stmt.getObject()); 
+                        }
+                    } else {
+                        temp.add(stmt);
+                    }
+                }
+                remaining = temp;
+            }
+        }
+        if(output.size() != originalSize) {
+            throw new RuntimeException("original list size was " + originalSize + 
+                    " but sorted size is " + output.size());      
+        }
+        return output;
+    }
+    
+    private String getHexString(Node node) {
+        String label = node.getBlankNodeLabel().replaceAll("\\W", "").toUpperCase();
+        if (label.length() > 7) {
+            return label.substring(label.length() - 7);
+        } else {
+            return label;
+        }
+    }
+    
     private static final boolean WHERE_CLAUSE = true;
     
-    private void addStatementPatterns(StmtIterator stmtIt, StringBuffer patternBuff, boolean whereClause) {
+    private void addStatementPatterns(List<Statement> stmts, StringBuffer patternBuff, boolean whereClause) {
-        while(stmtIt.hasNext()) {
+        for(Statement stmt : stmts) {
-            Triple t = stmtIt.next().asTriple();
+            Triple t = stmt.asTriple();
             patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));
             patternBuff.append(" ");
             patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getPredicate(), null));

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java

@@ -9,6 +9,7 @@ import java.io.StringWriter;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 import org.apache.commons.httpclient.HttpClient;
 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
@@ -715,7 +716,9 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
         if (graphURI != null) {
             queryBuff.append("    GRAPH <" + graphURI + "> { \n");
         }
-        addStatementPatterns(stmtIt, queryBuff, !WHERE_CLAUSE);
+        List<Statement> stmts = stmtIt.toList();
+        sort(stmts);
+        addStatementPatterns(stmts, queryBuff, !WHERE_CLAUSE);
         if (graphURI != null) {
             queryBuff.append("    } \n");
         }
@@ -724,7 +727,9 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
             queryBuff.append("    GRAPH <" + graphURI + "> { \n");
         }
         stmtIt = model.listStatements();
-        addStatementPatterns(stmtIt, queryBuff, WHERE_CLAUSE);
+        stmts = stmtIt.toList();
+        sort(stmts);
+        addStatementPatterns(stmts, queryBuff, WHERE_CLAUSE);
         if (graphURI != null) {
             queryBuff.append("    } \n");
         }
@@ -736,11 +741,53 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
         executeUpdate(queryBuff.toString());
     }
     
+    private List<Statement> sort(List<Statement> stmts) {
+        List<Statement> output = new ArrayList<Statement>();
+        int originalSize = stmts.size();
+        List <Statement> remaining = stmts;
+        ConcurrentLinkedQueue<com.hp.hpl.jena.rdf.model.Resource> subjQueue = 
+                new ConcurrentLinkedQueue<com.hp.hpl.jena.rdf.model.Resource>();
+        for(Statement stmt : remaining) {
+            if(stmt.getSubject().isURIResource()) {
+                subjQueue.add(stmt.getSubject());
+                break;
+            }
+        }
+        if (subjQueue.isEmpty()) {
+            throw new RuntimeException("No named subject in statement patterns");
+        }
+        while(remaining.size() > 0) {
+            if(subjQueue.isEmpty()) {
+                subjQueue.add(remaining.get(0).getSubject());
+            }
+            while(!subjQueue.isEmpty()) {
+                com.hp.hpl.jena.rdf.model.Resource subj = subjQueue.poll();
+                List<Statement> temp = new ArrayList<Statement>();
+                for (Statement stmt : remaining) {
+                    if(stmt.getSubject().equals(subj)) {
+                        output.add(stmt);
+                        if (stmt.getObject().isResource()) {
+                            subjQueue.add((com.hp.hpl.jena.rdf.model.Resource) stmt.getObject()); 
+                        }
+                    } else {
+                        temp.add(stmt);
+                    }
+                }
+                remaining = temp;
+            }
+        }
+        if(output.size() != originalSize) {
+            throw new RuntimeException("original list size was " + originalSize + 
+                    " but sorted size is " + output.size());      
+        }
+        return output;
+    }
+    
     private static final boolean WHERE_CLAUSE = true;
     
-    private void addStatementPatterns(StmtIterator stmtIt, StringBuffer patternBuff, boolean whereClause) {
+    private void addStatementPatterns(List<Statement> stmts, StringBuffer patternBuff, boolean whereClause) {
-        while(stmtIt.hasNext()) {
+        for(Statement stmt : stmts) {
-            Triple t = stmtIt.next().asTriple();
+            Triple t = stmt.asTriple();
             patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));
             patternBuff.append(" ");
             patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getPredicate(), null));

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java

@@ -122,7 +122,10 @@ public class UpdateKnowledgeBase implements ServletContextListener {
 				settings.setLoadedAtStartupDisplayModel(loadedAtStartupFiles);
 				OntModel oldDisplayModelVivoListView = loadModelFromFile(ctx.getRealPath(OLD_DISPLAYMODEL_VIVOLISTVIEW_PATH));
 				settings.setVivoListViewConfigDisplayModel(oldDisplayModelVivoListView);
-			} catch (Exception e) {
+			} catch (ModelFileNotFoundException e) {
+                // expected if no display migration was intended
+                tryMigrateDisplay = false;
+            } catch (Exception e) {
 				log.info("Unable to read display model migration files. ", e);
 				tryMigrateDisplay = false;
 			}