Model operations performance improvements (#347)

* created RDFServiceBulkUnionUpdater

* fix and indents

* use bulk updater for RDFServiceGraph in blankNodeFilteringGraph

* use removeAll to optimize removal all triples from TDB modeles

* avoid additional serialization/deserialization cycle of removing model chunk

* fixed VitroModelFactory tests

* fixes for BulkUpdating models

* Created custom ModelCom, OntModelImpl, BulkGraphMem to avoid triple by triple transactions on add/remove model
Load n3 files from home directory into in-memory model to use bulk
loading.

* refact: simplified BulkGraphMem checks

* removed unused import

api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/BlankNodeFilteringGraph.java

@@ -24,6 +24,10 @@ public class BlankNodeFilteringGraph implements Graph {
 
     private Graph graph;
 
+	public Graph getInnerGraph() {
+		return graph;
+	}
+
 	public BlankNodeFilteringGraph(Graph graph) {
         this.graph = graph;
     }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceModelMaker.java

@@ -103,7 +103,7 @@ public class RDFServiceModelMaker implements ModelMaker {
 	@Override
 	public void removeModel(String name) {
 		Model m = getModel(name);
-		m.removeAll(null, null, null);
+		m.removeAll();
 	}
 
 	@Override

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkGraphMem.java

@@ -0,0 +1,17 @@
+package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
+
+import org.apache.jena.graph.Triple;
+import org.apache.jena.mem.GraphMem;
+
+public class BulkGraphMem extends GraphMem {
+	
+	public void addWithoutNotify(Triple t) {
+		checkOpen();
+		performAdd(t);
+	}
+
+	public final void deleteWithoutNotify(Triple t) {
+		checkOpen();
+		performDelete(t);
+	}
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkModelCom.java

@@ -0,0 +1,34 @@
+package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
+
+import org.apache.jena.graph.Graph;
+import org.apache.jena.rdf.model.Model;
+import org.apache.jena.rdf.model.impl.ModelCom;
+
+public class BulkModelCom extends ModelCom {
+
+	public BulkModelCom(Graph graph) {
+		super(graph);
+	}
+
+	@Override
+	public Model remove(Model m) {
+		Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph);
+		if (unwrappedGraph instanceof BulkGraphMem) {
+			GraphUtils.deleteFrom((BulkGraphMem) unwrappedGraph, m.getGraph());	
+		} else {
+			super.remove(m);
+		}
+		return this;
+	}
+
+	@Override
+	public Model add(Model m) {
+		Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph);
+		if (unwrappedGraph instanceof BulkGraphMem) {
+			GraphUtils.addInto((BulkGraphMem) unwrappedGraph, m.getGraph());
+		} else {
+			super.add(m);
+		}
+		return this;
+	}
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkOntModelImpl.java

@@ -0,0 +1,39 @@
+package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
+
+import org.apache.jena.graph.Graph;
+import org.apache.jena.ontology.OntModelSpec;
+import org.apache.jena.ontology.impl.OntModelImpl;
+import org.apache.jena.rdf.model.Model;
+
+public class BulkOntModelImpl extends OntModelImpl {
+
+	public BulkOntModelImpl(OntModelSpec spec) {
+		super(spec);
+	}
+	
+	public BulkOntModelImpl(OntModelSpec owlMem, Model bareModel) {
+		super(owlMem, bareModel);
+	}
+
+	@Override
+	public Model remove(Model m) {
+		Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph);
+		if (unwrappedGraph instanceof BulkGraphMem) {
+			GraphUtils.deleteFrom((BulkGraphMem) unwrappedGraph, m.getGraph());	
+		} else {
+			super.remove(m);
+		}
+		return this;
+	}
+
+	@Override
+	public Model add(Model m) {
+		Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph);
+		if (unwrappedGraph instanceof BulkGraphMem) {
+			GraphUtils.addInto((BulkGraphMem) unwrappedGraph, m.getGraph());
+		} else {
+			super.add(m);
+		}
+		return this;
+	}
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingModel.java

@@ -4,6 +4,8 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
 
 import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion;
+
 import org.apache.jena.graph.Graph;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
@@ -22,30 +24,20 @@ import java.util.List;
 
 public class BulkUpdatingModel extends AbstractModelDecorator {
     private static final RDFReaderF readerFactory = new RDFReaderFImpl();
-    private AbstractBulkUpdater updater;
+    protected AbstractBulkUpdater updater;
 
-    protected BulkUpdatingModel(Model m) {
+    public BulkUpdatingModel(Model m) {
         super(m);
-        if (m instanceof BulkUpdatingModel) {
+        Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph());
+        if(graph instanceof BulkUpdatingUnion){
+            updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph);
+            return;
+        } 
+        if (m instanceof BulkUpdatingOntModel) {
+            this.updater = ((BulkUpdatingOntModel) m).updater;
+        } else  if (m instanceof BulkUpdatingModel) {
             this.updater = ((BulkUpdatingModel) m).updater;
         } else {
-            Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph());
-            if (graph instanceof RDFServiceGraph) {
-                updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph);
-            } else if (graph instanceof SparqlGraph) {
-                updater = new SparqlBulkUpdater((SparqlGraph) graph);
-            } else {
-                updater = null;
-            }
-        }
-    }
-
-    protected BulkUpdatingModel(Model m, Model baseModel) {
-        super(m);
-        if (baseModel instanceof BulkUpdatingModel) {
-            this.updater = ((BulkUpdatingModel) baseModel).updater;
-        } else {
-            Graph graph = GraphUtils.unwrapUnionGraphs(baseModel.getGraph());
             if (graph instanceof RDFServiceGraph) {
                 updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph);
             } else if (graph instanceof SparqlGraph) {

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingOntModel.java

@@ -4,6 +4,8 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
 
 import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion;
+
 import org.apache.jena.graph.Graph;
 import org.apache.jena.ontology.OntModel;
 import org.apache.jena.rdf.model.Model;
@@ -23,30 +25,20 @@ import java.util.List;
 
 public class BulkUpdatingOntModel extends AbstractOntModelDecorator {
     private static final RDFReaderF readerFactory = new RDFReaderFImpl();
-    private AbstractBulkUpdater updater;
+    protected AbstractBulkUpdater updater;
 
     protected BulkUpdatingOntModel(OntModel m) {
         super(m);
+        Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph());
+        if(graph instanceof BulkUpdatingUnion){
+            updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph);
+            return;
+        } 
         if (m instanceof BulkUpdatingOntModel) {
             this.updater = ((BulkUpdatingOntModel) m).updater;
+        } else  if (m instanceof BulkUpdatingModel) {
+            this.updater = ((BulkUpdatingModel) m).updater;
         } else {
-            Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph());
-            if (graph instanceof RDFServiceGraph) {
-                updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph);
-            } else if (graph instanceof SparqlGraph) {
-                updater = new SparqlBulkUpdater((SparqlGraph) graph);
-            } else {
-                updater = null;
-            }
-        }
-    }
-
-    protected BulkUpdatingOntModel(OntModel m, OntModel baseModel) {
-        super(m);
-        if (baseModel instanceof BulkUpdatingOntModel) {
-            this.updater = ((BulkUpdatingOntModel) baseModel).updater;
-        } else {
-            Graph graph = GraphUtils.unwrapUnionGraphs(baseModel.getGraph());
             if (graph instanceof RDFServiceGraph) {
                 updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph);
             } else if (graph instanceof SparqlGraph) {

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/GraphUtils.java

@@ -2,20 +2,128 @@
 
 package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
 
+import org.apache.jena.atlas.iterator.Iter;
 import org.apache.jena.graph.Graph;
+import org.apache.jena.graph.Triple;
 import org.apache.jena.graph.compose.MultiUnion;
+import org.apache.jena.util.IteratorCollection;
+import org.apache.jena.util.iterator.ExtendedIterator;
 
+import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 
 final public class GraphUtils {
-    public static Graph unwrapUnionGraphs(Graph graph) {
-        if (graph != null && graph instanceof MultiUnion) {
-            List<Graph> subGraphs = ((MultiUnion)graph).getSubGraphs();
-            if (subGraphs == null || subGraphs.isEmpty()) {
-                return ((MultiUnion)graph).getBaseGraph();
-            }
-        }
 
-        return graph;
-    }
+	private static final int CMP_GREATER = 1;
+	private static final int CMP_EQUAL = 0;
+	private static final int CMP_LESS = -1;
+	private static int MIN_SRC_SIZE = 1000;
+	// If source and destination are large, limit the search for the best way round
+	// to "deleteFrom"
+	private static int DST_SRC_RATIO = 2;
+
+	public static Graph unwrapUnionGraphs(Graph graph) {
+		if (graph != null && graph instanceof MultiUnion) {
+			List<Graph> subGraphs = ((MultiUnion) graph).getSubGraphs();
+			if (subGraphs == null || subGraphs.isEmpty()) {
+				return ((MultiUnion) graph).getBaseGraph();
+			}
+		}
+		return graph;
+	}
+
+	public static void deleteFrom(BulkGraphMem bulkGraphMem, Graph srcGraph) {
+		boolean events = bulkGraphMem.getEventManager().listening();
+		if (bulkGraphMem == srcGraph && !events) {
+			bulkGraphMem.clear();
+			return;
+		}
+		boolean loopOnSrc = decideHowtoExecuteBySizeStep(bulkGraphMem, srcGraph);
+		if (loopOnSrc) {
+			deleteLoopSrc(bulkGraphMem, srcGraph);
+			return;
+		}
+		deleteLoopDst(bulkGraphMem, srcGraph);
+	}
+
+	public static void addInto(BulkGraphMem bulkGraphMem, Graph srcGraph) {
+		if (bulkGraphMem == srcGraph && !bulkGraphMem.getEventManager().listening()) {
+			return;
+		}
+		bulkGraphMem.getPrefixMapping().setNsPrefixes(srcGraph.getPrefixMapping());
+		addIteratorWorker(bulkGraphMem, findAll(srcGraph));
+		bulkGraphMem.getEventManager().notifyAddGraph(bulkGraphMem, srcGraph);
+	}
+
+	private static ExtendedIterator<Triple> findAll(Graph g) {
+		return g.find();
+	}
+
+	private static void addIteratorWorker(BulkGraphMem bulkGraphMem, Iterator<Triple> it) {
+		List<Triple> s = IteratorCollection.iteratorToList(it);
+		addIteratorWorkerDirect(bulkGraphMem, s.iterator());
+	}
+
+	private static void addIteratorWorkerDirect(BulkGraphMem bulkGraphMem, Iterator<Triple> it) {
+		it.forEachRemaining(bulkGraphMem::addWithoutNotify);
+	}
+
+	private static void deleteLoopSrc(BulkGraphMem bulkGraphMem, Graph srcGraph) {
+		deleteIteratorWorker(bulkGraphMem, findAll(srcGraph));
+		bulkGraphMem.getEventManager().notifyDeleteGraph(bulkGraphMem, srcGraph);
+	}
+
+	private static void deleteLoopDst(BulkGraphMem bulkGraphMem, Graph srcGraph) {
+		// Size the list to avoid reallocation on growth.
+		int dstSize = bulkGraphMem.size();
+		List<Triple> toBeDeleted = new ArrayList<>(dstSize);
+
+		Iterator<Triple> iter = findAll(bulkGraphMem);
+		for (; iter.hasNext();) {
+			Triple t = iter.next();
+			if (srcGraph.contains(t)) {
+				toBeDeleted.add(t);
+			}
+		}
+		deleteIteratorWorkerDirect(bulkGraphMem, toBeDeleted.iterator());
+		bulkGraphMem.getEventManager().notifyDeleteGraph(bulkGraphMem, srcGraph);
+	}
+
+	private static void deleteIteratorWorker(BulkGraphMem bulkGraphMem, Iterator<Triple> it) {
+		List<Triple> s = IteratorCollection.iteratorToList(it);
+		deleteIteratorWorkerDirect(bulkGraphMem, s.iterator());
+	}
+
+	private static void deleteIteratorWorkerDirect(BulkGraphMem bulkGraphMem, Iterator<Triple> it) {
+		it.forEachRemaining(bulkGraphMem::deleteWithoutNotify);
+	}
+
+	private static boolean decideHowtoExecuteBySizeStep(BulkGraphMem bulkGraphMem, Graph srcGraph) {
+		int srcSize = srcGraph.size();
+		if (srcSize <= MIN_SRC_SIZE)
+			return true;
+		boolean loopOnSrc = (srcSize <= MIN_SRC_SIZE
+				|| compareSizeTo(bulkGraphMem, DST_SRC_RATIO * srcSize) == CMP_GREATER);
+		return loopOnSrc;
+	}
+
+	private static int compareSizeTo(Graph graph, int size) {
+		ExtendedIterator<Triple> it = graph.find();
+		try {
+			int stepsTake = Iter.step(it, size);
+			if (stepsTake < size) {
+				// Iterator ran out.
+				return CMP_LESS;
+			}
+			if (!it.hasNext()) {
+				// Finished at the same time.
+				return CMP_EQUAL;
+			}
+			// Still more to go
+			return CMP_GREATER;
+		} finally {
+			it.close();
+		}
+	}
 }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUnionUpdater.java

@@ -0,0 +1,118 @@
+/* $This file is distributed under the terms of the license in LICENSE$ */
+
+package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
+
+import edu.cornell.mannlib.vitro.webapp.dao.jena.BlankNodeFilteringGraph;
+import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph;
+import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion;
+
+import org.apache.jena.graph.Graph;
+import org.apache.jena.rdf.model.Model;
+
+public class RDFServiceBulkUnionUpdater extends AbstractBulkUpdater {
+
+	private AbstractBulkUpdater leftUpdater;
+	private AbstractBulkUpdater rightUpdater;
+	private Model baseModel;
+	private Model plusModel;
+
+	public RDFServiceBulkUnionUpdater(Model baseModel, Model plusModel) {
+		this.baseModel = baseModel;
+		this.plusModel = plusModel;
+		if (baseModel != null) {
+			leftUpdater = getUpdater(baseModel);
+		}
+		if (plusModel != null) {
+			rightUpdater = getUpdater(plusModel);
+		}
+	}
+	
+	public RDFServiceBulkUnionUpdater(BulkUpdatingUnion union) {
+		baseModel = union.getBaseModel();
+		if (baseModel != null) {
+			leftUpdater = getUpdater(baseModel);
+		}
+		plusModel = union.getPlusModel();
+		if (plusModel != null) {
+			rightUpdater = getUpdater(plusModel);
+		}
+	}
+	
+	private AbstractBulkUpdater getUpdater(Model model) {
+		AbstractBulkUpdater updater = null;
+		if (model instanceof BulkUpdatingOntModel) {
+            updater = ((BulkUpdatingOntModel) model).updater;
+            return updater;
+        } else if (model instanceof BulkUpdatingModel) {
+            updater = ((BulkUpdatingModel) model).updater;
+            return updater;
+        } 
+	    Graph graph = GraphUtils.unwrapUnionGraphs(model.getGraph());
+		if (graph instanceof BulkUpdatingUnion) {
+			updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph);
+		} else if (graph instanceof RDFServiceGraph) {
+			updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph);
+		} else if (hasBlankNodeFilterginGraphInnerRdfServiceGraph(graph)) {
+			BlankNodeFilteringGraph blankNodeGraph = (BlankNodeFilteringGraph) graph;
+			RDFServiceGraph rdfServiceGraph = (RDFServiceGraph)blankNodeGraph.getInnerGraph();
+			updater = new RDFServiceBulkUpdater(rdfServiceGraph);
+		} else if (graph instanceof BulkUpdatingUnion) {
+			updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph);
+		} else if (graph instanceof SparqlGraph) {
+			updater = new SparqlBulkUpdater((SparqlGraph) graph);
+		} else {
+			updater = null;
+		}
+		return updater;
+	}
+
+	private boolean hasBlankNodeFilterginGraphInnerRdfServiceGraph(Graph graph) {
+		if (!(graph instanceof BlankNodeFilteringGraph)) {
+			return false;
+		}
+		BlankNodeFilteringGraph blankNodeGraph = (BlankNodeFilteringGraph) graph;
+		final Graph innerGraph = blankNodeGraph.getInnerGraph();
+		if (innerGraph instanceof RDFServiceGraph) {
+			return true;
+		}
+		return false;
+	}
+
+	@Override
+	protected void performAddModel(Model model) {
+		if (leftUpdater != null) {
+			leftUpdater.performAddModel(model);
+		} else if (baseModel != null) {
+			baseModel.add(model);
+		}
+	}
+
+	@Override
+	protected void performRemoveModel(Model model) {
+		if (leftUpdater != null) {
+			leftUpdater.performRemoveModel(model);
+		} else if (baseModel != null) {
+			baseModel.remove(model);
+		}
+		if (rightUpdater != null) {
+			rightUpdater.performRemoveModel(model);
+		} else if (plusModel != null) {
+			plusModel.remove(model);
+		}
+	}
+
+	@Override
+	protected void performRemoveAll() {
+		if (leftUpdater != null) {
+			leftUpdater.performRemoveAll();
+		} else if (baseModel != null) {
+			baseModel.removeAll();
+		}
+		if (rightUpdater != null) {
+			rightUpdater.performRemoveAll();
+		} else if (plusModel != null) {
+			plusModel.removeAll();
+		}
+	}
+}

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUpdater.java

@@ -6,15 +6,20 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
-import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 import org.apache.jena.graph.GraphEvents;
 import org.apache.jena.rdf.model.Model;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
 
 public class RDFServiceBulkUpdater extends AbstractBulkUpdater {
     RDFServiceGraph graph;
+	private static final int chunkSize = 50000;
+    private static final String REMOVE_CHUNK_QUERY = 
+			"CONSTRUCT { ?s ?p ?o } " +
+			"WHERE { ?s ?p ?o } " +
+			"LIMIT " + chunkSize;
 
     public RDFServiceBulkUpdater(RDFServiceGraph graph) {
         this.graph = graph;
@@ -53,49 +58,35 @@ public class RDFServiceBulkUpdater extends AbstractBulkUpdater {
     @Override
     protected void performRemoveAll() {
         String graphURI = graph.getGraphURI();
-
-        String findPattern = "?s ?p ?o";
-
-        StringBuilder findQuery = new StringBuilder("CONSTRUCT { ")
-                .append(findPattern)
-                .append(" } WHERE { \n");
+        String queryString;
         if (graphURI != null) {
-            findQuery.append("  GRAPH <").append(graphURI).append("> { ");
+        	queryString = getRemoveGraphChunkPattern(graphURI);
+        } else {
+        	queryString = REMOVE_CHUNK_QUERY;	
         }
-        findQuery.append(findPattern);
-        if (graphURI != null) {
-            findQuery.append(" } ");
-        }
-        findQuery.append("\n}");
-
-        String queryString = findQuery.toString();
-
-        int chunkSize = 50000;
-        boolean done = false;
-
-        while (!done) {
-            String chunkQueryString = queryString + " LIMIT " + chunkSize;
-
+        while (!graph.isEmpty()) {
             try {
-                Model chunkToRemove = RDFServiceUtils.parseModel(
-                        graph.getRDFService().sparqlConstructQuery(
-                                chunkQueryString, RDFService.ModelSerializationFormat.N3),
-                        RDFService.ModelSerializationFormat.N3);
-                if (chunkToRemove.size() > 0) {
-                    ChangeSet cs = graph.getRDFService().manufactureChangeSet();
-                    ByteArrayOutputStream out = new ByteArrayOutputStream();
-                    chunkToRemove.write(out, "N-TRIPLE");
-                    cs.addRemoval(new ByteArrayInputStream(out.toByteArray()),
-                            RDFService.ModelSerializationFormat.N3, graphURI);
-                    graph.getRDFService().changeSetUpdate(cs);
-                } else {
-                    done = true;
-                }
+                InputStream chunkToRemove = graph.getRDFService().sparqlConstructQuery(
+                queryString, RDFService.ModelSerializationFormat.N3);
+                ChangeSet cs = graph.getRDFService().manufactureChangeSet();
+                ByteArrayOutputStream out = new ByteArrayOutputStream();
+                cs.addRemoval(chunkToRemove, RDFService.ModelSerializationFormat.N3, graphURI);
+                graph.getRDFService().changeSetUpdate(cs);
             } catch (RDFServiceException e) {
                 throw new RuntimeException(e);
             }
         }
-
         graph.getEventManager().notifyEvent(graph, GraphEvents.removeAll);
     }
+    
+    private static String getRemoveGraphChunkPattern(String uri) {
+    	return 
+    			"CONSTRUCT { ?s ?p ?o } " +
+    			"WHERE { " +
+    				"GRAPH <" + uri + "> {" +
+    					"?s ?p ?o " +
+    				"} "+
+    			"} "+ 
+    			"LIMIT " + chunkSize;
+    }
 }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactory.java

@@ -4,24 +4,16 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters;
 
 import static org.apache.jena.ontology.OntModelSpec.OWL_MEM;
 
-import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph;
-import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.jena.graph.Graph;
-import org.apache.jena.graph.compose.MultiUnion;
 import org.apache.jena.graph.compose.Union;
 import org.apache.jena.ontology.OntModel;
-import org.apache.jena.ontology.impl.OntModelImpl;
 import org.apache.jena.rdf.model.Model;
 import org.apache.jena.rdf.model.ModelFactory;
-import org.apache.jena.rdf.model.impl.ModelCom;
 
 import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString;
 
-import java.util.List;
-
 /**
  * Make models that will do proper bulk updates.
  */
@@ -29,52 +21,49 @@ public class VitroModelFactory {
 	private static final Log log = LogFactory.getLog(VitroModelFactory.class);
 
 	public static Model createModel() {
-		return ModelFactory.createDefaultModel();
+		return createBulkInMemoryModel();
+	}
+
+	private static BulkModelCom createBulkInMemoryModel() {
+		return new BulkModelCom(new BulkGraphMem());
 	}
 
 	public static OntModel createOntologyModel() {
-		return ModelFactory.createOntologyModel(OWL_MEM);
+		return new BulkOntModelImpl(OWL_MEM, createBulkInMemoryModel());
 	}
 
 	public static OntModel createOntologyModel(Model model) {
 		Graph graph = model.getGraph();
-		Model bareModel = new ModelCom(graph);
-		OntModel ontModel = new OntModelImpl(OWL_MEM, bareModel);
+		Model bareModel = new BulkModelCom(graph);
+		OntModel ontModel = new BulkOntModelImpl(OWL_MEM, bareModel);
 		return new BulkUpdatingOntModel(ontModel);
 	}
 
 	public static Model createUnion(Model baseModel, Model plusModel) {
-		Graph baseGraph = baseModel.getGraph();
-		Graph plusGraph = plusModel.getGraph();
-
-		BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseGraph, plusGraph);
-
+		BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseModel, plusModel);
 		Model unionModel = ModelFactory.createModelForGraph(unionGraph);
-
-		return new BulkUpdatingModel(unionModel, baseModel);
+		return new BulkUpdatingModel(unionModel);
 	}
 
 	public static OntModel createUnion(OntModel baseModel, OntModel plusModel) {
-		Graph baseGraph = baseModel.getGraph();
-		Graph plusGraph = plusModel.getGraph();
-
-		BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseGraph, plusGraph);
-
+		BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseModel, plusModel);
 		Model unionModel = ModelFactory.createModelForGraph(unionGraph);
 		OntModel unionOntModel = ModelFactory.createOntologyModel(OWL_MEM, unionModel);
-
-
-		return new BulkUpdatingOntModel(unionOntModel, baseModel);
+		return new BulkUpdatingOntModel(unionOntModel);
 	}
 
 	public static Model createModelForGraph(Graph g) {
 		return new BulkUpdatingModel(ModelFactory.createModelForGraph(g));
 	}
 
-	private static class BulkUpdatingUnion extends Union {
-		@SuppressWarnings("deprecation")
-		public BulkUpdatingUnion(Graph L, Graph R) {
-			super(L, R);
+	public static class BulkUpdatingUnion extends Union {
+		private Model baseModel;
+		private Model plusModel;
+		
+		public BulkUpdatingUnion(Model baseModel, Model plusModel) {
+			super(baseModel.getGraph(), plusModel.getGraph());
+			this.baseModel = baseModel;
+			this.plusModel = plusModel;
 		}
 
 		@Override
@@ -83,5 +72,13 @@ public class VitroModelFactory {
 					+ ToString.graphToString(L) + ", R="
 					+ ToString.graphToString(R) + "]";
 		}
+
+		public Model getBaseModel() {
+			return baseModel;
+		}
+
+		public Model getPlusModel() {
+			return plusModel;
+		}
 	}
 }

api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java

@@ -18,6 +18,7 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import edu.cornell.mannlib.vitro.webapp.i18n.selection.SelectedLocale;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -182,12 +183,16 @@ public class RDFFilesLoader {
 	private static void readOntologyFileIntoModel(Path p, Model model) {
 		String format = getRdfFormat(p);
 		log.debug("Loading " + p);
+		Model memModel = ModelFactory.createDefaultModel();
 		try (InputStream stream = new FileInputStream(p.toFile())) {
-			model.read(stream, null, format);
+			memModel.read(stream, null, format);
+			model.add(memModel);
 			log.debug("...successful");
 		} catch (Exception e) {
 			log.warn("Could not load file '" + p + "' as " + format
 					+ ". Check that it contains valid data.", e);
+		} finally {
+			memModel.close();
 		}
 	}
 

api/src/test/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactoryTest.java

@@ -346,10 +346,13 @@ public class VitroModelFactoryTest extends AbstractTestClass {
 	public void addMultipleToVitroUnion() {
 		umg = new VitroUnionModelGroup();
 		umg.m.add(MULTIPLE_STATEMENTS);
-		new MethodCalls().add(umg.base.g, "add", "add")
-				.add(umg.base.l, "addedStatement", "addedStatement")
-				.add(umg.plus.g).add(umg.plus.l)
-				.add(umg.l, "addedStatements").test();
+		new MethodCalls()
+				.add(umg.base.g, "performAdd", "performAdd")
+				.add(umg.base.l, "addedStatements", "addedStatements")
+				.add(umg.plus.g)
+				.add(umg.plus.l)
+				.add(umg.l)
+				.test();
 	}
 
 	// ----------------------------------------------------------------------
@@ -452,10 +455,12 @@ public class VitroModelFactoryTest extends AbstractTestClass {
 	public void addMultipleToVitroOntUnion() {
 		uomg = new VitroUnionOntModelGroup();
 		uomg.om.add(MULTIPLE_STATEMENTS);
-		new MethodCalls().add(uomg.base.g, "add", "add")
+		new MethodCalls()
+				.add(uomg.base.g, "add", "add")
 				.add(uomg.base.l, "addedStatement", "addedStatement")
 				.add(uomg.plus.g).add(uomg.plus.l)
-				.add(uomg.l, "addedStatements").test();
+				.add(uomg.l)
+				.test();
 	}
 
 	// ----------------------------------------------------------------------
@@ -566,12 +571,16 @@ public class VitroModelFactoryTest extends AbstractTestClass {
 	public void addMultipleToVitroOntModeledUnionModel() {
 		omumg = new VitroOntModelUnionModelGroup();
 		omumg.om.add(MULTIPLE_STATEMENTS);
-		new MethodCalls().add(omumg.om, "add").add(omumg.ol, "addedStatements")
-				.add(omumg.union.base.g, "add", "add")
-				.add(omumg.union.base.m)
-				.add(omumg.union.base.l, "addedStatement", "addedStatement")
+		new MethodCalls()
+				.add(omumg.om, "add")
+				.add(omumg.ol)
+				.add(omumg.union.base.g, "performAdd", "performAdd")
+				.add(omumg.union.base.m, "add", "add")
+				.add(omumg.union.base.l, "addedStatements", "addedStatements")
 				.add(omumg.union.plus.g)
-				.add(omumg.union.plus.m).add(omumg.union.plus.l).test();
+				.add(omumg.union.plus.m)
+				.add(omumg.union.plus.l)
+				.test();
 	}
 
 	// ----------------------------------------------------------------------