NIHVIVO-3978 reduction in memory use for clearing inference scratchpad and rebuild models

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ABoxJenaChangeListener.java

@@ -2,24 +2,17 @@
 
 package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
-import java.util.HashSet;
-
 import com.hp.hpl.jena.rdf.model.ModelChangedListener;
 
 import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
-import edu.cornell.mannlib.vitro.webapp.servlet.setup.SimpleReasonerSetup;
 
 public class ABoxJenaChangeListener extends JenaChangeListener {
 
-    private HashSet<String> ignoredGraphs = new HashSet<String>();
-    
     public ABoxJenaChangeListener(ModelChangedListener listener) {
         super(listener);
         ignoredGraphs.add(JenaDataSourceSetupBase.JENA_INF_MODEL);
         ignoredGraphs.add(JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
         ignoredGraphs.add(JenaDataSourceSetupBase.JENA_TBOX_INF_MODEL);
-        ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_REBUILD);
-        ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_SCRATCHPAD);
     }
     
     @Override

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java

@@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
 import java.io.ByteArrayInputStream;
 import java.io.UnsupportedEncodingException;
+import java.util.HashSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -16,6 +17,8 @@ import com.hp.hpl.jena.rdf.model.Statement;
 import com.hp.hpl.jena.rdf.model.StmtIterator;
 
 import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
+import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
+import edu.cornell.mannlib.vitro.webapp.servlet.setup.SimpleReasonerSetup;
 
 /**
  * A ChangeListener that forwards events to a Jena ModelChangedListener 
@@ -26,23 +29,35 @@ public class JenaChangeListener implements ChangeListener {
 
     private static final Log log = LogFactory.getLog(JenaChangeListener.class);
     
+    protected HashSet<String> ignoredGraphs = new HashSet<String>();
+    
     private ModelChangedListener listener;
     private Model m = ModelFactory.createDefaultModel();
     
     public JenaChangeListener(ModelChangedListener listener) {
         this.listener = listener;
+        ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_REBUILD);
+        ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_SCRATCHPAD);
     }
     
     @Override
     public void addedStatement(String serializedTriple, String graphURI) {
-        listener.addedStatement(parseTriple(serializedTriple));
+        if (isRelevantGraph(graphURI)) {
+            listener.addedStatement(parseTriple(serializedTriple));
+        }
     }
 
     @Override
     public void removedStatement(String serializedTriple, String graphURI) {
-        listener.removedStatement(parseTriple(serializedTriple));
+        if (isRelevantGraph(graphURI)) {
+            listener.removedStatement(parseTriple(serializedTriple));
+        }
     }
 
+    private boolean isRelevantGraph(String graphURI) {
+        return (graphURI == null || !ignoredGraphs.contains(graphURI)); 
+    }
+    
     @Override
     public void notifyEvent(String graphURI, Object event) {
         log.debug("event: " + event.getClass());

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java

@@ -4,9 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
+import java.io.InputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
 import java.util.Iterator;
 import java.util.List;
 
@@ -15,11 +13,9 @@ import org.apache.commons.logging.LogFactory;
 
 import com.hp.hpl.jena.graph.Graph;
 import com.hp.hpl.jena.graph.GraphEvents;
-import com.hp.hpl.jena.graph.GraphUtil;
 import com.hp.hpl.jena.graph.Node;
 import com.hp.hpl.jena.graph.Triple;
 import com.hp.hpl.jena.graph.impl.SimpleBulkUpdateHandler;
-import com.hp.hpl.jena.query.ResultSet;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
 import com.hp.hpl.jena.rdf.model.Statement;
@@ -150,7 +146,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
 
     @Override 
     public void removeAll() {
-        removeAll(graph);
+        removeAll(graph, null, null, null);
         notifyRemoveAll(); 
     }
 
@@ -183,7 +179,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
         
         StringBuffer findQuery = new StringBuffer("CONSTRUCT { ")
         .append(findPattern)
-        .append("} WHERE { \n");
+        .append(" } WHERE { \n");
         if (graphURI != null) {
             findQuery.append("  GRAPH <" + graphURI + "> { ");
         }
@@ -195,14 +191,30 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
         
         String queryString = findQuery.toString();
         
-        try {
+        int chunkSize = 50000;
-            ChangeSet cs = graph.getRDFService().manufactureChangeSet();
+        boolean done = false;
-            cs.addRemoval(graph.getRDFService().sparqlConstructQuery(
+        
-                    queryString, RDFService.ModelSerializationFormat.N3), 
+        while (!done) { 
+            String chunkQueryString = queryString + " LIMIT " + chunkSize;
+            
+            try {
+                Model chunkToRemove = RDFServiceUtils.parseModel(
+                        graph.getRDFService().sparqlConstructQuery(
+                                chunkQueryString, RDFService.ModelSerializationFormat.N3), 
+                                         RDFService.ModelSerializationFormat.N3);
+                if (chunkToRemove.size() > 0) {
+                    ChangeSet cs = graph.getRDFService().manufactureChangeSet();
+                    ByteArrayOutputStream out = new ByteArrayOutputStream();
+                    chunkToRemove.write(out, "N-TRIPLE");    
+                    cs.addRemoval(new ByteArrayInputStream(out.toByteArray()), 
                             RDFService.ModelSerializationFormat.N3, graphURI);
-            graph.getRDFService().changeSetUpdate(cs);
+                    graph.getRDFService().changeSetUpdate(cs);
-        } catch (RDFServiceException e) {
+                } else {
-            throw new RuntimeException(e);
+                    done = true;
+                }
+            } catch (RDFServiceException e) {
+                throw new RuntimeException(e);
+            }
         }
         
     }

webapp/src/edu/cornell/mannlib/vitro/webapp/reasoner/ABoxRecomputer.java

@@ -6,6 +6,7 @@ import java.io.InputStream;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
@@ -115,6 +116,8 @@ public class ABoxRecomputer {
 		// recompute class subsumption inferences 
 		inferenceRebuildModel.enterCriticalSection(Lock.WRITE);			
 		try {
+		    
+		    log.info("Clearing inference rebuild model.");
 			HashSet<String> unknownTypes = new HashSet<String>();
 			inferenceRebuildModel.removeAll();
 			
@@ -130,12 +133,15 @@ public class ABoxRecomputer {
 				try {
 					addedABoxTypeAssertion(individual, inferenceRebuildModel, unknownTypes);
 					simpleReasoner.setMostSpecificTypes(individual, inferenceRebuildModel, unknownTypes);
-					StmtIterator sit = aboxModel.listStatements(individual, null, (RDFNode) null);
+					List<ReasonerPlugin> pluginList = simpleReasoner.getPluginList();
-					while (sit.hasNext()) {
+					if (pluginList.size() > 0) {
-						Statement s = sit.nextStatement();
+    					StmtIterator sit = aboxModel.listStatements(individual, null, (RDFNode) null);
-						for (ReasonerPlugin plugin : simpleReasoner.getPluginList()) {
+    					while (sit.hasNext()) {
-							plugin.addedABoxStatement(s, aboxModel, inferenceRebuildModel, tboxModel);
+    						Statement s = sit.nextStatement();
-						}
+    						for (ReasonerPlugin plugin : pluginList) {
+    							plugin.addedABoxStatement(s, aboxModel, inferenceRebuildModel, tboxModel);
+    						}
+    					}
 					}
 				} catch (NullPointerException npe) {
 	            	log.error("a NullPointerException was received while recomputing the ABox inferences. Halting inference computation.");
@@ -346,7 +352,9 @@ public class ABoxRecomputer {
 				simpleReasoner.addedABoxTypeAssertion(stmt, inferenceModel, unknownTypes);
 			}
 		} finally {
-			iter.close();
+		    if (iter != null) {
+			    iter.close();
+		    }
 			aboxModel.leaveCriticalSection();
 		}
 	}
@@ -384,7 +392,9 @@ public class ABoxRecomputer {
                 }
 			}
 		} finally {
-			iter.close();
+		    if (iter != null) {
+			    iter.close();
+		    }
             inferenceModel.leaveCriticalSection();
 		}
 		
@@ -401,7 +411,9 @@ public class ABoxRecomputer {
 				}
 			}
 		} finally {
-			iter.close();
+		    if (iter != null) {
+		        iter.close();    
+		    }
 		}
 					
 		// Add everything from the recomputed inference model that is not already
@@ -432,22 +444,29 @@ public class ABoxRecomputer {
                 }
 			}
 		} finally {
-			iter.close();	
+		    if (iter != null) {
+			    iter.close();	
+		    }
 		}
 					
 		iter = scratchpadModel.listStatements();
-		while (iter.hasNext()) {
+		try {
-			Statement stmt = iter.next();
+    		while (iter.hasNext()) {
-			
+    			Statement stmt = iter.next();
-			inferenceModel.enterCriticalSection(Lock.WRITE);
+    			
-			try {
+    			inferenceModel.enterCriticalSection(Lock.WRITE);
-				inferenceModel.add(stmt);
+    			try {
-			} finally {
+    				inferenceModel.add(stmt);
-				inferenceModel.leaveCriticalSection();
+    			} finally {
-			}
+    				inferenceModel.leaveCriticalSection();
+    			}
+    		}
+		} finally {
+		    if (iter != null) {
+		        iter.close();
+		    }
 		}
 	} finally {
-		iter.close();
 		scratchpadModel.removeAll();
 		scratchpadModel.leaveCriticalSection();			
 	}