litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3978 reduction in memory use for clearing inference scratchpad and rebuild models

Browse source
This commit is contained in:
brianjlowe 2012-10-01 20:08:28 +00:00
parent 1c5e2a88b7
commit b7e2c71cc1
4 changed files with 82 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ABoxJenaChangeListener.java
View file

@ -2,24 +2,17 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.util.HashSet;
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.SimpleReasonerSetup;
public class ABoxJenaChangeListener extends JenaChangeListener {
private HashSet<String> ignoredGraphs = new HashSet<String>();
public ABoxJenaChangeListener(ModelChangedListener listener) {
super(listener);
ignoredGraphs.add(JenaDataSourceSetupBase.JENA_INF_MODEL);
ignoredGraphs.add(JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
ignoredGraphs.add(JenaDataSourceSetupBase.JENA_TBOX_INF_MODEL);
ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_REBUILD);
ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_SCRATCHPAD);
}
@Override

19
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java
View file

@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import java.util.HashSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -16,6 +17,8 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.SimpleReasonerSetup;
/**
* A ChangeListener that forwards events to a Jena ModelChangedListener
@ -26,21 +29,33 @@ public class JenaChangeListener implements ChangeListener {
private static final Log log = LogFactory.getLog(JenaChangeListener.class);
protected HashSet<String> ignoredGraphs = new HashSet<String>();
private ModelChangedListener listener;
private Model m = ModelFactory.createDefaultModel();
public JenaChangeListener(ModelChangedListener listener) {
this.listener = listener;
ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_REBUILD);
ignoredGraphs.add(SimpleReasonerSetup.JENA_INF_MODEL_SCRATCHPAD);
}
@Override
public void addedStatement(String serializedTriple, String graphURI) {
listener.addedStatement(parseTriple(serializedTriple));
if (isRelevantGraph(graphURI)) {
listener.addedStatement(parseTriple(serializedTriple));
}
}
@Override
public void removedStatement(String serializedTriple, String graphURI) {
listener.removedStatement(parseTriple(serializedTriple));
if (isRelevantGraph(graphURI)) {
listener.removedStatement(parseTriple(serializedTriple));
}
}
private boolean isRelevantGraph(String graphURI) {
return (graphURI == null || !ignoredGraphs.contains(graphURI));
}
@Override

40
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java
View file

@ -4,9 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
@ -15,11 +13,9 @@ import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.graph.Graph;
import com.hp.hpl.jena.graph.GraphEvents;
import com.hp.hpl.jena.graph.GraphUtil;
import com.hp.hpl.jena.graph.Node;
import com.hp.hpl.jena.graph.Triple;
import com.hp.hpl.jena.graph.impl.SimpleBulkUpdateHandler;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Statement;
@ -150,7 +146,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
@Override
public void removeAll() {
removeAll(graph);
removeAll(graph, null, null, null);
notifyRemoveAll();
}
@ -183,7 +179,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
StringBuffer findQuery = new StringBuffer("CONSTRUCT { ")
.append(findPattern)
.append("} WHERE { \n");
.append(" } WHERE { \n");
if (graphURI != null) {
findQuery.append(" GRAPH <" + graphURI + "> { ");
}
@ -195,14 +191,30 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
String queryString = findQuery.toString();
try {
ChangeSet cs = graph.getRDFService().manufactureChangeSet();
cs.addRemoval(graph.getRDFService().sparqlConstructQuery(
queryString, RDFService.ModelSerializationFormat.N3),
int chunkSize = 50000;
boolean done = false;
while (!done) {
String chunkQueryString = queryString + " LIMIT " + chunkSize;
try {
Model chunkToRemove = RDFServiceUtils.parseModel(
graph.getRDFService().sparqlConstructQuery(
chunkQueryString, RDFService.ModelSerializationFormat.N3),
RDFService.ModelSerializationFormat.N3);
if (chunkToRemove.size() > 0) {
ChangeSet cs = graph.getRDFService().manufactureChangeSet();
ByteArrayOutputStream out = new ByteArrayOutputStream();
chunkToRemove.write(out, "N-TRIPLE");
cs.addRemoval(new ByteArrayInputStream(out.toByteArray()),
RDFService.ModelSerializationFormat.N3, graphURI);
graph.getRDFService().changeSetUpdate(cs);
} catch (RDFServiceException e) {
throw new RuntimeException(e);
graph.getRDFService().changeSetUpdate(cs);
} else {
done = true;
}
} catch (RDFServiceException e) {
throw new RuntimeException(e);
}
}
}

57
webapp/src/edu/cornell/mannlib/vitro/webapp/reasoner/ABoxRecomputer.java
View file

@ -6,6 +6,7 @@ import java.io.InputStream;
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
@ -115,6 +116,8 @@ public class ABoxRecomputer {
// recompute class subsumption inferences
inferenceRebuildModel.enterCriticalSection(Lock.WRITE);
try {
log.info("Clearing inference rebuild model.");
HashSet<String> unknownTypes = new HashSet<String>();
inferenceRebuildModel.removeAll();
@ -130,12 +133,15 @@ public class ABoxRecomputer {
try {
addedABoxTypeAssertion(individual, inferenceRebuildModel, unknownTypes);
simpleReasoner.setMostSpecificTypes(individual, inferenceRebuildModel, unknownTypes);
StmtIterator sit = aboxModel.listStatements(individual, null, (RDFNode) null);
while (sit.hasNext()) {
Statement s = sit.nextStatement();
for (ReasonerPlugin plugin : simpleReasoner.getPluginList()) {
plugin.addedABoxStatement(s, aboxModel, inferenceRebuildModel, tboxModel);
}
List<ReasonerPlugin> pluginList = simpleReasoner.getPluginList();
if (pluginList.size() > 0) {
StmtIterator sit = aboxModel.listStatements(individual, null, (RDFNode) null);
while (sit.hasNext()) {
Statement s = sit.nextStatement();
for (ReasonerPlugin plugin : pluginList) {
plugin.addedABoxStatement(s, aboxModel, inferenceRebuildModel, tboxModel);
}
}
}
} catch (NullPointerException npe) {
log.error("a NullPointerException was received while recomputing the ABox inferences. Halting inference computation.");
@ -346,7 +352,9 @@ public class ABoxRecomputer {
simpleReasoner.addedABoxTypeAssertion(stmt, inferenceModel, unknownTypes);
}
} finally {
iter.close();
if (iter != null) {
iter.close();
}
aboxModel.leaveCriticalSection();
}
}
@ -384,7 +392,9 @@ public class ABoxRecomputer {
}
}
} finally {
iter.close();
if (iter != null) {
iter.close();
}
inferenceModel.leaveCriticalSection();
}
@ -401,7 +411,9 @@ public class ABoxRecomputer {
}
}
} finally {
iter.close();
if (iter != null) {
iter.close();
}
}
// Add everything from the recomputed inference model that is not already
@ -432,22 +444,29 @@ public class ABoxRecomputer {
}
}
} finally {
iter.close();
if (iter != null) {
iter.close();
}
}
iter = scratchpadModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.next();
try {
while (iter.hasNext()) {
Statement stmt = iter.next();
inferenceModel.enterCriticalSection(Lock.WRITE);
try {
inferenceModel.add(stmt);
} finally {
inferenceModel.leaveCriticalSection();
}
inferenceModel.enterCriticalSection(Lock.WRITE);
try {
inferenceModel.add(stmt);
} finally {
inferenceModel.leaveCriticalSection();
}
}
} finally {
if (iter != null) {
iter.close();
}
}
} finally {
iter.close();
scratchpadModel.removeAll();
scratchpadModel.leaveCriticalSection();
}