litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

applying ABox update to all graphs

Browse source
This commit is contained in:
brianjlowe 2013-09-12 10:37:24 -04:00
parent f7d03b6de8
commit d5c032f33e
4 changed files with 334 additions and 256 deletions
Download patch file Download diff file Expand all files Collapse all files

79
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/ABoxUpdater.java
View file

@ -14,6 +14,7 @@ import com.hp.hpl.jena.ontology.OntClass;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.ontology.OntProperty;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -27,7 +28,9 @@ import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
import edu.cornell.mannlib.vitro.webapp.ontology.update.AtomicOntologyChange.AtomicChangeType;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
/**
* Performs knowledge base updates to the abox to align with a new ontology version
@ -38,7 +41,8 @@ public class ABoxUpdater {
private final Log log = LogFactory.getLog(ABoxUpdater.class);
private OntModel oldTboxModel;
private OntModel newTboxModel;
private OntModel aboxModel;
private Dataset dataset;
private RDFService rdfService;
private OntModel newTBoxAnnotationsModel;
private ChangeLogger logger;
private ChangeRecord record;
@ -59,14 +63,15 @@ public class ABoxUpdater {
*/
public ABoxUpdater(OntModel oldTboxModel,
OntModel newTboxModel,
OntModel aboxModel,
RDFService rdfService,
OntModel newAnnotationsModel,
ChangeLogger logger,
ChangeRecord record) {
this.oldTboxModel = oldTboxModel;
this.newTboxModel = newTboxModel;
this.aboxModel = aboxModel;
this.dataset = new RDFServiceDataset(rdfService);
this.rdfService = rdfService;
this.newTBoxAnnotationsModel = newAnnotationsModel;
this.logger = logger;
this.record = record;
@ -126,8 +131,15 @@ public class ABoxUpdater {
public void renameClass(AtomicOntologyChange change) throws IOException {
//logger.log("Processing a class rename from: " + change.getSourceURI() + " to " + change.getDestinationURI());
aboxModel.enterCriticalSection(Lock.WRITE);
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
Model additions = ModelFactory.createDefaultModel();
@ -187,7 +199,7 @@ public class ABoxUpdater {
} finally {
aboxModel.leaveCriticalSection();
}
}
}
/**
@ -233,16 +245,17 @@ public class ABoxUpdater {
if (!parentOfAddedClass.equals(OWL.Thing)) {
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
StmtIterator stmtIter = aboxModel.listStatements(null, RDF.type, parentOfAddedClass);
int count = stmtIter.toList().size();
if (count > 0) {
String indList = "";
while (stmtIter.hasNext()) {
Statement stmt = stmtIter.next();
indList += "\n\t" + stmt.getSubject().getURI();
}
if (count > 0) {
//TODO - take out the detailed logging after our internal testing is completed.
@ -250,6 +263,7 @@ public class ABoxUpdater {
" and a new subclass of that class has been added: " + addedClass.getURI() + ". " +
"Please review " + ((count > 1) ? "these" : "this") + " individual" + ((count > 1) ? "s" : "") + " to see whether " + ((count > 1) ? "they" : "it") + " should be of type: " + addedClass.getURI() );
}
}
}
}
@ -332,6 +346,13 @@ public class ABoxUpdater {
}
// Remove instances of the deleted class
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
int count = 0;
@ -352,12 +373,20 @@ public class ABoxUpdater {
aboxModel.leaveCriticalSection();
}
}
}
protected int deleteIndividual(Resource individual) throws IOException {
Model retractions = ModelFactory.createDefaultModel();
int refCount = 0;
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
StmtIterator iter = aboxModel.listStatements(individual, (Property) null, (RDFNode) null);
@ -381,6 +410,7 @@ public class ABoxUpdater {
} finally {
aboxModel.leaveCriticalSection();
}
}
return refCount;
}
@ -439,6 +469,13 @@ public class ABoxUpdater {
if (inverseOfAddedProperty != null) {
Model additions = ModelFactory.createDefaultModel();
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
@ -472,6 +509,7 @@ public class ABoxUpdater {
}
}
}
}
private void deleteProperty(AtomicOntologyChange propObj) throws IOException{
@ -507,6 +545,13 @@ public class ABoxUpdater {
}
}
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
Model deletePropModel = ModelFactory.createDefaultModel();
if (replacementProperty == null) {
@ -528,6 +573,7 @@ public class ABoxUpdater {
AtomicOntologyChange chg = new AtomicOntologyChange(deletedProperty.getURI(), replacementProperty.getURI(), AtomicChangeType.RENAME, propObj.getNotes());
renameProperty(chg);
}
}
}
@ -548,6 +594,14 @@ public class ABoxUpdater {
return;
}
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!KnowledgeBaseUpdater.isUpdatableABoxGraph(graph)){
continue;
}
Model aboxModel = dataset.getNamedModel(graph);
Model renamePropAddModel = ModelFactory.createDefaultModel();
Model renamePropRetractModel = ModelFactory.createDefaultModel();
@ -578,6 +632,7 @@ public class ABoxUpdater {
propObj.getDestinationURI() + " instead");
}
}
}
public void logChanges(Statement oldStatement, Statement newStatement) throws IOException {
logChange(oldStatement,false);

73
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -24,19 +24,16 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
@ -123,10 +120,10 @@ public class KnowledgeBaseUpdater {
updateABox(changes);
log.info("performing SPARQL CONSTRUCT additions");
performSparqlConstructs(settings.getSparqlConstructAdditionsDir(), readModel, writeModel, ADD);
performSparqlConstructs(settings.getSparqlConstructAdditionsDir(), settings.getRDFService(), ADD);
log.info("performing SPARQL CONSTRUCT retractions");
performSparqlConstructs(settings.getSparqlConstructDeletionsDir(), readModel, writeModel, RETRACT);
performSparqlConstructs(settings.getSparqlConstructDeletionsDir(), settings.getRDFService(), RETRACT);
}
@ -144,10 +141,11 @@ public class KnowledgeBaseUpdater {
* @param add (add = true; retract = false)
*/
private void performSparqlConstructs(String sparqlConstructDir,
OntModel readModel, OntModel writeModel,
RDFService rdfService,
boolean add) throws IOException {
Dataset dataset = new RDFServiceDataset(rdfService);
File sparqlConstructDirectory = new File(sparqlConstructDir);
log.info("Using SPARQL CONSTRUCT director " + sparqlConstructDirectory);
log.info("Using SPARQL CONSTRUCT directory " + sparqlConstructDirectory);
if (!sparqlConstructDirectory.isDirectory()) {
String logMsg = this.getClass().getName() +
"performSparqlConstructs() expected to find a directory " +
@ -161,7 +159,6 @@ public class KnowledgeBaseUpdater {
Collections.sort(sparqlFiles); // queries may depend on being run in a certain order
JenaIngestUtils jiu = new JenaIngestUtils();
for (File sparqlFile : sparqlFiles) {
Model anonModel = ModelFactory.createDefaultModel();
StringBuffer fileContents = new StringBuffer();
try {
BufferedReader reader = new BufferedReader(new FileReader(sparqlFile));
@ -176,16 +173,16 @@ public class KnowledgeBaseUpdater {
log.info(logMsg);
continue;
}
Model anonModel = ModelFactory.createDefaultModel();
try {
log.info("\t\tprocessing SPARQL construct query from file " + sparqlFile.getName());
Query q = QueryFactory.create(fileContents.toString(), Syntax.syntaxARQ);
readModel.enterCriticalSection(Lock.READ);
try {
QueryExecution qe = QueryExecutionFactory.create(q, readModel);
long numBefore = anonModel.size();
qe.execConstruct(anonModel);
long numAfter = anonModel.size();
long num = numAfter - numBefore;
anonModel = RDFServiceUtils.parseModel(
rdfService.sparqlConstructQuery(fileContents.toString(),
RDFService.ModelSerializationFormat.NTRIPLE),
ModelSerializationFormat.NTRIPLE);
long num = anonModel.size();
if (num > 0) {
String logMsg = (add ? "Added " : "Removed ") + num +
" statement" + ((num > 1) ? "s" : "") +
@ -195,23 +192,34 @@ public class KnowledgeBaseUpdater {
logger.log(logMsg);
log.info(logMsg);
}
qe.close();
} finally {
readModel.leaveCriticalSection();
}
} catch (Exception e) {
logger.logError(this.getClass().getName() +
".performSparqlConstructs() unable to execute " +
"query at " + sparqlFile + ". Error message is: " + e.getMessage());
log.error(e,e);
}
writeModel.enterCriticalSection(Lock.WRITE);
try {
if(!add) {
writeModel.remove(anonModel);
StmtIterator sit = anonModel.listStatements();
while (sit.hasNext()) {
Statement stmt = sit.nextStatement();
Iterator<String> graphIt = dataset.listNames();
while(graphIt.hasNext()) {
String graph = graphIt.next();
if(!isUpdatableABoxGraph(graph)) {
continue;
}
Model writeModel = dataset.getNamedModel(graph);
if (writeModel.contains(stmt)) {
writeModel.remove(stmt);
}
}
}
record.recordRetractions(anonModel);
//log.info("removed " + anonModel.size() + " statements from SPARQL CONSTRUCTs");
} else {
Model writeModel = dataset.getNamedModel(JenaDataSourceSetupBase.JENA_DB_MODEL);
Model additions = jiu.renameBNodes(
anonModel, settings.getDefaultNamespace() + "n", writeModel);
Model actualAdditions = ModelFactory.createDefaultModel();
@ -226,9 +234,7 @@ public class KnowledgeBaseUpdater {
//log.info("added " + actualAdditions.size() + " statements from SPARQL CONSTRUCTs");
record.recordAdditions(actualAdditions);
}
} finally {
writeModel.leaveCriticalSection();
}
}
}
@ -245,8 +251,8 @@ public class KnowledgeBaseUpdater {
OntModel oldTBoxModel = settings.getOldTBoxModel();
OntModel newTBoxModel = settings.getNewTBoxModel();
OntModel ABoxModel = settings.getAssertionOntModelSelector().getABoxModel();
ABoxUpdater aboxUpdater = new ABoxUpdater(oldTBoxModel, newTBoxModel, ABoxModel,settings.getNewTBoxAnnotationsModel(), logger, record);
RDFService rdfService = settings.getRDFService();
ABoxUpdater aboxUpdater = new ABoxUpdater(oldTBoxModel, newTBoxModel, rdfService, settings.getNewTBoxAnnotationsModel(), logger, record);
aboxUpdater.processPropertyChanges(changes.getAtomicPropertyChanges());
aboxUpdater.processClassChanges(changes.getAtomicClassChanges());
}
@ -367,7 +373,9 @@ public class KnowledgeBaseUpdater {
}
}
public static boolean isUpdatableABoxGraph(String graphName) {
return (!graphName.contains("tbox") && !graphName.contains("filegraph"));
}
/**
* A class that allows to access two different ontology change lists,
@ -434,5 +442,6 @@ public class KnowledgeBaseUpdater {
public List<AtomicOntologyChange> getAtomicPropertyChanges() {
return atomicPropertyChanges;
}
}
}

10
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/UpdateSettings.java
View file

@ -5,6 +5,7 @@ package edu.cornell.mannlib.vitro.webapp.ontology.update;
import com.hp.hpl.jena.ontology.OntModel;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
public class UpdateSettings {
@ -37,6 +38,8 @@ public class UpdateSettings {
private OntModel newDisplayModelFromFile;
private OntModel loadedAtStartupDisplayModel;
private OntModel oldDisplayModelVivoListViewConfig;
private RDFService rdfService;
public String getDataDir() {
return dataDir;
}
@ -223,6 +226,13 @@ public class UpdateSettings {
return this.oldDisplayModelVivoListViewConfig;
}
public RDFService getRDFService() {
return this.rdfService;
}
public void setRDFService(RDFService rdfService) {
this.rdfService = rdfService;
}
}

4
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -41,6 +41,9 @@ import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.ontology.update.KnowledgeBaseUpdater;
import edu.cornell.mannlib.vitro.webapp.ontology.update.UpdateSettings;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
/**
@ -95,6 +98,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
settings.setOldTBoxAnnotationsModel(oldTBoxAnnotationsModel);
OntModel newTBoxAnnotationsModel = loadModelFromDirectory(createDirectory(homeDir, "rdf", "tbox", "everytime").toString());
settings.setNewTBoxAnnotationsModel(newTBoxAnnotationsModel);
settings.setRDFService(RDFServiceUtils.getRDFServiceFactory(ctx).getRDFService());
boolean tryMigrateDisplay = true;
try {