improving logging in the update code; refactored some class names; added an additional sparql construct to delete data

This commit is contained in:
sjm222 2011-01-15 15:38:48 +00:00
parent eb7a3933e4
commit 99e1668862
11 changed files with 92 additions and 75 deletions

View file

@ -7,7 +7,6 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.hp.hpl.jena.ontology.DatatypeProperty;
import com.hp.hpl.jena.ontology.OntClass;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
@ -38,8 +37,8 @@ public class ABoxUpdater {
private OntModel newTboxModel;
private OntModel aboxModel;
private OntModel newTBoxAnnotationsModel;
private OntologyChangeLogger logger;
private OntologyChangeRecord record;
private ChangeLogger logger;
private ChangeRecord record;
private OntClass OWL_THING = (ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM)).createClass(OWL.Thing.getURI());
/**
@ -59,8 +58,8 @@ public class ABoxUpdater {
OntModel newTboxModel,
OntModel aboxModel,
OntModel newAnnotationsModel,
OntologyChangeLogger logger,
OntologyChangeRecord record) {
ChangeLogger logger,
ChangeRecord record) {
this.oldTboxModel = oldTboxModel;
this.newTboxModel = newTboxModel;
@ -177,10 +176,12 @@ public class ABoxUpdater {
//log summary of changes
if (renameCount > 0) {
logger.log("Changed " + renameCount + " subject reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class to be " + newClass.getURI());
//logger.log("Changed " + renameCount + " subject reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class to be " + newClass.getURI());
logger.log(renameCount + " subject reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class " + ((renameCount > 1) ? "were" : "was") +" changed to " + newClass.getURI());
}
if (removeCount > 0) {
logger.log("Removed " + removeCount + " remaining subject reference" + ((removeCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class");
//logger.log("Removed " + removeCount + " remaining subject reference" + ((removeCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class");
logger.log(removeCount + " remaining subject reference" + ((removeCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class " + ((removeCount > 1) ? "were" : "was") + "removed." );
}
// Change class references in the objects of rdf:type statements
@ -193,14 +194,12 @@ public class ABoxUpdater {
Statement newStatement = ResourceFactory.createStatement(oldStatement.getSubject(), oldStatement.getPredicate(), newClass);
retractions.add(oldStatement);
additions.add(newStatement);
//TODO - worried about logging changes before the changes have actually been made
// in the model
//logChanges(oldStatement, newStatement);
}
//log summary of changes
if (renameCount > 0) {
logger.log("Changed " + renameCount + " object reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class to be " + newClass.getURI());
logger.log(renameCount + " object reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class " + ((renameCount > 1) ? "were" : "was") + " changed to " + newClass.getURI());
//logger.log("Changed " + renameCount + " object reference" + ((renameCount > 1) ? "s" : "") + " to the " + oldClass.getURI() + " class to be " + newClass.getURI());
}
aboxModel.remove(retractions);

View file

@ -4,7 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.ontology.update;
import java.io.IOException;
public interface OntologyChangeLogger {
public interface ChangeLogger {
public void log(String logMessage) throws IOException;

View file

@ -4,7 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.ontology.update;
import com.hp.hpl.jena.rdf.model.Model;
public interface OntologyChangeRecord {
public interface ChangeRecord {
public void recordAdditions(Model incrementalAdditions);

View file

@ -35,8 +35,8 @@ import com.hp.hpl.jena.shared.Lock;
public class DateTimeMigration {
private OntModel aboxModel;
private OntologyChangeLogger logger;
private OntologyChangeRecord record;
private ChangeLogger logger;
private ChangeRecord record;
private static final String dateTimeURI = "http://vivoweb.org/ontology/core#dateTime";
private static final String dateTimePrecisionURI = "http://vivoweb.org/ontology/core#dateTimePrecision";
@ -68,7 +68,7 @@ public class DateTimeMigration {
* @param record - for writing to the additions model
* and the retractions model.
*/
public DateTimeMigration(OntModel aboxModel,OntologyChangeLogger logger, OntologyChangeRecord record) {
public DateTimeMigration(OntModel aboxModel,ChangeLogger logger, ChangeRecord record) {
this.aboxModel = aboxModel;
this.logger = logger;
@ -216,7 +216,7 @@ public class DateTimeMigration {
if (additions.size() > 0) {
logger.log(additions.size() + " date/time literal" +
((additions.size() > 1) ? "s" : "") + ((additions.size() > 1) ? " were " : " was ") +
"updated to the 1.2 representation.");
"updated to the xsd:dateTime representation.");
}
} finally {
aboxModel.leaveCriticalSection();

View file

@ -34,18 +34,18 @@ import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils;
* @author bjl23
*
*/
public class OntologyUpdater {
public class KnowledgeBaseUpdater {
//private final Log log = LogFactory.getLog(OntologyUpdater.class);
private OntologyUpdateSettings settings;
private OntologyChangeLogger logger;
private OntologyChangeRecord record;
private UpdateSettings settings;
private ChangeLogger logger;
private ChangeRecord record;
public OntologyUpdater(OntologyUpdateSettings settings) {
public KnowledgeBaseUpdater(UpdateSettings settings) {
this.settings = settings;
this.logger = null;
this.record = new SimpleOntologyChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
}
public boolean update() throws IOException {
@ -56,7 +56,7 @@ public class OntologyUpdater {
if (updateRequired) {
if (this.logger == null) {
this.logger = new SimpleOntologyChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
}
try {
@ -101,7 +101,7 @@ public class OntologyUpdater {
private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel aboxModel) throws IOException {
Model anonModel = performSparqlConstructs(sparqlConstructDir, aboxModel);
Model anonModel = performSparqlConstructs(sparqlConstructDir, aboxModel, true);
if (anonModel == null) {
return;
@ -110,24 +110,28 @@ public class OntologyUpdater {
aboxModel.enterCriticalSection(Lock.WRITE);
try {
JenaIngestUtils jiu = new JenaIngestUtils();
Model additions = jiu.renameBNodes(anonModel, settings.getDefaultNamespace() + "n",
aboxModel);
Model additions = jiu.renameBNodes(anonModel, settings.getDefaultNamespace() + "n", aboxModel);
Model actualAdditions = ModelFactory.createDefaultModel();
StmtIterator stmtIt = additions.listStatements();
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (!aboxModel.contains(stmt)) {
actualAdditions.add(stmt);
}
}
aboxModel.add(actualAdditions);
record.recordAdditions(actualAdditions);
/*
if (actualAdditions.size() > 0) {
logger.log("Constructed " + actualAdditions.size() + " new " +
"statement"
+ ((actualAdditions.size() > 1) ? "s" : "") +
" using SPARQL CONSTRUCT queries.");
" using SPARQL construct queries.");
}
record.recordAdditions(actualAdditions);
*/
} finally {
aboxModel.leaveCriticalSection();
}
@ -136,7 +140,7 @@ public class OntologyUpdater {
private void performSparqlConstructRetractions(String sparqlConstructDir, OntModel aboxModel) throws IOException {
Model retractions = performSparqlConstructs(sparqlConstructDir, aboxModel);
Model retractions = performSparqlConstructs(sparqlConstructDir, aboxModel, false);
if (retractions == null) {
return;
@ -153,12 +157,13 @@ public class OntologyUpdater {
}
}
aboxModel.remove(actualRetractions);
if (actualRetractions.size() > 0) {
logger.log("Removed " + actualRetractions.size() + " statement"
+ ((actualRetractions.size() > 1) ? "s" : "") +
" using SPARQL CONSTRUCT queries.");
}
record.recordRetractions(actualRetractions);
/*
if (actualRetractions.size() > 0) {
logger.log("Removed " + actualRetractions.size() + " statement" + ((actualRetractions.size() > 1) ? "s" : "") + " using SPARQL CONSTRUCT queries.");
}
*/
} finally {
aboxModel.leaveCriticalSection();
}
@ -174,7 +179,7 @@ public class OntologyUpdater {
* @param aboxModel
*/
private Model performSparqlConstructs(String sparqlConstructDir,
OntModel aboxModel) throws IOException {
OntModel aboxModel, boolean add) throws IOException {
Model anonModel = ModelFactory.createDefaultModel();
File sparqlConstructDirectory = new File(sparqlConstructDir);
@ -186,6 +191,7 @@ public class OntologyUpdater {
" SPARQL CONSTRUCTS.");
return null;
}
File[] sparqlFiles = sparqlConstructDirectory.listFiles();
for (int i = 0; i < sparqlFiles.length; i ++) {
File sparqlFile = sparqlFiles[i];
@ -198,13 +204,21 @@ public class OntologyUpdater {
fileContents.append(ln).append('\n');
}
try {
Query q = QueryFactory.create(fileContents.toString(),
Syntax.syntaxARQ);
Query q = QueryFactory.create(fileContents.toString(), Syntax.syntaxARQ);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
QueryExecution qe = QueryExecutionFactory.create(q,
aboxModel);
QueryExecution qe = QueryExecutionFactory.create(q, aboxModel);
long numBefore = anonModel.size();
qe.execConstruct(anonModel);
long numAfter = anonModel.size();
long num = numAfter - numBefore;
if (num > 0) {
logger.log((add ? "Added " : "Removed ") + num +
" statement" + ((num > 1) ? "s" : "") +
" using the SPARQL construct query from file " + sparqlFiles[i].getName());
}
} finally {
aboxModel.leaveCriticalSection();
}
@ -214,8 +228,7 @@ public class OntologyUpdater {
"query at " + sparqlFile + ". Error message is: " + e.getMessage());
}
} catch (FileNotFoundException fnfe) {
logger.logError(this.getClass().getName() +
".performSparqlConstructs() could not find " +
logger.log("WARNING: performSparqlConstructs() could not find " +
" SPARQL CONSTRUCT file " + sparqlFile + ". Skipping.");
}
}
@ -252,8 +265,8 @@ public class OntologyUpdater {
settings.getNewTBoxAnnotationsModel(),
settings.getOntModelSelector().getABoxModel(), logger, record);
tboxUpdater.updateVitroPropertyDefaultValues();
tboxUpdater.updateVitroAnnotationsModel();
tboxUpdater.updateDefaultAnnotationValues();
tboxUpdater.updateAnnotationModel();
}
/**

View file

@ -23,9 +23,9 @@ import edu.cornell.mannlib.vitro.webapp.ontology.update.AtomicOntologyChange.Ato
public class OntologyChangeParser {
private OntologyChangeLogger logger;
private ChangeLogger logger;
public OntologyChangeParser(OntologyChangeLogger logger) {
public OntologyChangeParser(ChangeLogger logger) {
this.logger = logger;
}

View file

@ -8,14 +8,14 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
public class SimpleOntologyChangeLogger implements OntologyChangeLogger {
public class SimpleChangeLogger implements ChangeLogger {
private Writer logWriter;
private Writer errorWriter;
private boolean errorsWritten = false;
public SimpleOntologyChangeLogger( String logPath,
public SimpleChangeLogger( String logPath,
String errorPath ) {
File logFile = new File(logPath);
File errorFile = new File(errorPath);
@ -36,7 +36,8 @@ public class SimpleOntologyChangeLogger implements OntologyChangeLogger {
className = className.substring(className.lastIndexOf('.') + 1 );
String methodName = ((StackTraceElement)elements[1]).getMethodName();
logWriter.write(className + "." + methodName + ": " + logMessage + "\n\n");
logWriter.write(className + ": " + logMessage + "\n\n");
//logWriter.write(className + "." + methodName + ": " + logMessage + "\n\n");
logWriter.flush();
}

View file

@ -14,10 +14,10 @@ import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
public class SimpleOntologyChangeRecord implements OntologyChangeRecord {
public class SimpleChangeRecord implements ChangeRecord {
private final static Log log =
LogFactory.getLog(SimpleOntologyChangeRecord.class);
LogFactory.getLog(SimpleChangeRecord.class);
private final static String RDF_SYNTAX = "N3";
@ -26,7 +26,7 @@ public class SimpleOntologyChangeRecord implements OntologyChangeRecord {
private File additionsFile;
private File retractionsFile;
public SimpleOntologyChangeRecord(
public SimpleChangeRecord(
String additionsFile, String retractionsFile) {
this.additionsFile = new File(additionsFile);
try {

View file

@ -5,7 +5,6 @@ package edu.cornell.mannlib.vitro.webapp.ontology.update;
import java.io.IOException;
import java.util.List;
import com.hp.hpl.jena.ontology.DatatypeProperty;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.rdf.model.Literal;
@ -32,8 +31,8 @@ public class TBoxUpdater {
private OntModel oldTboxAnnotationsModel;
private OntModel newTboxAnnotationsModel;
private OntModel siteModel;
private OntologyChangeLogger logger;
private OntologyChangeRecord record;
private ChangeLogger logger;
private ChangeRecord record;
private boolean detailLogs = false;
private static final String classGroupURI = "http://vitro.mannlib.cornell.edu/ns/vitro/0.7#ClassGroup";
@ -57,8 +56,8 @@ public class TBoxUpdater {
public TBoxUpdater(OntModel oldTboxAnnotationsModel,
OntModel newTboxAnnotationsModel,
OntModel siteModel,
OntologyChangeLogger logger,
OntologyChangeRecord record) {
ChangeLogger logger,
ChangeRecord record) {
this.oldTboxAnnotationsModel = oldTboxAnnotationsModel;
this.newTboxAnnotationsModel = newTboxAnnotationsModel;
@ -92,7 +91,7 @@ public class TBoxUpdater {
* Note: as specified, this method for now assumes that no new vitro annotation
* properties have been introduced. This should be updated for future versions.
*/
public void updateVitroPropertyDefaultValues() throws IOException {
public void updateDefaultAnnotationValues() throws IOException {
siteModel.enterCriticalSection(Lock.WRITE);
@ -296,9 +295,9 @@ public class TBoxUpdater {
// log the additions - summary
if (newAnnotationSettingsToAdd.size() > 0) {
boolean plural = (newAnnotationSettingsToAdd.size() > 1);
logger.log("Added " + newAnnotationSettingsToAdd.size() + " new annotation property setting" + (plural ? "s" : "") + " to the knowledge base. This includes " +
logger.log("Added " + newAnnotationSettingsToAdd.size() + " new annotation property setting" + (plural ? "s" : "") + " to the knowledge base. This includes only " +
"existing annotation properties applied to existing classes where they weren't applied before, or existing " +
"properties applied to new classes. No new annotation properties have been introduced.");
"properties applied to new classes.");
}
} finally {
@ -319,7 +318,7 @@ public class TBoxUpdater {
* knowledge base.
*
*/
public void updateVitroAnnotationsModel() throws IOException {
public void updateAnnotationModel() throws IOException {
// for each ClassGroup in the old vitro annotations model: if it is not in
// the new vitro annotations model and the site has no classes asserted to
@ -335,22 +334,20 @@ public void updateVitroAnnotationsModel() throws IOException {
StmtIterator iter = oldTboxAnnotationsModel.listStatements((Resource) null, RDF.type, classGroupClass);
int count = 0;
while (iter.hasNext()) {
Statement stmt = iter.next();
if (!newTboxAnnotationsModel.contains(stmt) && !usesGroup(siteModel, stmt.getSubject())) {
count++;
retractions.add(siteModel.listStatements(stmt.getSubject(),(Property) null,(RDFNode)null));
logger.log("Removed the " + stmt.getSubject().getURI() + " ClassGroup from the annotations model.");
}
}
if (retractions.size() > 0) {
siteModel.remove(retractions);
record.recordRetractions(retractions);
logger.log("Removed " + count + " Class Group" + (count > 1 ? "s" : "") + " from the annotations model.");
}
} finally {
siteModel.leaveCriticalSection();
}
@ -364,9 +361,16 @@ public void updateVitroAnnotationsModel() throws IOException {
public boolean usesGroup(Model model, Resource theClassGroup) throws IOException {
//logger.log("called for " + theClassGroup.getLocalName() );
model.enterCriticalSection(Lock.READ);
try {
StmtIterator iter = model.listStatements((Resource) null, inClassGroupProp, theClassGroup);
while (iter.hasNext()) {
logger.log("statement: " + ABoxUpdater.stmtString(iter.next()));
}
return (model.contains((Resource) null, inClassGroupProp, theClassGroup) ? true : false);
} finally {
model.leaveCriticalSection();

View file

@ -6,7 +6,7 @@ import com.hp.hpl.jena.ontology.OntModel;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
public class OntologyUpdateSettings {
public class UpdateSettings {
private String dataDir;
private String sparqlConstructAdditionsDir;

View file

@ -27,8 +27,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDao;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
import edu.cornell.mannlib.vitro.webapp.dao.jena.SimpleOntModelSelector;
import edu.cornell.mannlib.vitro.webapp.ontology.update.OntologyUpdateSettings;
import edu.cornell.mannlib.vitro.webapp.ontology.update.OntologyUpdater;
import edu.cornell.mannlib.vitro.webapp.ontology.update.UpdateSettings;
import edu.cornell.mannlib.vitro.webapp.ontology.update.KnowledgeBaseUpdater;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup;
/**
@ -71,7 +71,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
(OntModel) sce.getServletContext().getAttribute(
JenaBaseDao.ASSERTIONS_ONT_MODEL_ATTRIBUTE_NAME));
OntologyUpdateSettings settings = new OntologyUpdateSettings();
UpdateSettings settings = new UpdateSettings();
settings.setAskQueryFile(ctx.getRealPath(ASK_QUERY_FILE));
settings.setDataDir(ctx.getRealPath(DATA_DIR));
settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR));
@ -99,7 +99,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
try {
OntologyUpdater ontologyUpdater = new OntologyUpdater(settings);
KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
try {
if (ontologyUpdater.updateRequired()) {