litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

annotation data migration

Browse source
This commit is contained in:
brianjlowe 2013-09-18 16:21:36 -04:00
parent e739a94222
commit c062201a9a
7 changed files with 569 additions and 402 deletions
Download patch file Download diff file Expand all files Collapse all files

2
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/VitroVocabulary.java
View file

@ -12,6 +12,8 @@ public class VitroVocabulary {
public static final String VITRO_AUTH = "http://vitro.mannlib.cornell.edu/ns/vitro/authorization#";
public static final String VITRO_PUBLIC = "http://vitro.mannlib.cornell.edu/ns/vitro/public#";
public static final String VITRO_PUBLIC_ONTOLOGY = "http://vitro.mannlib.cornell.edu/ns/vitro/public";
// TODO change the following before 1.6 release
public static final String PROPERTY_CONFIG_DATA = "http://example.org/appConfig/";
/** BJL23 2008-02-25:

1
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ObjectPropertyDaoJena.java
View file

@ -949,7 +949,6 @@ public class ObjectPropertyDaoJena extends PropertyDaoJena implements ObjectProp
" { ?property display:listViewConfigFile ?filename \n" +
" } UNION { \n" +
" ?configuration config:listViewConfigFile ?filename . \n " +
// " ?configuration config:hasListView ?lv . " +
" ?context config:hasConfiguration ?configuration . \n" +
" ?context config:configContextFor ?property . \n" +
" ?context config:qualifiedBy ?range . \n" +

16
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/ABoxUpdater.java
View file

@ -44,6 +44,7 @@ public class ABoxUpdater {
private Dataset dataset;
private RDFService rdfService;
private OntModel newTBoxAnnotationsModel;
private TBoxUpdater tboxUpdater;
private ChangeLogger logger;
private ChangeRecord record;
private OntClass OWL_THING = (ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM)).createClass(OWL.Thing.getURI());
@ -61,20 +62,19 @@ public class ABoxUpdater {
* and the retractions model.
*
*/
public ABoxUpdater(OntModel oldTboxModel,
OntModel newTboxModel,
RDFService rdfService,
OntModel newAnnotationsModel,
public ABoxUpdater(UpdateSettings settings,
ChangeLogger logger,
ChangeRecord record) {
this.oldTboxModel = oldTboxModel;
this.newTboxModel = newTboxModel;
this.oldTboxModel = settings.getOldTBoxModel();
this.newTboxModel = settings.getNewTBoxModel();
RDFService rdfService = settings.getRDFService();
this.dataset = new RDFServiceDataset(rdfService);
this.rdfService = rdfService;
this.newTBoxAnnotationsModel = newAnnotationsModel;
this.newTBoxAnnotationsModel = settings.getNewTBoxAnnotationsModel();
this.logger = logger;
this.record = record;
this.tboxUpdater = new TBoxUpdater(settings, logger, record);
}
/**
@ -632,6 +632,8 @@ public class ABoxUpdater {
propObj.getDestinationURI() + " instead");
}
}
tboxUpdater.renameProperty(propObj);
}
public void logChanges(Statement oldStatement, Statement newStatement) throws IOException {

37
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -64,7 +64,6 @@ public class KnowledgeBaseUpdater {
}
long startTime = System.currentTimeMillis();
System.out.println("Migrating the knowledge base");
log.info("Migrating the knowledge base");
logger.log("Started knowledge base migration");
@ -84,7 +83,6 @@ public class KnowledgeBaseUpdater {
logger.closeLogs();
long elapsedSecs = (System.currentTimeMillis() - startTime)/1000;
System.out.println("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
log.info("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
return;
@ -111,20 +109,15 @@ public class KnowledgeBaseUpdater {
log.error("unable to migrate migration metadata " + e.getMessage());
}
log.warn("KnowledgeBaseUpdater needs to be modified to work on all graphs!");
OntModel readModel = settings.getUnionOntModelSelector().getABoxModel();
OntModel writeModel = settings.getAssertionOntModelSelector().getABoxModel();
// TODO make sure the ABox update applies to all graphs
log.info("\tupdating the abox");
updateABox(changes);
log.info("performing SPARQL CONSTRUCT additions");
performSparqlConstructs(settings.getSparqlConstructAdditionsDir(), settings.getRDFService(), ADD);
log.info("performing SPARQL CONSTRUCT retractions");
performSparqlConstructs(settings.getSparqlConstructDeletionsDir(), settings.getRDFService(), RETRACT);
log.info("\tupdating the abox");
updateABox(changes);
}
private static final boolean ADD = true;
@ -249,10 +242,8 @@ public class KnowledgeBaseUpdater {
private void updateABox(AtomicOntologyChangeLists changes)
throws IOException {
OntModel oldTBoxModel = settings.getOldTBoxModel();
OntModel newTBoxModel = settings.getNewTBoxModel();
RDFService rdfService = settings.getRDFService();
ABoxUpdater aboxUpdater = new ABoxUpdater(oldTBoxModel, newTBoxModel, rdfService, settings.getNewTBoxAnnotationsModel(), logger, record);
ABoxUpdater aboxUpdater = new ABoxUpdater(settings, logger, record);
aboxUpdater.processPropertyChanges(changes.getAtomicPropertyChanges());
aboxUpdater.processClassChanges(changes.getAtomicClassChanges());
}
@ -290,14 +281,18 @@ public class KnowledgeBaseUpdater {
rdfService.changeSetUpdate(removeChangeSet);
}
private void updateTBoxAnnotations() throws IOException {
TBoxUpdater tboxUpdater = new TBoxUpdater(settings.getOldTBoxAnnotationsModel(),
settings.getNewTBoxAnnotationsModel(),
settings.getAssertionOntModelSelector().getTBoxModel(), logger, record);
private void updateTBoxAnnotations() {
TBoxUpdater tboxUpdater = new TBoxUpdater(settings, logger, record);
try {
tboxUpdater.modifyPropertyQualifications();
} catch (Exception e) {
log.error("Unable to modify qualified property config file ", e);
}
try {
tboxUpdater.updateDefaultAnnotationValues();
//tboxUpdater.updateAnnotationModel();
} catch (Exception e) {
log.error("Unable to update default annotation values ", e);
}
}
/**

188
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/TBoxUpdater.java
View file

@ -2,11 +2,22 @@
package edu.cornell.mannlib.vitro.webapp.ontology.update;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -14,6 +25,7 @@ import com.hp.hpl.jena.rdf.model.NodeIterator;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
@ -21,13 +33,18 @@ import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
/**
* Performs knowledge base updates to the tbox to align with a new ontology version
*
*/
* Performs knowledge base updates to the tbox to align with a new ontology version
*
*/
public class TBoxUpdater {
private static final Log log = LogFactory.getLog(TBoxUpdater.class);
private UpdateSettings settings;
private OntModel oldTboxAnnotationsModel;
private OntModel newTboxAnnotationsModel;
private OntModel siteModel;
@ -53,19 +70,33 @@ public class TBoxUpdater {
* and the retractions model.
*
*/
public TBoxUpdater(OntModel oldTboxAnnotationsModel,
OntModel newTboxAnnotationsModel,
OntModel siteModel,
public TBoxUpdater(UpdateSettings settings,
ChangeLogger logger,
ChangeRecord record) {
this.oldTboxAnnotationsModel = oldTboxAnnotationsModel;
this.newTboxAnnotationsModel = newTboxAnnotationsModel;
this.siteModel = siteModel;
this.settings = settings;
this.oldTboxAnnotationsModel = settings.getOldTBoxAnnotationsModel();
this.newTboxAnnotationsModel = settings.getNewTBoxAnnotationsModel();
this.siteModel = settings.getAssertionOntModelSelector().getTBoxModel();
this.logger = logger;
this.record = record;
}
/**
* Update application ontology data for domain and range-qualified properties
* to use any applicable settings from obsolete subproperties
*/
public void modifyPropertyQualifications() throws IOException {
}
private Model mergeConfigurations(Model oldConfig, Model newConfig) {
return null;
}
public void updateDefaultAnnotationValues() throws IOException {
updateDefaultAnnotationValues(null);
}
/**
*
* Update a knowledge base to align with changes to vitro annotation property default
@ -91,7 +122,7 @@ public class TBoxUpdater {
* Note: as specified, this method for now assumes that no new vitro annotation
* properties have been introduced. This should be updated for future versions.
*/
public void updateDefaultAnnotationValues() throws IOException {
public void updateDefaultAnnotationValues(String subjectURI) throws IOException {
siteModel.enterCriticalSection(Lock.WRITE);
@ -109,7 +140,9 @@ public class TBoxUpdater {
return;
}
StmtIterator iter = oldTboxAnnotationsModel.listStatements();
Resource subj = (subjectURI == null) ? null : ResourceFactory.createResource(subjectURI);
StmtIterator iter = oldTboxAnnotationsModel.listStatements(subj, null, (RDFNode) null);
while (iter.hasNext()) {
@ -307,9 +340,9 @@ public class TBoxUpdater {
} finally {
siteModel.leaveCriticalSection();
}
}
}
/**
/**
*
* Update a knowledge base to align with changes to the vitro annotation model
* in a new version of the ontology. The two versions of the ontology and the
@ -322,7 +355,7 @@ public class TBoxUpdater {
* knowledge base.
*
*/
public void updateAnnotationModel() throws IOException {
public void updateAnnotationModel() throws IOException {
// for each ClassGroup in the old vitro annotations model: if it is not in
// the new vitro annotations model and the site has no classes asserted to
@ -364,9 +397,9 @@ public void updateAnnotationModel() throws IOException {
// the old vitro annotations and it is not in the site model, then
// add it.
}
}
public boolean usesGroup(Model model, Resource theClassGroup) throws IOException {
public boolean usesGroup(Model model, Resource theClassGroup) throws IOException {
model.enterCriticalSection(Lock.READ);
@ -375,9 +408,9 @@ public boolean usesGroup(Model model, Resource theClassGroup) throws IOException
} finally {
model.leaveCriticalSection();
}
}
}
public void removeObsoleteAnnotations() throws IOException {
public void removeObsoleteAnnotations() throws IOException {
Resource subj1 = (ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM)).createResource("http://vivoweb.org/ontology/florida#StatewideGoalAndFocusArea");
Resource obj1 = (ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM)).createResource("http://vivoweb.org/ontology#vitroClassGrouptopics");
@ -418,6 +451,121 @@ public void removeObsoleteAnnotations() throws IOException {
} finally {
siteModel.leaveCriticalSection();
}
}
}
public void renameProperty(AtomicOntologyChange changeObj) throws IOException {
if(changeObj.getNotes() != null && changeObj.getNotes().startsWith("cc:")) {
mergePropertyAnnotationsToPropertyConfig(changeObj);
}
}
private void mergePropertyAnnotationsToPropertyConfig(AtomicOntologyChange changeObj) throws IOException {
String contextURI = VitroVocabulary.PROPERTY_CONFIG_DATA + changeObj.getNotes().substring(3);
String oldPropertyURI = changeObj.getSourceURI();
Model oldAnnotationsModel = settings.getOldTBoxAnnotationsModel();
Dataset dataset = new RDFServiceDataset(settings.getRDFService());
Model userAnnotationsModel = dataset.getNamedModel(
JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
String propertyAnnotationsQuery =
"PREFIX config: <" + VitroVocabulary.configURI + "> \n" +
"PREFIX vitro: <" + VitroVocabulary.vitroURI + "> \n" +
"CONSTRUCT { \n" +
" <" + oldPropertyURI + "> vitro:inPropertyGroupAnnot ?group . \n" +
" <" + oldPropertyURI + "> <" + RDFS.label.getURI() + "> ?label . \n" +
" <" + oldPropertyURI + "> vitro:displayRankAnnot ?displayRank . \n" +
" <" + oldPropertyURI + "> vitro:customEntryFormAnnot ?customForm . \n" +
" <" + oldPropertyURI + "> vitro:hiddenFromDisplayBelowRoleLevelAnnot ?displayLevel . \n" +
" <" + oldPropertyURI + "> vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel . \n " +
"} WHERE { \n" +
" { <" + oldPropertyURI + "> vitro:inPropertyGroupAnnot ?group } \n" +
" UNION { <" + oldPropertyURI + "> <" + RDFS.label.getURI() + "> ?label } \n" +
" UNION { <" + oldPropertyURI + "> vitro:displayRankAnnot ?displayRank } \n" +
" UNION { <" + oldPropertyURI + "> vitro:customEntryFormAnnot ?customForm } \n" +
" UNION { <" + oldPropertyURI + "> vitro:hiddenFromDisplayBelowRoleLevelAnnot ?displayLevel } \n" +
" UNION { <" + oldPropertyURI + "> vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel } \n " +
"} \n" ;
Model userChangesModel = construct(
propertyAnnotationsQuery, userAnnotationsModel).difference(
construct(propertyAnnotationsQuery, oldAnnotationsModel));
String addQuery = "PREFIX config: <" + VitroVocabulary.configURI + "> \n" +
"PREFIX vitro: <" + VitroVocabulary.vitroURI + "> \n" +
"CONSTRUCT { \n" +
" ?configuration config:propertyGroup ?group . \n" +
" ?configuration config:displayName ?label . \n" +
" ?configuration vitro:displayRankAnnot ?displayRank . \n" +
" ?configuration vitro:customEntryFormAnnot ?customForm . \n" +
" ?configuration vitro:hiddenFromDisplayBelowRoleLevelAnnot ?displayLevel . \n" +
" ?configuration vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel . \n " +
"} WHERE { \n" +
" <" + contextURI + "> config:hasConfiguration ?configuration . \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:inPropertyGroupAnnot ?group } \n" +
" OPTIONAL { <" + oldPropertyURI + "> <" + RDFS.label.getURI() + "> ?label } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:displayRankAnnot ?displayRank } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:customEntryFormAnnot ?customForm } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:hiddenFromDisplayBelowRoleLevelAnnot ?displayLevel } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel } \n " +
"} \n" ;
String retractQuery = "PREFIX config: <" + VitroVocabulary.configURI + "> \n" +
"PREFIX vitro: <" + VitroVocabulary.vitroURI + "> \n" +
"CONSTRUCT { \n" +
" <" + oldPropertyURI + "> config:propertyGroup ?rgroup . \n" +
" ?configuration config:displayName ?rlabel . \n" +
" ?configuration vitro:displayRankAnnot ?rdisplayRank . \n" +
" ?configuration vitro:customEntryFormAnnot ?rcustomForm . \n" +
" ?configuration vitro:hiddenFromDisplayBelowRoleLevelAnnot ?rdisplayLevel . \n" +
" ?configuration vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?rupdateLevel . \n " +
"} WHERE { \n" +
" <" + contextURI + "> config:hasConfiguration ?configuration . \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:inPropertyGroupAnnot ?group . \n" +
" ?configuration config:propertyGroup ?rgroup } \n" +
" OPTIONAL { <" + oldPropertyURI + "> <" + RDFS.label.getURI() + "> ?label . \n" +
" ?configuration config:displayName ?rlabel } \n " +
" OPTIONAL { <" + oldPropertyURI + "> vitro:displayRankAnnot ?displayRank . \n" +
" ?configuration vitro:displayRantAnnot ?rdisplayRank } \n " +
" OPTIONAL { <" + oldPropertyURI + "> vitro:customEntryFormAnnot ?customForm . \n" +
" ?configuration vitro:customEntryFormAnnot ?rcustomForm } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:hiddenFromDisplayBelowRoleLevelAnnot ?displayLevel . \n" +
" ?configuration vitro:hiddenFromDisplayBelowRoleLevelAnnot ?rdisplayLevel } \n" +
" OPTIONAL { <" + oldPropertyURI + "> vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel . \n " +
" ?configuration vitro:prohibitedFromUpdateBelowRoleLevelAnnot ?updateLevel } " +
"} \n" ;
Model configModel = ModelFactory.createDefaultModel();
String configFileName = settings.getQualifiedPropertyConfigFile();
File file = new File(configFileName);
FileInputStream fis = new FileInputStream(file);
configModel.read(fis, null, "N3");
Model union = ModelFactory.createUnion(configModel,
userChangesModel);
Model additions = construct(addQuery, union);
Model retractions = construct(retractQuery, union);
if (additions.size() > 0 || retractions.size() > 0) {
configModel.remove(retractions);
log.info("Removing " + retractions.size() + " statements from " + contextURI);
configModel.add(additions);
log.info("Adding " + additions.size() + " statements from " + contextURI);
FileOutputStream fos = new FileOutputStream(file);
configModel.write(fos, "N3");
}
}
private Model construct(String queryStr, Model model) {
Query query = QueryFactory.create(queryStr);
QueryExecution qe = QueryExecutionFactory.create(query, model);
try {
return qe.execConstruct();
} finally {
qe.close();
}
}
}

7
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/UpdateSettings.java
View file

@ -21,6 +21,7 @@ public class UpdateSettings {
private String errorLogFile;
private String addedDataFile;
private String removedDataFile;
private String qualifiedPropertyConfigFile;
private String defaultNamespace;
private OntModelSelector assertionOntModelSelector;
private OntModelSelector inferenceOntModelSelector;
@ -121,6 +122,12 @@ public class UpdateSettings {
public void setRemovedDataFile(String removedDataFile) {
this.removedDataFile = removedDataFile;
}
public String getQualifiedPropertyConfigFile() {
return qualifiedPropertyConfigFile;
}
public void setQualifiedPropertyConfigFile(String qualifiedPropertyConfigFile) {
this.qualifiedPropertyConfigFile = qualifiedPropertyConfigFile;
}
public String getDefaultNamespace() {
return defaultNamespace;
}

16
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -96,7 +96,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
settings.setNewTBoxModel(newTBoxModel);
OntModel oldTBoxAnnotationsModel = loadModelFromDirectory(ctx.getRealPath(OLD_TBOX_ANNOTATIONS_DIR));
settings.setOldTBoxAnnotationsModel(oldTBoxAnnotationsModel);
OntModel newTBoxAnnotationsModel = loadModelFromDirectory(createDirectory(homeDir, "rdf", "tbox", "everytime").toString());
OntModel newTBoxAnnotationsModel = loadModelFromDirectory(createDirectory(homeDir, "rdf", "tbox", "firsttime").toString());
settings.setNewTBoxAnnotationsModel(newTBoxAnnotationsModel);
settings.setRDFService(RDFServiceUtils.getRDFServiceFactory(ctx).getRDFService());
@ -145,6 +145,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
log.warn("unable to successfully update display model: " + e.getMessage());
}
}
// reload the display model since the TBoxUpdater may have
// modified it
new ApplicationModelSetup().contextInitialized(sce);
}
} catch (Exception ioe) {
ss.fatal(this, "Exception updating knowledge base for ontology changes: ", ioe);
@ -189,6 +192,17 @@ public class UpdateKnowledgeBase implements ServletContextListener {
Path logDir = createDirectory(dataDir, "logs");
settings.setLogFile(logDir.resolve(timestampedFileName("knowledgeBaseUpdate", "log")).toString());
settings.setErrorLogFile(logDir.resolve(timestampedFileName("knowledgeBaseUpdate.error", "log")).toString());
Path qualifiedPropertyConfigFile = getFilePath(homeDir, "rdf", "display", "everytime", "PropertyConfig.n3");
settings.setQualifiedPropertyConfigFile(qualifiedPropertyConfigFile.toString());
}
private Path getFilePath(Path parent, String... children) throws IOException {
Path path = parent;
for (String child : children) {
path = path.resolve(child);
}
return path;
}
private Path createDirectory(Path parent, String... children) throws IOException {