litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-2426 knowledge base migration

Browse source
This commit is contained in:
stellamit 2011-06-29 18:43:42 +00:00
parent 32e69d257d
commit 19f69c62ba
6 changed files with 49 additions and 103 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

58
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -90,8 +90,8 @@ public class KnowledgeBaseUpdater {
private void performUpdate() throws IOException { private void performUpdate() throws IOException {
performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getOntModelSelector().getABoxModel()); performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getAssertionOntModelSelector().getABoxModel());
performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getOntModelSelector().getABoxModel()); performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getInferenceOntModelSelector().getABoxModel());
List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges(); List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges();
@ -127,49 +127,27 @@ public class KnowledgeBaseUpdater {
aboxModel.add(actualAdditions); aboxModel.add(actualAdditions);
record.recordAdditions(actualAdditions); record.recordAdditions(actualAdditions);
/*
if (actualAdditions.size() > 0) {
logger.log("Constructed " + actualAdditions.size() + " new " +
"statement"
+ ((actualAdditions.size() > 1) ? "s" : "") +
" using SPARQL construct queries.");
}
*/
} finally { } finally {
aboxModel.leaveCriticalSection(); aboxModel.leaveCriticalSection();
} }
} }
private void performSparqlConstructRetractions(String sparqlConstructDir, OntModel aboxModel) throws IOException { private void performSparqlConstructRetractions(String sparqlConstructDir, OntModel model) throws IOException {
Model retractions = performSparqlConstructs(sparqlConstructDir, aboxModel, false); Model retractions = performSparqlConstructs(sparqlConstructDir, model, false);
if (retractions == null) { if (retractions == null) {
return; return;
} }
aboxModel.enterCriticalSection(Lock.WRITE); model.enterCriticalSection(Lock.WRITE);
try {
Model actualRetractions = ModelFactory.createDefaultModel();
StmtIterator stmtIt = retractions.listStatements();
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (aboxModel.contains(stmt)) {
actualRetractions.add(stmt);
}
}
aboxModel.remove(actualRetractions);
record.recordRetractions(actualRetractions);
/*
if (actualRetractions.size() > 0) {
logger.log("Removed " + actualRetractions.size() + " statement" + ((actualRetractions.size() > 1) ? "s" : "") + " using SPARQL CONSTRUCT queries.");
}
*/
try {
model.remove(retractions);
record.recordRetractions(retractions);
} finally { } finally {
aboxModel.leaveCriticalSection(); model.leaveCriticalSection();
} }
} }
@ -218,7 +196,7 @@ public class KnowledgeBaseUpdater {
long num = numAfter - numBefore; long num = numAfter - numBefore;
if (num > 0) { if (num > 0) {
logger.log((add ? "Added " : "Removed ") + num + logger.log((add ? "Added " : "Removed ") + num + (add ? "" : " inferred") +
" statement" + ((num > 1) ? "s" : "") + " statement" + ((num > 1) ? "s" : "") +
" using the SPARQL construct query from file " + sparqlFiles[i].getName()); " using the SPARQL construct query from file " + sparqlFiles[i].getName());
} }
@ -254,7 +232,7 @@ public class KnowledgeBaseUpdater {
OntModel oldTBoxModel = settings.getOldTBoxModel(); OntModel oldTBoxModel = settings.getOldTBoxModel();
OntModel newTBoxModel = settings.getNewTBoxModel(); OntModel newTBoxModel = settings.getNewTBoxModel();
OntModel ABoxModel = settings.getOntModelSelector().getABoxModel(); OntModel ABoxModel = settings.getAssertionOntModelSelector().getABoxModel();
ABoxUpdater aboxUpdater = new ABoxUpdater( ABoxUpdater aboxUpdater = new ABoxUpdater(
oldTBoxModel, newTBoxModel, ABoxModel, oldTBoxModel, newTBoxModel, ABoxModel,
settings.getNewTBoxAnnotationsModel(), logger, record); settings.getNewTBoxAnnotationsModel(), logger, record);
@ -267,10 +245,10 @@ public class KnowledgeBaseUpdater {
TBoxUpdater tboxUpdater = new TBoxUpdater(settings.getOldTBoxAnnotationsModel(), TBoxUpdater tboxUpdater = new TBoxUpdater(settings.getOldTBoxAnnotationsModel(),
settings.getNewTBoxAnnotationsModel(), settings.getNewTBoxAnnotationsModel(),
settings.getOntModelSelector().getABoxModel(), logger, record); settings.getAssertionOntModelSelector().getTBoxModel(), logger, record);
tboxUpdater.updateDefaultAnnotationValues(); tboxUpdater.updateDefaultAnnotationValues();
tboxUpdater.updateAnnotationModel(); //tboxUpdater.updateAnnotationModel();
} }
/** /**
@ -286,9 +264,9 @@ public class KnowledgeBaseUpdater {
return required; return required;
} }
Model m = settings.getOntModelSelector().getApplicationMetadataModel(); Model abox = settings.getAssertionOntModelSelector().getABoxModel();
Query query = QueryFactory.create(sparqlQueryStr); Query query = QueryFactory.create(sparqlQueryStr);
QueryExecution isUpdated = QueryExecutionFactory.create(query, m); QueryExecution isUpdated = QueryExecutionFactory.create(query, abox);
// if the ASK query DOES have a solution (i.e. the assertions exist // if the ASK query DOES have a solution (i.e. the assertions exist
// showing that the update has already been performed), then the update // showing that the update has already been performed), then the update
@ -298,10 +276,11 @@ public class KnowledgeBaseUpdater {
required = false; required = false;
} else { } else {
required = true; required = true;
Model tbox = settings.getAssertionOntModelSelector().getTBoxModel();
String sparqlQueryStr2 = loadSparqlQuery(settings.getAskEmptyQueryFile()); String sparqlQueryStr2 = loadSparqlQuery(settings.getAskEmptyQueryFile());
if (sparqlQueryStr2 != null) { if (sparqlQueryStr2 != null) {
Query query2 = QueryFactory.create(sparqlQueryStr2); Query query2 = QueryFactory.create(sparqlQueryStr2);
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, m); QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, tbox);
required = isNotEmpty.execAsk(); required = isNotEmpty.execAsk();
} }
} }
@ -332,7 +311,8 @@ public class KnowledgeBaseUpdater {
private void assertSuccess() throws FileNotFoundException, IOException { private void assertSuccess() throws FileNotFoundException, IOException {
try { try {
Model m = settings.getOntModelSelector().getApplicationMetadataModel(); //Model m = settings.getAssertionOntModelSelector().getApplicationMetadataModel();
Model m = settings.getAssertionOntModelSelector().getABoxModel();
File successAssertionsFile = new File(settings.getSuccessAssertionsFile()); File successAssertionsFile = new File(settings.getSuccessAssertionsFile());
InputStream inStream = new FileInputStream(successAssertionsFile); InputStream inStream = new FileInputStream(successAssertionsFile);
m.enterCriticalSection(Lock.WRITE); m.enterCriticalSection(Lock.WRITE);

2
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/TBoxUpdater.java
View file

@ -206,10 +206,12 @@ public class TBoxUpdater {
} }
if (i > 1) { if (i > 1) {
/*
logger.log("WARNING: found " + i + logger.log("WARNING: found " + i +
" statements with subject = " + subject.getURI() + " statements with subject = " + subject.getURI() +
" and property = " + predicate.getURI() + " and property = " + predicate.getURI() +
" in the site annotations model (maximum of one is expected) "); " in the site annotations model (maximum of one is expected) ");
*/
continue; continue;
} }

17
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/UpdateSettings.java
View file

@ -23,7 +23,8 @@ public class UpdateSettings {
private String addedDataFile; private String addedDataFile;
private String removedDataFile; private String removedDataFile;
private String defaultNamespace; private String defaultNamespace;
private OntModelSelector ontModelSelector; private OntModelSelector assertionOntModelSelector;
private OntModelSelector inferenceOntModelSelector;
private OntModel oldTBoxModel; private OntModel oldTBoxModel;
private OntModel newTBoxModel; private OntModel newTBoxModel;
private OntModel oldTBoxAnnotationsModel; private OntModel oldTBoxAnnotationsModel;
@ -89,8 +90,11 @@ public class UpdateSettings {
public void setDiffFile(String diffFile) { public void setDiffFile(String diffFile) {
this.diffFile = diffFile; this.diffFile = diffFile;
} }
public OntModelSelector getOntModelSelector() { public OntModelSelector getAssertionOntModelSelector() {
return ontModelSelector; return assertionOntModelSelector;
}
public OntModelSelector getInferenceOntModelSelector() {
return inferenceOntModelSelector;
} }
public String getLogFile() { public String getLogFile() {
return logFile; return logFile;
@ -122,8 +126,11 @@ public class UpdateSettings {
public void setDefaultNamespace(String defaultNamespace) { public void setDefaultNamespace(String defaultNamespace) {
this.defaultNamespace = defaultNamespace; this.defaultNamespace = defaultNamespace;
} }
public void setOntModelSelector(OntModelSelector ontModelSelector) { public void setAssertionOntModelSelector(OntModelSelector ontModelSelector) {
this.ontModelSelector = ontModelSelector; this.assertionOntModelSelector = ontModelSelector;
}
public void setInferenceOntModelSelector(OntModelSelector ontModelSelector) {
this.inferenceOntModelSelector = ontModelSelector;
} }
public OntModel getOldTBoxModel() { public OntModel getOldTBoxModel() {
return oldTBoxModel; return oldTBoxModel;

16
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java
View file

@ -12,6 +12,7 @@ import java.util.Set;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener; import javax.servlet.ServletContextListener;
@ -83,21 +84,15 @@ public class FileGraphSetup implements ServletContextListener {
t.printStackTrace(); t.printStackTrace();
} }
if (aboxChanged || tboxChanged) { if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext()) ) {
if ( !JenaDataSourceSetup.updateRequired(sce.getServletContext(), baseOms.getTBoxModel())) {
log.info("a full recompute of the Abox will be performed because" + log.info("a full recompute of the Abox will be performed because" +
" the filegraph abox(s) and/or tbox(s) have changed, or are being read for the first time." ); " the filegraph abox(s) and/or tbox(s) have changed." );
SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext()); SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext());
} else {
log.info("A knowledgebase update is required. A full recompute of the Abox will not be" +
" performed; instead inferences will be updated incrementally as the knowledge base is updated.");
}
} }
} }
/* /*
* Reads the graphs stored as files in sub-directories of * Reads the graphs stored as files in sub-directories of
* FileGraphSetup.PATH_ROOT and for each graph:
* 1. updates the SDB store to reflect the current contents of the graph. * 1. updates the SDB store to reflect the current contents of the graph.
* 2. adds the graph as an in-memory submodel of the base in-memory graph * 2. adds the graph as an in-memory submodel of the base in-memory graph
* *
@ -255,4 +250,9 @@ public class FileGraphSetup implements ServletContextListener {
public void contextDestroyed( ServletContextEvent sce ) { public void contextDestroyed( ServletContextEvent sce ) {
// nothing to do // nothing to do
} }
private static boolean isUpdateRequired(ServletContext ctx) {
return (ctx.getAttribute(UpdateKnowledgeBase.KBM_REQURIED_AT_STARTUP) != null);
}
} }

40
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetup.java
View file

@ -593,45 +593,5 @@ public class JenaDataSourceSetup extends JenaDataSourceSetupBase implements java
public static Store getApplicationStore(ServletContext ctx) { public static Store getApplicationStore(ServletContext ctx) {
return (Store) ctx.getAttribute(STORE_ATTR); return (Store) ctx.getAttribute(STORE_ATTR);
} }
/**
* Executes a SPARQL ASK query to determine whether the knowledge base
* needs to be updated to conform to a new ontology version
*/
public static boolean updateRequired(ServletContext ctx, OntModel m) {
boolean required = false;
try {
String sparqlQueryStr = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskQueryPath(ctx));
if (sparqlQueryStr == null) {
return required;
}
Query query = QueryFactory.create(sparqlQueryStr);
QueryExecution isUpdated = QueryExecutionFactory.create(query, m);
// if the ASK query DOES have a solution (i.e. the assertions exist
// showing that the update has already been performed), then the update
// is NOT required.
if (isUpdated.execAsk()) {
required = false;
} else {
required = true;
String sparqlQueryStr2 = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskEmptyQueryPath(ctx));
if (sparqlQueryStr2 != null) {
Query query2 = QueryFactory.create(sparqlQueryStr2);
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, m);
required = isNotEmpty.execAsk();
}
}
} catch (IOException e) {
log.error("error while trying to determine if a knowledgbase update is required", e);
return false;
}
return required;
}
} }

13
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -42,6 +42,7 @@ import edu.cornell.mannlib.vitro.webapp.search.IndexConstants;
*/ */
public class UpdateKnowledgeBase implements ServletContextListener { public class UpdateKnowledgeBase implements ServletContextListener {
public static final String KBM_REQURIED_AT_STARTUP = "KNOWLEDGE_BASE_MIGRATION_REQUIRED_AT_STARTUP";
private final static Log log = LogFactory.getLog(UpdateKnowledgeBase.class); private final static Log log = LogFactory.getLog(UpdateKnowledgeBase.class);
private static final String DATA_DIR = "/WEB-INF/ontologies/update/"; private static final String DATA_DIR = "/WEB-INF/ontologies/update/";
@ -49,7 +50,6 @@ public class UpdateKnowledgeBase implements ServletContextListener {
private static final String CHANGED_DATA_DIR = "changedData/"; private static final String CHANGED_DATA_DIR = "changedData/";
private static final String ASK_QUERY_FILE = DATA_DIR + "ask.sparql"; private static final String ASK_QUERY_FILE = DATA_DIR + "ask.sparql";
private static final String ASK_EMPTY_QUERY_FILE = DATA_DIR + "askEmpty.sparql"; private static final String ASK_EMPTY_QUERY_FILE = DATA_DIR + "askEmpty.sparql";
private static final String ASK_EVER_QUERY_FILE = DATA_DIR + "askEver.sparql";
private static final String SUCCESS_ASSERTIONS_FILE = DATA_DIR + "success.n3"; private static final String SUCCESS_ASSERTIONS_FILE = DATA_DIR + "success.n3";
private static final String SUCCESS_RDF_FORMAT = "N3"; private static final String SUCCESS_RDF_FORMAT = "N3";
private static final String DIFF_FILE = DATA_DIR + "diff.tab.txt"; private static final String DIFF_FILE = DATA_DIR + "diff.tab.txt";
@ -72,14 +72,12 @@ public class UpdateKnowledgeBase implements ServletContextListener {
try { try {
ServletContext ctx = sce.getServletContext(); ServletContext ctx = sce.getServletContext();
OntModelSelector assertionsOms = ModelContext.getBaseOntModelSelector(ctx);
String logFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate", "log"); String logFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate", "log");
String errorLogFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate.error", "log"); String errorLogFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate.error", "log");
UpdateSettings settings = new UpdateSettings(); UpdateSettings settings = new UpdateSettings();
settings.setAskQueryFile(getAskQueryPath(ctx)); settings.setAskQueryFile(getAskQueryPath(ctx));
settings.setAskEverQueryFile(getAskEverQueryPath(ctx));
settings.setAskEmptyQueryFile(getAskEmptyQueryPath(ctx)); settings.setAskEmptyQueryFile(getAskEmptyQueryPath(ctx));
settings.setDataDir(ctx.getRealPath(DATA_DIR)); settings.setDataDir(ctx.getRealPath(DATA_DIR));
settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR)); settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR));
@ -93,8 +91,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
settings.setRemovedDataFile(ctx.getRealPath(REMOVED_DATA_FILE)); settings.setRemovedDataFile(ctx.getRealPath(REMOVED_DATA_FILE));
WebappDaoFactory wadf = (WebappDaoFactory) ctx.getAttribute("webappDaoFactory"); WebappDaoFactory wadf = (WebappDaoFactory) ctx.getAttribute("webappDaoFactory");
settings.setDefaultNamespace(wadf.getDefaultNamespace()); settings.setDefaultNamespace(wadf.getDefaultNamespace());
settings.setAssertionOntModelSelector(ModelContext.getBaseOntModelSelector(ctx));
settings.setInferenceOntModelSelector(ModelContext.getInferenceOntModelSelector(ctx));
settings.setOntModelSelector(assertionsOms);
try { try {
OntModel oldTBoxModel = loadModelFromDirectory(ctx.getRealPath(OLD_TBOX_MODEL_DIR)); OntModel oldTBoxModel = loadModelFromDirectory(ctx.getRealPath(OLD_TBOX_MODEL_DIR));
settings.setOldTBoxModel(oldTBoxModel); settings.setOldTBoxModel(oldTBoxModel);
@ -117,6 +116,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
try { try {
if (ontologyUpdater.updateRequired()) { if (ontologyUpdater.updateRequired()) {
ctx.setAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP, Boolean.TRUE); ctx.setAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP, Boolean.TRUE);
ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE);
//doMiscAppMetadataReplacements(ctx.getRealPath(MISC_REPLACEMENTS_FILE), oms); //doMiscAppMetadataReplacements(ctx.getRealPath(MISC_REPLACEMENTS_FILE), oms);
reloadDisplayModel(ctx); reloadDisplayModel(ctx);
} }
@ -243,17 +243,14 @@ public class UpdateKnowledgeBase implements ServletContextListener {
public void contextDestroyed(ServletContextEvent arg0) { public void contextDestroyed(ServletContextEvent arg0) {
// nothing to do // nothing to do
} }
public static String getAskQueryPath(ServletContext ctx) { public static String getAskQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_QUERY_FILE); return ctx.getRealPath(ASK_QUERY_FILE);
} }
public static String getAskEverQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_EVER_QUERY_FILE);
}
public static String getAskEmptyQueryPath(ServletContext ctx) { public static String getAskEmptyQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_EMPTY_QUERY_FILE); return ctx.getRealPath(ASK_EMPTY_QUERY_FILE);
} }
private static String timestampedFileName(String prefix, String suffix) { private static String timestampedFileName(String prefix, String suffix) {