litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updates to knowledge base migration

Browse source
This commit is contained in:
stellamit 2011-07-01 20:46:08 +00:00
parent 2fcebd1c5f
commit 7f67559db4
5 changed files with 73 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

99
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -28,6 +28,7 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils;
/**
@ -51,65 +52,59 @@ public class KnowledgeBaseUpdater {
this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
}
public boolean update() throws IOException {
public void update() throws IOException {
// Check to see if the update is necessary.
boolean updateRequired = updateRequired();
if (updateRequired) {
if (this.logger == null) {
this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
}
long startTime = System.currentTimeMillis();
System.out.println("Migrating the knowledge base");
log.info("Migrating the knowledge base");
logger.log("Started knowledge base migration");
try {
performUpdate();
} catch (Exception e) {
logger.logError(e.getMessage());
e.printStackTrace();
}
if (!logger.errorsWritten()) {
// add assertions to the knowledge base showing that the
// update was successful, so we don't need to run it again.
assertSuccess();
}
record.writeChanges();
logger.closeLogs();
long elapsedSecs = (System.currentTimeMillis() - startTime)/1000;
System.out.println("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
log.info("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
if (this.logger == null) {
this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
}
return updateRequired;
long startTime = System.currentTimeMillis();
System.out.println("Migrating the knowledge base");
log.info("Migrating the knowledge base");
logger.log("Started knowledge base migration");
try {
performUpdate();
} catch (Exception e) {
logger.logError(e.getMessage());
e.printStackTrace();
}
if (!logger.errorsWritten()) {
// add assertions to the knowledge base showing that the
// update was successful, so we don't need to run it again.
assertSuccess();
}
record.writeChanges();
logger.closeLogs();
long elapsedSecs = (System.currentTimeMillis() - startTime)/1000;
System.out.println("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
log.info("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
return;
}
private void performUpdate() throws IOException {
log.info("\tperforming SPARQL construct additions (abox)");
performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getAssertionOntModelSelector().getABoxModel());
log.info("finished sparql construct additions");
log.info("\tperforming SPARQL construct deletions (infenences)");
performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getInferenceOntModelSelector().getABoxModel());
log.info("finished sparql construct retractions");
List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges();
AtomicOntologyChangeLists changes = new AtomicOntologyChangeLists(rawChanges,settings.getNewTBoxModel(),settings.getOldTBoxModel());
//process the TBox before the ABox
updateTBoxAnnotations();
log.info("finished updating tbox annotations");
log.info("\tupdating tbox annotations");
updateTBoxAnnotations();
log.info("\tupdating the abox");
updateABox(changes);
log.info("finished updating abox");
}
private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel aboxModel) throws IOException {
@ -195,7 +190,7 @@ public class KnowledgeBaseUpdater {
fileContents.append(ln).append('\n');
}
try {
log.info("processing SPARQL construct query from file " + sparqlFiles[i].getName());
log.debug("\t\tprocessing SPARQL construct query from file " + sparqlFiles[i].getName());
Query q = QueryFactory.create(fileContents.toString(), Syntax.syntaxARQ);
aboxModel.enterCriticalSection(Lock.WRITE);
try {
@ -269,7 +264,7 @@ public class KnowledgeBaseUpdater {
boolean required = false;
String sparqlQueryStr = loadSparqlQuery(settings.getAskQueryFile());
String sparqlQueryStr = loadSparqlQuery(settings.getAskUpdatedQueryFile());
if (sparqlQueryStr == null) {
return required;
}
@ -286,12 +281,10 @@ public class KnowledgeBaseUpdater {
required = false;
} else {
required = true;
Model tbox = settings.getAssertionOntModelSelector().getTBoxModel();
String sparqlQueryStr2 = loadSparqlQuery(settings.getAskEmptyQueryFile());
if (sparqlQueryStr2 != null) {
Query query2 = QueryFactory.create(sparqlQueryStr2);
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, tbox);
required = isNotEmpty.execAsk();
if (JenaDataSourceSetupBase.isFirstStartup()) {
assertSuccess();
log.info("The application is starting with a fresh DB, an indication will be added to the DB that a knowledge base migration to the current version is not required.");
required = false;
}
}
@ -328,13 +321,17 @@ public class KnowledgeBaseUpdater {
m.enterCriticalSection(Lock.WRITE);
try {
m.read(inStream, null, settings.getSuccessRDFFormat());
logger.logWithDate("Finished knowledge base migration");
if (logger != null) {
logger.logWithDate("Finished knowledge base migration");
}
} finally {
m.leaveCriticalSection();
}
} catch (Exception e) {
logger.logError(" unable to make RDF assertions about successful " +
if (logger != null) {
logger.logError(" unable to make RDF assertions about successful " +
" update to new ontology version: " + e.getMessage());
}
}
}

25
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/UpdateSettings.java
View file

@ -12,9 +12,7 @@ public class UpdateSettings {
private String sparqlConstructAdditionsDir;
private String sparqlConstructAdditionsPass2Dir;
private String sparqlConstructDeletionsDir;
private String askQueryFile;
private String askEmptyQueryFile;
private String askEverQueryFile;
private String askUpdatedQueryFile;
private String successAssertionsFile;
private String successRDFFormat = "N3";
private String diffFile;
@ -54,23 +52,11 @@ public class UpdateSettings {
public void setSparqlConstructDeletionsDir(String sparqlConstructDeletionsDir) {
this.sparqlConstructDeletionsDir = sparqlConstructDeletionsDir;
}
public String getAskQueryFile() {
return askQueryFile;
public String getAskUpdatedQueryFile() {
return askUpdatedQueryFile;
}
public void setAskQueryFile(String askQueryFile) {
this.askQueryFile = askQueryFile;
}
public String getAskEverQueryFile() {
return askEverQueryFile;
}
public void setAskEverQueryFile(String askEverQueryFile) {
this.askEverQueryFile = askEverQueryFile;
}
public String getAskEmptyQueryFile() {
return askEmptyQueryFile;
}
public void setAskEmptyQueryFile(String askEmptyQueryFile) {
this.askEmptyQueryFile = askEmptyQueryFile;
public void setAskUpdatedQueryFile(String askQueryFile) {
this.askUpdatedQueryFile = askQueryFile;
}
public String getSuccessAssertionsFile() {
return successAssertionsFile;
@ -156,5 +142,4 @@ public class UpdateSettings {
public void setNewTBoxAnnotationsModel(OntModel newTBoxAnnotationsModel) {
this.newTBoxAnnotationsModel = newTBoxAnnotationsModel;
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java
View file

@ -86,7 +86,7 @@ public class FileGraphSetup implements ServletContextListener {
if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext()) ) {
log.info("a full recompute of the Abox will be performed because" +
" the filegraph abox(s) and/or tbox(s) have changed." );
" the filegraph abox(s) and/or tbox(s) have changed or are being read for the first time." );
SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext());
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -259,7 +259,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
RDB, SDB
}
protected boolean isFirstStartup() {
public static boolean isFirstStartup() {
return firstStartup;
}

42
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -48,8 +48,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
private static final String DATA_DIR = "/WEB-INF/ontologies/update/";
private static final String LOG_DIR = "logs/";
private static final String CHANGED_DATA_DIR = "changedData/";
private static final String ASK_QUERY_FILE = DATA_DIR + "ask.sparql";
private static final String ASK_EMPTY_QUERY_FILE = DATA_DIR + "askEmpty.sparql";
private static final String ASK_QUERY_FILE = DATA_DIR + "askUpdated.sparql";
private static final String SUCCESS_ASSERTIONS_FILE = DATA_DIR + "success.n3";
private static final String SUCCESS_RDF_FORMAT = "N3";
private static final String DIFF_FILE = DATA_DIR + "diff.tab.txt";
@ -77,8 +76,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
String errorLogFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate.error", "log");
UpdateSettings settings = new UpdateSettings();
settings.setAskQueryFile(getAskQueryPath(ctx));
settings.setAskEmptyQueryFile(getAskEmptyQueryPath(ctx));
settings.setAskUpdatedQueryFile(getAskUpdatedQueryPath(ctx));
settings.setDataDir(ctx.getRealPath(DATA_DIR));
settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR));
settings.setSparqlConstructDeletionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_DELETIONS_DIR));
@ -110,32 +108,28 @@ public class UpdateKnowledgeBase implements ServletContextListener {
}
try {
KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
try {
try {
if (ontologyUpdater.updateRequired()) {
ctx.setAttribute(IndexConstants.INDEX_REBUILD_REQUESTED_AT_STARTUP, Boolean.TRUE);
ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE);
//doMiscAppMetadataReplacements(ctx.getRealPath(MISC_REPLACEMENTS_FILE), oms);
reloadDisplayModel(ctx);
ontologyUpdater.update();
}
} catch (Throwable t){
} catch (IOException ioe) {
String errMsg = "IOException updating knowledge base " +
"for ontology changes: ";
// Tomcat doesn't always seem to print exceptions thrown from
// context listeners
System.out.println(errMsg);
ioe.printStackTrace();
throw new RuntimeException(errMsg, ioe);
}
} catch (Throwable t){
log.warn("warning", t);
}
ontologyUpdater.update();
} catch (IOException ioe) {
String errMsg = "IOException updating knowledge base " +
"for ontology changes: ";
// Tomcat doesn't always seem to print exceptions thrown from
// context listeners
System.out.println(errMsg);
ioe.printStackTrace();
throw new RuntimeException(errMsg, ioe);
}
} catch (Throwable t) {
t.printStackTrace();
}
@ -244,13 +238,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
// nothing to do
}
public static String getAskQueryPath(ServletContext ctx) {
public static String getAskUpdatedQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_QUERY_FILE);
}
public static String getAskEmptyQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_EMPTY_QUERY_FILE);
}
private static String timestampedFileName(String prefix, String suffix) {