litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

updates to knowledge base migration

Browse source
This commit is contained in:
stellamit 2011-07-01 20:46:08 +00:00
parent 2fcebd1c5f
commit 7f67559db4
5 changed files with 73 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

43
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -28,6 +28,7 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator; import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock; import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils; import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils;
/** /**
@ -51,18 +52,12 @@ public class KnowledgeBaseUpdater {
this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile()); this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
} }
public boolean update() throws IOException { public void update() throws IOException {
// Check to see if the update is necessary.
boolean updateRequired = updateRequired();
if (updateRequired) {
if (this.logger == null) { if (this.logger == null) {
this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile()); this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
} }
long startTime = System.currentTimeMillis(); long startTime = System.currentTimeMillis();
System.out.println("Migrating the knowledge base"); System.out.println("Migrating the knowledge base");
log.info("Migrating the knowledge base"); log.info("Migrating the knowledge base");
@ -87,29 +82,29 @@ public class KnowledgeBaseUpdater {
long elapsedSecs = (System.currentTimeMillis() - startTime)/1000; long elapsedSecs = (System.currentTimeMillis() - startTime)/1000;
System.out.println("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : "")); System.out.println("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
log.info("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : "")); log.info("Finished knowledge base migration in " + elapsedSecs + " second" + (elapsedSecs != 1 ? "s" : ""));
return;
} }
return updateRequired;
}
private void performUpdate() throws IOException { private void performUpdate() throws IOException {
log.info("\tperforming SPARQL construct additions (abox)");
performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getAssertionOntModelSelector().getABoxModel()); performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getAssertionOntModelSelector().getABoxModel());
log.info("finished sparql construct additions"); log.info("\tperforming SPARQL construct deletions (infenences)");
performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getInferenceOntModelSelector().getABoxModel()); performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getInferenceOntModelSelector().getABoxModel());
log.info("finished sparql construct retractions");
List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges(); List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges();
AtomicOntologyChangeLists changes = new AtomicOntologyChangeLists(rawChanges,settings.getNewTBoxModel(),settings.getOldTBoxModel()); AtomicOntologyChangeLists changes = new AtomicOntologyChangeLists(rawChanges,settings.getNewTBoxModel(),settings.getOldTBoxModel());
//process the TBox before the ABox //process the TBox before the ABox
updateTBoxAnnotations();
log.info("finished updating tbox annotations");
log.info("\tupdating tbox annotations");
updateTBoxAnnotations();
log.info("\tupdating the abox");
updateABox(changes); updateABox(changes);
log.info("finished updating abox");
} }
private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel aboxModel) throws IOException { private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel aboxModel) throws IOException {
@ -195,7 +190,7 @@ public class KnowledgeBaseUpdater {
fileContents.append(ln).append('\n'); fileContents.append(ln).append('\n');
} }
try { try {
log.info("processing SPARQL construct query from file " + sparqlFiles[i].getName()); log.debug("\t\tprocessing SPARQL construct query from file " + sparqlFiles[i].getName());
Query q = QueryFactory.create(fileContents.toString(), Syntax.syntaxARQ); Query q = QueryFactory.create(fileContents.toString(), Syntax.syntaxARQ);
aboxModel.enterCriticalSection(Lock.WRITE); aboxModel.enterCriticalSection(Lock.WRITE);
try { try {
@ -269,7 +264,7 @@ public class KnowledgeBaseUpdater {
boolean required = false; boolean required = false;
String sparqlQueryStr = loadSparqlQuery(settings.getAskQueryFile()); String sparqlQueryStr = loadSparqlQuery(settings.getAskUpdatedQueryFile());
if (sparqlQueryStr == null) { if (sparqlQueryStr == null) {
return required; return required;
} }
@ -286,12 +281,10 @@ public class KnowledgeBaseUpdater {
required = false; required = false;
} else { } else {
required = true; required = true;
Model tbox = settings.getAssertionOntModelSelector().getTBoxModel(); if (JenaDataSourceSetupBase.isFirstStartup()) {
String sparqlQueryStr2 = loadSparqlQuery(settings.getAskEmptyQueryFile()); assertSuccess();
if (sparqlQueryStr2 != null) { log.info("The application is starting with a fresh DB, an indication will be added to the DB that a knowledge base migration to the current version is not required.");
Query query2 = QueryFactory.create(sparqlQueryStr2); required = false;
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, tbox);
required = isNotEmpty.execAsk();
} }
} }
@ -328,15 +321,19 @@ public class KnowledgeBaseUpdater {
m.enterCriticalSection(Lock.WRITE); m.enterCriticalSection(Lock.WRITE);
try { try {
m.read(inStream, null, settings.getSuccessRDFFormat()); m.read(inStream, null, settings.getSuccessRDFFormat());
if (logger != null) {
logger.logWithDate("Finished knowledge base migration"); logger.logWithDate("Finished knowledge base migration");
}
} finally { } finally {
m.leaveCriticalSection(); m.leaveCriticalSection();
} }
} catch (Exception e) { } catch (Exception e) {
if (logger != null) {
logger.logError(" unable to make RDF assertions about successful " + logger.logError(" unable to make RDF assertions about successful " +
" update to new ontology version: " + e.getMessage()); " update to new ontology version: " + e.getMessage());
} }
} }
}
/** /**
* A class that allows to access two different ontology change lists, * A class that allows to access two different ontology change lists,

25
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/UpdateSettings.java
View file

@ -12,9 +12,7 @@ public class UpdateSettings {
private String sparqlConstructAdditionsDir; private String sparqlConstructAdditionsDir;
private String sparqlConstructAdditionsPass2Dir; private String sparqlConstructAdditionsPass2Dir;
private String sparqlConstructDeletionsDir; private String sparqlConstructDeletionsDir;
private String askQueryFile; private String askUpdatedQueryFile;
private String askEmptyQueryFile;
private String askEverQueryFile;
private String successAssertionsFile; private String successAssertionsFile;
private String successRDFFormat = "N3"; private String successRDFFormat = "N3";
private String diffFile; private String diffFile;
@ -54,23 +52,11 @@ public class UpdateSettings {
public void setSparqlConstructDeletionsDir(String sparqlConstructDeletionsDir) { public void setSparqlConstructDeletionsDir(String sparqlConstructDeletionsDir) {
this.sparqlConstructDeletionsDir = sparqlConstructDeletionsDir; this.sparqlConstructDeletionsDir = sparqlConstructDeletionsDir;
} }
public String getAskQueryFile() { public String getAskUpdatedQueryFile() {
return askQueryFile; return askUpdatedQueryFile;
} }
public void setAskQueryFile(String askQueryFile) { public void setAskUpdatedQueryFile(String askQueryFile) {
this.askQueryFile = askQueryFile; this.askUpdatedQueryFile = askQueryFile;
}
public String getAskEverQueryFile() {
return askEverQueryFile;
}
public void setAskEverQueryFile(String askEverQueryFile) {
this.askEverQueryFile = askEverQueryFile;
}
public String getAskEmptyQueryFile() {
return askEmptyQueryFile;
}
public void setAskEmptyQueryFile(String askEmptyQueryFile) {
this.askEmptyQueryFile = askEmptyQueryFile;
} }
public String getSuccessAssertionsFile() { public String getSuccessAssertionsFile() {
return successAssertionsFile; return successAssertionsFile;
@ -156,5 +142,4 @@ public class UpdateSettings {
public void setNewTBoxAnnotationsModel(OntModel newTBoxAnnotationsModel) { public void setNewTBoxAnnotationsModel(OntModel newTBoxAnnotationsModel) {
this.newTBoxAnnotationsModel = newTBoxAnnotationsModel; this.newTBoxAnnotationsModel = newTBoxAnnotationsModel;
} }
} }

2
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java
View file

@ -86,7 +86,7 @@ public class FileGraphSetup implements ServletContextListener {
if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext()) ) { if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext()) ) {
log.info("a full recompute of the Abox will be performed because" + log.info("a full recompute of the Abox will be performed because" +
" the filegraph abox(s) and/or tbox(s) have changed." ); " the filegraph abox(s) and/or tbox(s) have changed or are being read for the first time." );
SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext()); SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext());
} }
} }

2
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -259,7 +259,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
RDB, SDB RDB, SDB
} }
protected boolean isFirstStartup() { public static boolean isFirstStartup() {
return firstStartup; return firstStartup;
} }

24
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -48,8 +48,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
private static final String DATA_DIR = "/WEB-INF/ontologies/update/"; private static final String DATA_DIR = "/WEB-INF/ontologies/update/";
private static final String LOG_DIR = "logs/"; private static final String LOG_DIR = "logs/";
private static final String CHANGED_DATA_DIR = "changedData/"; private static final String CHANGED_DATA_DIR = "changedData/";
private static final String ASK_QUERY_FILE = DATA_DIR + "ask.sparql"; private static final String ASK_QUERY_FILE = DATA_DIR + "askUpdated.sparql";
private static final String ASK_EMPTY_QUERY_FILE = DATA_DIR + "askEmpty.sparql";
private static final String SUCCESS_ASSERTIONS_FILE = DATA_DIR + "success.n3"; private static final String SUCCESS_ASSERTIONS_FILE = DATA_DIR + "success.n3";
private static final String SUCCESS_RDF_FORMAT = "N3"; private static final String SUCCESS_RDF_FORMAT = "N3";
private static final String DIFF_FILE = DATA_DIR + "diff.tab.txt"; private static final String DIFF_FILE = DATA_DIR + "diff.tab.txt";
@ -77,8 +76,7 @@ public class UpdateKnowledgeBase implements ServletContextListener {
String errorLogFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate.error", "log"); String errorLogFileName = DATA_DIR + LOG_DIR + timestampedFileName("knowledgeBaseUpdate.error", "log");
UpdateSettings settings = new UpdateSettings(); UpdateSettings settings = new UpdateSettings();
settings.setAskQueryFile(getAskQueryPath(ctx)); settings.setAskUpdatedQueryFile(getAskUpdatedQueryPath(ctx));
settings.setAskEmptyQueryFile(getAskEmptyQueryPath(ctx));
settings.setDataDir(ctx.getRealPath(DATA_DIR)); settings.setDataDir(ctx.getRealPath(DATA_DIR));
settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR)); settings.setSparqlConstructAdditionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_ADDITIONS_DIR));
settings.setSparqlConstructDeletionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_DELETIONS_DIR)); settings.setSparqlConstructDeletionsDir(ctx.getRealPath(SPARQL_CONSTRUCT_DELETIONS_DIR));
@ -110,7 +108,6 @@ public class UpdateKnowledgeBase implements ServletContextListener {
} }
try { try {
KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings); KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
try { try {
@ -119,13 +116,8 @@ public class UpdateKnowledgeBase implements ServletContextListener {
ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE); ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE);
//doMiscAppMetadataReplacements(ctx.getRealPath(MISC_REPLACEMENTS_FILE), oms); //doMiscAppMetadataReplacements(ctx.getRealPath(MISC_REPLACEMENTS_FILE), oms);
reloadDisplayModel(ctx); reloadDisplayModel(ctx);
}
} catch (Throwable t){
log.warn("warning", t);
}
ontologyUpdater.update(); ontologyUpdater.update();
}
} catch (IOException ioe) { } catch (IOException ioe) {
String errMsg = "IOException updating knowledge base " + String errMsg = "IOException updating knowledge base " +
"for ontology changes: "; "for ontology changes: ";
@ -135,7 +127,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
ioe.printStackTrace(); ioe.printStackTrace();
throw new RuntimeException(errMsg, ioe); throw new RuntimeException(errMsg, ioe);
} }
} catch (Throwable t){
log.warn("warning", t);
}
} catch (Throwable t) { } catch (Throwable t) {
t.printStackTrace(); t.printStackTrace();
} }
@ -244,13 +238,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
// nothing to do // nothing to do
} }
public static String getAskQueryPath(ServletContext ctx) { public static String getAskUpdatedQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_QUERY_FILE); return ctx.getRealPath(ASK_QUERY_FILE);
}
public static String getAskEmptyQueryPath(ServletContext ctx) {
return ctx.getRealPath(ASK_EMPTY_QUERY_FILE);
} }
private static String timestampedFileName(String prefix, String suffix) { private static String timestampedFileName(String prefix, String suffix) {