litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3673, updates for KB migration, and RDFService fix for inputstreams that don't support mark()/reset()

Browse source
This commit is contained in:
stellamit 2012-06-19 17:49:02 +00:00
parent 496d04243e
commit 4b00e781b4
8 changed files with 62 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

91
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java
View file

@ -13,6 +13,8 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -28,6 +30,10 @@ import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils;
@ -49,7 +55,7 @@ public class KnowledgeBaseUpdater {
this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
}
public void update() throws IOException {
public void update(ServletContext servletContext) throws IOException {
if (this.logger == null) {
this.logger = new SimpleChangeLogger(settings.getLogFile(), settings.getErrorLogFile());
@ -68,9 +74,8 @@ public class KnowledgeBaseUpdater {
}
if (!logger.errorsWritten()) {
// add assertions to the knowledge base showing that the
// update was successful, so we don't need to run it again.
assertSuccess();
assertSuccess(servletContext);
logger.logWithDate("Finished knowledge base migration");
}
record.writeChanges();
@ -86,22 +91,9 @@ public class KnowledgeBaseUpdater {
private void performUpdate() throws IOException {
log.info("\tperforming SPARQL construct additions (abox)");
performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getUnionOntModelSelector().getABoxModel() , settings.getAssertionOntModelSelector().getABoxModel());
log.info("\tperforming SPARQL construct deletions (abox)");
performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getUnionOntModelSelector().getABoxModel() , settings.getAssertionOntModelSelector().getABoxModel());
List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges();
AtomicOntologyChangeLists changes = new AtomicOntologyChangeLists(rawChanges,settings.getNewTBoxModel(),settings.getOldTBoxModel());
//process the TBox before the ABox
// only annotations for 1.5
log.info("\tupdating tbox annotations");
updateTBoxAnnotations();
log.info("\tupdating the abox");
updateABox(changes);
}
private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel readModel, OntModel writeModel) throws IOException {
@ -254,7 +246,7 @@ public class KnowledgeBaseUpdater {
* Executes a SPARQL ASK query to determine whether the knowledge base
* needs to be updated to conform to a new ontology version
*/
public boolean updateRequired() throws IOException {
public boolean updateRequired(ServletContext servletContext) throws IOException {
boolean required = false;
@ -262,27 +254,28 @@ public class KnowledgeBaseUpdater {
if (sparqlQueryStr == null) {
return required;
}
Model abox = settings.getAssertionOntModelSelector().getABoxModel();
Query query = QueryFactory.create(sparqlQueryStr);
QueryExecution isUpdated = QueryExecutionFactory.create(query, abox);
RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(servletContext).getRDFService();
// if the ASK query DOES have a solution (i.e. the assertions exist
// showing that the update has already been performed), then the update
// is NOT required.
if (isUpdated.execAsk()) {
required = false;
} else {
required = true;
if (JenaDataSourceSetupBase.isFirstStartup()) {
assertSuccess();
log.info("The application is starting with an empty database. " +
"An indication will be added to the database that a " +
"knowledge base migration to the current version is " +
"not required.");
required = false;
}
try {
if (rdfService.sparqlAskQuery(sparqlQueryStr)) {
required = false;
} else {
required = true;
if (JenaDataSourceSetupBase.isFirstStartup()) {
assertSuccess(servletContext);
log.info("The application is starting with an empty database. " +
"An indication will be added to the database that a " +
"knowledge base migration to the current version is " +
"not required.");
required = false;
}
}
} catch (RDFServiceException e) {
log.error("error trying to execute query to find out if knowledge base update is required",e);
}
return required;
@ -308,27 +301,19 @@ public class KnowledgeBaseUpdater {
return fileContents.toString();
}
private void assertSuccess() throws FileNotFoundException, IOException {
try {
private void assertSuccess(ServletContext servletContext) throws FileNotFoundException, IOException {
try {
RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(servletContext).getRDFService();
ChangeSet changeSet = rdfService.manufactureChangeSet();
//Model m = settings.getAssertionOntModelSelector().getApplicationMetadataModel();
Model m = settings.getAssertionOntModelSelector().getABoxModel();
File successAssertionsFile = new File(settings.getSuccessAssertionsFile());
InputStream inStream = new FileInputStream(successAssertionsFile);
m.enterCriticalSection(Lock.WRITE);
try {
m.read(inStream, null, settings.getSuccessRDFFormat());
if (logger != null) {
logger.logWithDate("Finished knowledge base migration");
}
} finally {
m.leaveCriticalSection();
}
changeSet.addAddition(inStream, RDFService.ModelSerializationFormat.N3, JenaDataSourceSetupBase.JENA_DB_MODEL);
rdfService.changeSetUpdate(changeSet);
} catch (Exception e) {
if (logger != null) {
logger.logError(" unable to make RDF assertions about successful " +
" update to new ontology version: " + e.getMessage());
}
log.error("unable to make RDF assertions about successful " +
" update to new ontology version: ", e);
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/OntologyChangeParser.java
View file

@ -92,7 +92,7 @@ public class OntologyChangeParser {
}
if (changeObjects.size() == 0) {
logger.log("did not find any changes in PromptDiff output file.");
logger.log("No ABox updates are required.");
}
return changeObjects;
}

7
webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/TBoxUpdater.java
View file

@ -105,12 +105,9 @@ public class TBoxUpdater {
// the site hasn't overidden the previous default in their knowledge base.
StmtIterator iter = oldTboxAnnotationsModel.listStatements();
int stmtCount = 0;
while (iter.hasNext()) {
stmtCount++;
Statement stmt = iter.next();
Resource subject = stmt.getSubject();
Property predicate = stmt.getPredicate();
@ -160,10 +157,12 @@ public class TBoxUpdater {
}
if (i > 1) {
/*
logger.log("WARNING: found " + i +
" statements with subject = " + subject.getURI() +
" and property = " + predicate.getURI() +
" in the new version of the annotations ontology (maximum of one is expected)");
*/
continue;
}

1
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java
View file

@ -102,7 +102,6 @@ public interface RDFService {
*
* @return boolean - the result of the SPARQL query
*/
public boolean sparqlAskQuery(String query) throws RDFServiceException;
/**

5
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java
View file

@ -11,6 +11,7 @@ import java.util.Iterator;
import java.util.List;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -108,6 +109,10 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
Iterator<ModelChange> csIt = changeSet.getModelChanges().iterator();
while (csIt.hasNext()) {
ModelChange modelChange = csIt.next();
if (!modelChange.getSerializedModel().markSupported()) {
byte[] bytes = IOUtils.toByteArray(modelChange.getSerializedModel());
modelChange.setSerializedModel(new ByteArrayInputStream(bytes));
}
modelChange.getSerializedModel().mark(Integer.MAX_VALUE);
dataset.getLock().enterCriticalSection(Lock.WRITE);
try {

6
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java
View file

@ -83,10 +83,14 @@ public class FileGraphSetup implements ServletContextListener {
OntDocumentManager.getInstance().setProcessImports(false);
}
/*
if (isUpdateRequired(sce.getServletContext())) {
log.info("mostSpecificType will be computed because a knowledge base migration was performed." );
SimpleReasonerSetup.setMSTComputeRequired(sce.getServletContext());
} else if (aboxChanged || tboxChanged) {
} else
*/
if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext())) {
log.info("a full recompute of the Abox will be performed because" +
" the filegraph abox(s) and/or tbox(s) have changed or are being read for the first time." );
SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext());

4
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java
View file

@ -49,7 +49,6 @@ public class UpdateKnowledgeBase implements ServletContextListener {
private static final String ADDED_DATA_FILE = DATA_DIR + CHANGED_DATA_DIR + "addedData.n3";
private static final String SPARQL_CONSTRUCT_ADDITIONS_DIR = DATA_DIR + "sparqlConstructs/additions/";
private static final String SPARQL_CONSTRUCT_DELETIONS_DIR = DATA_DIR + "sparqlConstructs/deletions/";
//private static final String MISC_REPLACEMENTS_FILE = DATA_DIR + "miscReplacements.rdf";
private static final String OLD_TBOX_MODEL_DIR = DATA_DIR + "oldVersion/";
private static final String NEW_TBOX_MODEL_DIR = "/WEB-INF/filegraph/tbox/";
private static final String OLD_TBOX_ANNOTATIONS_DIR = DATA_DIR + "oldAnnotations/";
@ -99,8 +98,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
try {
if (ontologyUpdater.updateRequired()) {
if (ontologyUpdater.updateRequired(ctx)) {
ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE);
ontologyUpdater.update(ctx);
}
} catch (IOException ioe) {
String errMsg = "IOException updating knowledge base " +