NIHVIVO-3673, updates for KB migration, and RDFService fix for inputstreams that don't support mark()/reset()

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/KnowledgeBaseUpdater.java

@@ -13,6 +13,8 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import javax.servlet.ServletContext;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -28,6 +30,10 @@ import com.hp.hpl.jena.rdf.model.Statement;
 import com.hp.hpl.jena.rdf.model.StmtIterator;
 import com.hp.hpl.jena.shared.Lock;
 
+import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
 import edu.cornell.mannlib.vitro.webapp.utils.jena.JenaIngestUtils;
 
@@ -49,7 +55,7 @@ public class KnowledgeBaseUpdater {
 		this.record = new SimpleChangeRecord(settings.getAddedDataFile(), settings.getRemovedDataFile());
 	}
 	
-	public void update() throws IOException {	
+	public void update(ServletContext servletContext) throws IOException {	
 					
 		if (this.logger == null) {
 			this.logger = new SimpleChangeLogger(settings.getLogFile(),	settings.getErrorLogFile());
@@ -68,9 +74,8 @@ public class KnowledgeBaseUpdater {
 		}
 
 		if (!logger.errorsWritten()) {
-			// add assertions to the knowledge base showing that the 
+			assertSuccess(servletContext);
-			// update was successful, so we don't need to run it again.
+	    	logger.logWithDate("Finished knowledge base migration");
-			assertSuccess();
 		}
 		
 		record.writeChanges();
@@ -86,22 +91,9 @@ public class KnowledgeBaseUpdater {
 	
 	private void performUpdate() throws IOException {
 		
-		log.info("\tperforming SPARQL construct additions (abox)");
+		// only annotations for 1.5
-		performSparqlConstructAdditions(settings.getSparqlConstructAdditionsDir(), settings.getUnionOntModelSelector().getABoxModel() , settings.getAssertionOntModelSelector().getABoxModel());
-		log.info("\tperforming SPARQL construct deletions (abox)");
-		performSparqlConstructRetractions(settings.getSparqlConstructDeletionsDir(), settings.getUnionOntModelSelector().getABoxModel() , settings.getAssertionOntModelSelector().getABoxModel());
-		
-		List<AtomicOntologyChange> rawChanges = getAtomicOntologyChanges();
-		
-		AtomicOntologyChangeLists changes = new AtomicOntologyChangeLists(rawChanges,settings.getNewTBoxModel(),settings.getOldTBoxModel());
-		
-        //process the TBox before the ABox
-		
 		log.info("\tupdating tbox annotations");
 	    updateTBoxAnnotations();
-		
-		log.info("\tupdating the abox");
-    	updateABox(changes);
 	}
 	
 	private void performSparqlConstructAdditions(String sparqlConstructDir, OntModel readModel, OntModel writeModel) throws IOException {
@@ -254,7 +246,7 @@ public class KnowledgeBaseUpdater {
 	 * Executes a SPARQL ASK query to determine whether the knowledge base
 	 * needs to be updated to conform to a new ontology version
 	 */
-	public boolean updateRequired() throws IOException {
+	public boolean updateRequired(ServletContext servletContext) throws IOException {
 		
 		boolean required = false;
 		
@@ -262,27 +254,28 @@ public class KnowledgeBaseUpdater {
 		if (sparqlQueryStr == null) {
 			return required;
 		}
-				
-		Model abox = settings.getAssertionOntModelSelector().getABoxModel();
-		Query query = QueryFactory.create(sparqlQueryStr);
-		QueryExecution isUpdated = QueryExecutionFactory.create(query, abox);
 		
+		RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(servletContext).getRDFService();
+
 		// if the ASK query DOES have a solution (i.e. the assertions exist
 		// showing that the update has already been performed), then the update
 		// is NOT required.
-		
+		try {
-		if (isUpdated.execAsk()) {
+			if (rdfService.sparqlAskQuery(sparqlQueryStr)) {
-			required = false;
+				required = false;
-		} else {
+			} else {
-			required = true;
+				required = true;
-			if (JenaDataSourceSetupBase.isFirstStartup()) {
+				if (JenaDataSourceSetupBase.isFirstStartup()) {
-				assertSuccess();  
+					assertSuccess(servletContext);  
-				log.info("The application is starting with an empty database. " +
+					log.info("The application is starting with an empty database. " +
-				         "An indication will be added to the database that a " +
+					         "An indication will be added to the database that a " +
-				         "knowledge base migration to the current version is " +
+					         "knowledge base migration to the current version is " +
-				         "not required.");
+					         "not required.");
-			    required = false;	
+				    required = false;	
-			}
+				}
+			}		   
+		} catch (RDFServiceException e) {
+			log.error("error trying to execute query to find out if knowledge base update is required",e); 
 		}
 		
 		return required; 
@@ -308,27 +301,19 @@ public class KnowledgeBaseUpdater {
 		return fileContents.toString();				
 	}
 	
-	private void assertSuccess() throws FileNotFoundException, IOException {
+	private void assertSuccess(ServletContext servletContext) throws FileNotFoundException, IOException {
-		try {
+		try {				
+			RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(servletContext).getRDFService();
+			ChangeSet changeSet = rdfService.manufactureChangeSet();
 			
-		    //Model m = settings.getAssertionOntModelSelector().getApplicationMetadataModel();
-		    Model m = settings.getAssertionOntModelSelector().getABoxModel();
 		    File successAssertionsFile = new File(settings.getSuccessAssertionsFile()); 
 		    InputStream inStream = new FileInputStream(successAssertionsFile);
-		    m.enterCriticalSection(Lock.WRITE);
+		    
-		    try {
+		    changeSet.addAddition(inStream, RDFService.ModelSerializationFormat.N3, JenaDataSourceSetupBase.JENA_DB_MODEL);
-		    	m.read(inStream, null, settings.getSuccessRDFFormat());
+			rdfService.changeSetUpdate(changeSet);	
-		    	if (logger != null) {
-		    		logger.logWithDate("Finished knowledge base migration");
-		    	} 
-		    } finally {
-		    	m.leaveCriticalSection();
-		    }
 		} catch (Exception e) {
-			if (logger != null) {
+			log.error("unable to make RDF assertions about successful " +
-			    logger.logError(" unable to make RDF assertions about successful " +
+					" update to new ontology version: ", e);
-					" update to new ontology version: " + e.getMessage());
-			}
 		}
 	}
 	

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/OntologyChangeParser.java

@@ -92,7 +92,7 @@ public class OntologyChangeParser {
 		}
 		
 		if (changeObjects.size() == 0) {
-			logger.log("did not find any changes in PromptDiff output file.");
+			logger.log("No ABox updates are required.");
 		}
 		return changeObjects;
 	}

webapp/src/edu/cornell/mannlib/vitro/webapp/ontology/update/TBoxUpdater.java

@@ -105,12 +105,9 @@ public class TBoxUpdater {
          //  the site hasn't overidden the previous default in their knowledge base.
     		    
 		  StmtIterator iter = oldTboxAnnotationsModel.listStatements();
-		  		  
+		  		  		  
-		  int stmtCount = 0;
-		  
 		  while (iter.hasNext()) {
 			  
-			 stmtCount++;
 			 Statement stmt = iter.next();
 			 Resource subject = stmt.getSubject();
 			 Property predicate = stmt.getPredicate();
@@ -160,10 +157,12 @@ public class TBoxUpdater {
              } 
 			 
 			 if (i > 1) {
+				 /*
 				 logger.log("WARNING: found " + i +
 						 " statements with subject = " + subject.getURI() + 
 						 " and property = " + predicate.getURI() +
 						 " in the new version of the annotations ontology (maximum of one is expected)");
+				 */		 
 				 continue; 
 			 }
 			 

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java

@@ -102,7 +102,6 @@ public interface RDFService {
 	 * 
 	 * @return  boolean - the result of the SPARQL query 
 	 */
-	
 	public boolean sparqlAskQuery(String query) throws RDFServiceException;
 	
 	/**

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java

@@ -11,6 +11,7 @@ import java.util.Iterator;
 import java.util.List;
 
 import org.apache.commons.dbcp.BasicDataSource;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -108,6 +109,10 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
             Iterator<ModelChange> csIt = changeSet.getModelChanges().iterator();
             while (csIt.hasNext()) {
                 ModelChange modelChange = csIt.next();
+                if (!modelChange.getSerializedModel().markSupported()) {
+                    byte[] bytes = IOUtils.toByteArray(modelChange.getSerializedModel());
+                    modelChange.setSerializedModel(new ByteArrayInputStream(bytes));
+                }
                 modelChange.getSerializedModel().mark(Integer.MAX_VALUE);
                 dataset.getLock().enterCriticalSection(Lock.WRITE);
                 try {

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java

@@ -83,10 +83,14 @@ public class FileGraphSetup implements ServletContextListener {
             OntDocumentManager.getInstance().setProcessImports(false);
         }
 
+        /*
         if (isUpdateRequired(sce.getServletContext()))  {
             log.info("mostSpecificType will be computed because a knowledge base migration was performed." );
             SimpleReasonerSetup.setMSTComputeRequired(sce.getServletContext());
-        } else if (aboxChanged || tboxChanged) {
+        } else 
+        */
+        
+        if ( (aboxChanged || tboxChanged) && !isUpdateRequired(sce.getServletContext())) {
             log.info("a full recompute of the Abox will be performed because" +
                     " the filegraph abox(s) and/or tbox(s) have changed or are being read for the first time." );
             SimpleReasonerSetup.setRecomputeRequired(sce.getServletContext());

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/UpdateKnowledgeBase.java

@@ -49,7 +49,6 @@ public class UpdateKnowledgeBase implements ServletContextListener {
 	private static final String ADDED_DATA_FILE = DATA_DIR + CHANGED_DATA_DIR + "addedData.n3";
 	private static final String SPARQL_CONSTRUCT_ADDITIONS_DIR = DATA_DIR + "sparqlConstructs/additions/";
 	private static final String SPARQL_CONSTRUCT_DELETIONS_DIR = DATA_DIR + "sparqlConstructs/deletions/";
-	//private static final String MISC_REPLACEMENTS_FILE = DATA_DIR + "miscReplacements.rdf";
 	private static final String OLD_TBOX_MODEL_DIR = DATA_DIR + "oldVersion/";
 	private static final String NEW_TBOX_MODEL_DIR = "/WEB-INF/filegraph/tbox/";
 	private static final String OLD_TBOX_ANNOTATIONS_DIR = DATA_DIR + "oldAnnotations/";
@@ -99,8 +98,9 @@ public class UpdateKnowledgeBase implements ServletContextListener {
 			   KnowledgeBaseUpdater ontologyUpdater = new KnowledgeBaseUpdater(settings);
 			  
 			   try {
-				  if (ontologyUpdater.updateRequired()) {
+				  if (ontologyUpdater.updateRequired(ctx)) {
 					  ctx.setAttribute(KBM_REQURIED_AT_STARTUP, Boolean.TRUE);
+					  ontologyUpdater.update(ctx);
 				  }
 			   } catch (IOException ioe) {
 					String errMsg = "IOException updating knowledge base " +

webapp/test/edu/cornell/mannlib/vitro/webapp/reasoner/SimpleReasonerSameAsTest.java

@@ -446,6 +446,14 @@ public class SimpleReasonerSameAsTest extends AbstractTestClass {
 		
 		//create aBox and tBox, and SimpleReasoner to listen to them
 		OntModel tBox = ModelFactory.createOntologyModel(PelletReasonerFactory.THE_SPEC); 
+		// set up TBox	
+		OntClass classA = tBox.createClass("http://test.vivo/A");
+	    classA.setLabel("class A", "en-US");
+		OntClass classB = tBox.createClass("http://test.vivo/B");
+	    classB.setLabel("class B", "en-US");
+		OntClass classC = tBox.createClass("http://test.vivo/C");
+	    classC.setLabel("class C", "en-US");
+	    
 		OntModel aBox = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM); 
         Model inf = ModelFactory.createDefaultModel();
 		
@@ -454,14 +462,6 @@ public class SimpleReasonerSameAsTest extends AbstractTestClass {
 		SimpleReasonerTBoxListener simpleReasonerTBoxListener = getTBoxListener(simpleReasoner);
 		tBox.register(simpleReasonerTBoxListener);
 
-		// set up TBox	
-		OntClass classA = tBox.createClass("http://test.vivo/A");
-	    classA.setLabel("class A", "en-US");
-		OntClass classB = tBox.createClass("http://test.vivo/B");
-	    classB.setLabel("class B", "en-US");
-		OntClass classC = tBox.createClass("http://test.vivo/C");
-	    classC.setLabel("class C", "en-US");
-	    	   
 	    // set up ABox
 		Resource a = aBox.createResource("http://test.vivo/a");
 		Resource b = aBox.createResource("http://test.vivo/b");