NIHVIVO-3780 search indexer listening to RDF API and other RDF API related improvements

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java

@@ -21,6 +21,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDao;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroModelSource.ModelName;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 
 public class VitroRequest extends HttpServletRequestWrapper {
     
@@ -41,6 +43,16 @@ public class VitroRequest extends HttpServletRequestWrapper {
         this._req = _req;
     }
 
+    public RDFService getRDFService() {
+        Object o = getAttribute("rdfService");
+        if (o instanceof RDFService) {
+            return (RDFService) o;
+        } else {
+            RDFService rdfService = RDFServiceUtils.getRDFService(this);
+            setAttribute("rdfService", rdfService);
+            return rdfService;
+        }
+    }
     
     public void setWebappDaoFactory( WebappDaoFactory wdf){
         setAttribute("webappDaoFactory",wdf);

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java

@@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.controller.jena;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
 
@@ -37,6 +38,10 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.event.BulkUpdateEvent;
 import edu.cornell.mannlib.vitro.webapp.dao.jena.event.EditEvent;
 import edu.cornell.mannlib.vitro.webapp.filestorage.uploadrequest.FileUploadServletRequest;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
+import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
 
 public class RDFUploadController extends JenaIngestController {
     
@@ -119,12 +124,17 @@ public class RDFUploadController extends JenaIngestController {
                     && fileStreams.get("rdfStream").size() > 0 ) {
                 FileItem rdfStream = fileStreams.get("rdfStream").get(0);
                 try {
-                    uploadModel.enterCriticalSection(Lock.WRITE);
+                    if (directRead) {
-                    try {
+                        addUsingRDFService(rdfStream.getInputStream(), languageStr,
-                        uploadModel.read(
+                                request.getRDFService());
-                                rdfStream.getInputStream(), null, languageStr);
+                    } else {
-                    } finally {
+                        uploadModel.enterCriticalSection(Lock.WRITE);
-                        uploadModel.leaveCriticalSection();
+                        try {
+                            uploadModel.read(
+                                    rdfStream.getInputStream(), null, languageStr);
+                        } finally {
+                            uploadModel.leaveCriticalSection();
+                        }
                     }
                     uploadDesc = verb + " RDF from file " + rdfStream.getName();                
                 } catch (IOException e) {
@@ -198,6 +208,24 @@ public class RDFUploadController extends JenaIngestController {
         }
     }
 
+    private void addUsingRDFService(InputStream in, String languageStr, 
+            RDFService rdfService) {
+        ChangeSet changeSet = rdfService.manufactureChangeSet();
+        RDFService.ModelSerializationFormat format = 
+                ("RDF/XML".equals(languageStr) 
+                        || "RDF/XML-ABBREV".equals(languageStr))
+                                ? RDFService.ModelSerializationFormat.RDFXML
+                                : RDFService.ModelSerializationFormat.N3;
+        changeSet.addAddition(in, format, 
+                JenaDataSourceSetupBase.JENA_DB_MODEL);
+        try {
+            rdfService.changeSetUpdate(changeSet);
+        } catch (RDFServiceException rdfse) {
+            log.error(rdfse);
+            throw new RuntimeException(rdfse);
+        }
+    }
+    
     public void loadRDF(FileUploadServletRequest req,
                         VitroRequest request,
                         HttpServletResponse response) 

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java

@@ -0,0 +1,69 @@
+package edu.cornell.mannlib.vitro.webapp.dao.jena;
+
+import java.io.ByteArrayInputStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelChangedListener;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.Statement;
+import com.hp.hpl.jena.rdf.model.StmtIterator;
+
+import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
+
+/**
+ * A ChangeListener that forwards events to a Jena ModelChangedListener 
+ * @author bjl23
+ *
+ */
+public class JenaChangeListener implements ChangeListener {
+
+    private static final Log log = LogFactory.getLog(JenaChangeListener.class);
+    
+    private ModelChangedListener listener;
+    private Model m = ModelFactory.createDefaultModel();
+    
+    public JenaChangeListener(ModelChangedListener listener) {
+        this.listener = listener;
+    }
+    
+    @Override
+    public void addedStatement(String serializedTriple, String graphURI) {
+        listener.addedStatement(parseTriple(serializedTriple));
+    }
+
+    @Override
+    public void removedStatement(String serializedTriple, String graphURI) {
+        listener.removedStatement(parseTriple(serializedTriple));
+    }
+
+    @Override
+    public void notifyEvent(String graphURI, Object event) {
+        listener.notifyEvent(m, event);
+    }
+    
+    // TODO avoid overhead of Model
+    private Statement parseTriple(String serializedTriple) {
+        try {
+            Model m = ModelFactory.createDefaultModel();
+            m.read(new ByteArrayInputStream(
+                    serializedTriple.getBytes("UTF-8")), null, "N3");
+            StmtIterator sit = m.listStatements();
+            if (!sit.hasNext()) {
+                throw new RuntimeException("no triple parsed from change event");
+            } else {
+                Statement s = sit.nextStatement();
+                if (sit.hasNext()) {
+                    log.warn("More than one triple parsed from change event");
+                }
+                return s;
+            }         
+        } catch (UnsupportedEncodingException uee) {
+            throw new RuntimeException(uee);
+        }
+    }
+
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java

@@ -4,12 +4,19 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
 import javax.servlet.ServletContext;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import com.hp.hpl.jena.ontology.OntModel;
 import com.hp.hpl.jena.rdf.model.ModelChangedListener;
 
 import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 
 public class ModelContext {
+    
+    private static final Log log = LogFactory.getLog(ModelContext.class);
 	
 	private static final String ONT_MODEL_SELECTOR = "ontModelSelector";
 	private static final String UNION_ONT_MODEL_SELECTOR = "unionOntModelSelector";
@@ -96,15 +103,23 @@ public class ModelContext {
 	 *   Changes to application model
 	 */
 	public static void registerListenerForChanges(ServletContext ctx, ModelChangedListener ml){
-	    ModelContext.getJenaOntModel(ctx).register(ml);
+	    
-        ModelContext.getBaseOntModel(ctx).register(ml);
+        try {
-        ModelContext.getInferenceOntModel(ctx).register(ml);
+            RDFServiceUtils.getRDFServiceFactory(ctx).registerListener(
-        ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml);
+                    new JenaChangeListener(ml));
-        ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml);
+        } catch (RDFServiceException e) {
-        ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml);
+            log.error(e,e);
-        ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml);
+        }
         
-        ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml);
+//      ModelContext.getJenaOntModel(ctx).register(ml);
+//      ModelContext.getBaseOntModel(ctx).register(ml);
+//      ModelContext.getInferenceOntModel(ctx).register(ml);
+//      ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml);
+//      ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml);
+//      ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml);
+//      ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml);
+        
+//      ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml);
         
 	}
 	

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java

@@ -1,5 +1,7 @@
 package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
@@ -71,7 +73,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
 
     @Override
     public void add(Graph g, boolean arg1) {
-        log.info("adding graph");
+        log.info("bulk addGraph()");
         Model[] model = separateStatementsWithBlankNodes(g);
         addModel(model[1] /* nonBlankNodeModel */);
         // replace following call with different method
@@ -112,46 +114,34 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
     
     @Override 
     public void delete(Graph g) {
-        ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
+        deleteModel(ModelFactory.createModelForGraph(g));
-        Model m = ModelFactory.createModelForGraph(g);
-        PipedOutputStream out = new PipedOutputStream();
-        m.write(out, "N-TRIPLE");
-        try {
-            changeSet.addRemoval(new PipedInputStream(
-                    out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
-            graph.getRDFService().changeSetUpdate(changeSet);
-        } catch (IOException ioe) {
-            throw new RuntimeException(ioe);
-        } catch (RDFServiceException rdfse) {
-            throw new RuntimeException(rdfse);
-        }
     }
     
     public void addModel(Model model) {
+        log.info("bulk addModel()");
         ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
-        PipedOutputStream out = new PipedOutputStream();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        model.write(out, "N-TRIPLE");
+        model.write(out, "N-TRIPLE");    
+        changeSet.addAddition(new ByteArrayInputStream(
+                out.toByteArray()), RDFService.ModelSerializationFormat.N3, 
+                        graph.getGraphURI());
         try {
-            changeSet.addAddition(new PipedInputStream(
-                    out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
             graph.getRDFService().changeSetUpdate(changeSet);
-        } catch (IOException ioe) {
-            throw new RuntimeException(ioe);
         } catch (RDFServiceException rdfse) {
             throw new RuntimeException(rdfse);
         }
     }
     
     public void deleteModel(Model model) {
+        log.info("bulk addModel()");
         ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
-        PipedOutputStream out = new PipedOutputStream();
+        ByteArrayOutputStream out = new ByteArrayOutputStream();
-        model.write(out, "N-TRIPLE");
+        model.write(out, "N-TRIPLE");    
+        changeSet.addRemoval(new ByteArrayInputStream(
+                out.toByteArray()), RDFService.ModelSerializationFormat.N3, 
+                        graph.getGraphURI());
         try {
-            changeSet.addRemoval(new PipedInputStream(
-                    out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
             graph.getRDFService().changeSetUpdate(changeSet);
-        } catch (IOException ioe) {
-            throw new RuntimeException(ioe);
         } catch (RDFServiceException rdfse) {
             throw new RuntimeException(rdfse);
         }

webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java

@@ -94,12 +94,6 @@ public class WebappDaoFactorySDBPrep implements Filter {
 		WebappDaoFactory wadf = null;
 		VitroRequest vreq = new VitroRequest((HttpServletRequest) request);
 		
-        Enumeration<String> headStrs = vreq.getHeaderNames();
-        while (headStrs.hasMoreElements()) {
-            String head = headStrs.nextElement(); 
-            log.info(head + " : "  + vreq.getHeader(head));                     
-        }
-        
         List<String> langs = new ArrayList<String>();
         
         log.debug("Accept-Language: " + vreq.getHeader("Accept-Language"));

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java

@@ -6,6 +6,22 @@ public interface RDFServiceFactory {
 
     public RDFService getRDFService();
     
-    public RDFService getRDFService(VitroRequest vreq);
+    /**
+     * Register a listener to listen to changes in any graph in
+     * the RDF store.  Any RDFService objects returned by this factory should notify
+     * this listener of changes.
+     * 
+     * @param changeListener - the change listener
+     */
+    public void registerListener(ChangeListener changeListener) throws RDFServiceException;
+    
+    /**
+     * Unregister a listener from listening to changes in
+     * the RDF store.  Any RDFService objects returned by this factory should notify
+     * this listener of changes.
+     * 
+     * @param changeListener - the change listener
+     */
+    public void unregisterListener(ChangeListener changeListener) throws RDFServiceException;
     
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java

@@ -1,7 +1,8 @@
 package edu.cornell.mannlib.vitro.webapp.rdfservice.impl;
 
-import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
 
 /**
@@ -21,10 +22,15 @@ public class RDFServiceFactorySingle implements RDFServiceFactory {
     public RDFService getRDFService() {
         return this.rdfService;
     }
-
+    
     @Override
-    public RDFService getRDFService(VitroRequest vreq) {
+    public void registerListener(ChangeListener listener) throws RDFServiceException {
-        return this.rdfService;
+        this.rdfService.registerListener(listener);
+    }
+    
+    @Override
+    public void unregisterListener(ChangeListener listener) throws RDFServiceException {
+        this.rdfService.unregisterListener(listener);
     }
 
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java

@@ -168,7 +168,7 @@ public abstract class RDFServiceImpl implements RDFService {
     protected static String getSerializationFormatString(RDFService.ModelSerializationFormat format) {
         switch (format) {
             case RDFXML: 
-                return "RDFXML";
+                return "RDF/XML";
             case N3: 
                 return "N3";
             default: 

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java

@@ -6,6 +6,8 @@ import java.io.UnsupportedEncodingException;
 
 import javax.servlet.ServletContext;
 
+import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
 
 public class RDFServiceUtils {
@@ -31,4 +33,9 @@ public class RDFServiceUtils {
         }
     }
     
+    public static RDFService getRDFService(VitroRequest vreq) {
+        return getRDFServiceFactory(
+                vreq.getSession().getServletContext()).getRDFService();
+    }
+    
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java

@@ -94,7 +94,7 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
         }
         
         Dataset dataset = getDataset(conn);
-        boolean transaction = false; // conn.getTransactionHandler().transactionsSupported();
+        boolean transaction = conn.getTransactionHandler().transactionsSupported();
         
         try {
             if (transaction) {
@@ -109,8 +109,8 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
                     try {
                         model.register(new ModelListener(modelChange.getGraphURI(), this));
                         if (modelChange.getOperation() == ModelChange.Operation.ADD) {
-                            Model m = parseModel(modelChange);
+                            model.read(modelChange.getSerializedModel(), null,
-                            model.add(m);  
+                                    getSerializationFormatString(modelChange.getSerializationFormat()));  
                         } else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) {
                             model.remove(parseModel(modelChange));
                             removeBlankNodesWithSparqlUpdate(dataset, model, modelChange.getGraphURI());
@@ -156,6 +156,12 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
         
         StringBuffer patternBuff = new StringBuffer();
         StmtIterator stmtIt = model.listStatements();
+        
+        if (!stmtIt.hasNext()) {
+            stmtIt.close();
+            return;
+        }
+        
         while(stmtIt.hasNext()) {
             Triple t = stmtIt.next().asTriple();
             patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java

@@ -162,24 +162,15 @@ public class RDFServiceSetup extends JenaDataSourceSetupBase
         // ABox assertions
         Model aboxAssertions = dataset.getNamedModel(
                 JenaDataSourceSetupBase.JENA_DB_MODEL);
-        Model listenableAboxAssertions = ModelFactory.createUnion(
-                aboxAssertions, ModelFactory.createDefaultModel());
         baseOms.setABoxModel(
                 ModelFactory.createOntologyModel(
-                        OntModelSpec.OWL_MEM, listenableAboxAssertions));
+                        OntModelSpec.OWL_MEM, aboxAssertions));
         
         // ABox inferences
         Model aboxInferences = dataset.getNamedModel(
                 JenaDataSourceSetupBase.JENA_INF_MODEL);
-        Model listenableAboxInferences = ModelFactory.createUnion(
-                aboxInferences, ModelFactory.createDefaultModel());
         inferenceOms.setABoxModel(ModelFactory.createOntologyModel(
-                OntModelSpec.OWL_MEM, listenableAboxInferences));
+                OntModelSpec.OWL_MEM, aboxInferences));
-      
-        
-        // Since the TBox models are in memory, they do not have timeout issues 
-        // like the like the ABox models do (and so don't need the extra step 
-        // to make them listenable.)
         
         // TBox assertions
         try {