From cef7b583e26a7ff4478e7c947ae4485367481d3a Mon Sep 17 00:00:00 2001 From: brianjlowe Date: Thu, 7 Jun 2012 19:40:26 +0000 Subject: [PATCH] NIHVIVO-3780 search indexer listening to RDF API and other RDF API related improvements --- .../vitro/webapp/controller/VitroRequest.java | 12 ++++ .../controller/jena/RDFUploadController.java | 40 +++++++++-- .../webapp/dao/jena/JenaChangeListener.java | 69 +++++++++++++++++++ .../vitro/webapp/dao/jena/ModelContext.java | 31 ++++++--- .../dao/jena/RDFServiceGraphBulkUpdater.java | 42 +++++------ .../filters/WebappDaoFactorySDBPrep.java | 6 -- .../webapp/rdfservice/RDFServiceFactory.java | 18 ++++- .../impl/RDFServiceFactorySingle.java | 14 ++-- .../rdfservice/impl/RDFServiceImpl.java | 2 +- .../rdfservice/impl/RDFServiceUtils.java | 7 ++ .../rdfservice/impl/sdb/RDFServiceSDB.java | 12 +++- .../webapp/servlet/setup/RDFServiceSetup.java | 13 +--- 12 files changed, 200 insertions(+), 66 deletions(-) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java index 873872216..7f071e2d8 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java @@ -21,6 +21,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDao; import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector; import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroModelSource.ModelName; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; public class VitroRequest extends HttpServletRequestWrapper { @@ -41,6 +43,16 @@ public class VitroRequest extends HttpServletRequestWrapper { this._req = _req; } + public RDFService getRDFService() { + Object o = getAttribute("rdfService"); + if (o instanceof RDFService) { + return (RDFService) o; + } else { + RDFService rdfService = RDFServiceUtils.getRDFService(this); + setAttribute("rdfService", rdfService); + return rdfService; + } + } public void setWebappDaoFactory( WebappDaoFactory wdf){ setAttribute("webappDaoFactory",wdf); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java index 1e5f08ee2..108639c0b 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java @@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.controller.jena; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.util.List; import java.util.Map; @@ -37,6 +38,10 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector; import edu.cornell.mannlib.vitro.webapp.dao.jena.event.BulkUpdateEvent; import edu.cornell.mannlib.vitro.webapp.dao.jena.event.EditEvent; import edu.cornell.mannlib.vitro.webapp.filestorage.uploadrequest.FileUploadServletRequest; +import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; +import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase; public class RDFUploadController extends JenaIngestController { @@ -119,12 +124,17 @@ public class RDFUploadController extends JenaIngestController { && fileStreams.get("rdfStream").size() > 0 ) { FileItem rdfStream = fileStreams.get("rdfStream").get(0); try { - uploadModel.enterCriticalSection(Lock.WRITE); - try { - uploadModel.read( - rdfStream.getInputStream(), null, languageStr); - } finally { - uploadModel.leaveCriticalSection(); + if (directRead) { + addUsingRDFService(rdfStream.getInputStream(), languageStr, + request.getRDFService()); + } else { + uploadModel.enterCriticalSection(Lock.WRITE); + try { + uploadModel.read( + rdfStream.getInputStream(), null, languageStr); + } finally { + uploadModel.leaveCriticalSection(); + } } uploadDesc = verb + " RDF from file " + rdfStream.getName(); } catch (IOException e) { @@ -198,6 +208,24 @@ public class RDFUploadController extends JenaIngestController { } } + private void addUsingRDFService(InputStream in, String languageStr, + RDFService rdfService) { + ChangeSet changeSet = rdfService.manufactureChangeSet(); + RDFService.ModelSerializationFormat format = + ("RDF/XML".equals(languageStr) + || "RDF/XML-ABBREV".equals(languageStr)) + ? RDFService.ModelSerializationFormat.RDFXML + : RDFService.ModelSerializationFormat.N3; + changeSet.addAddition(in, format, + JenaDataSourceSetupBase.JENA_DB_MODEL); + try { + rdfService.changeSetUpdate(changeSet); + } catch (RDFServiceException rdfse) { + log.error(rdfse); + throw new RuntimeException(rdfse); + } + } + public void loadRDF(FileUploadServletRequest req, VitroRequest request, HttpServletResponse response) diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java new file mode 100644 index 000000000..ee8cf9fbb --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java @@ -0,0 +1,69 @@ +package edu.cornell.mannlib.vitro.webapp.dao.jena; + +import java.io.ByteArrayInputStream; +import java.io.UnsupportedEncodingException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import com.hp.hpl.jena.rdf.model.Model; +import com.hp.hpl.jena.rdf.model.ModelChangedListener; +import com.hp.hpl.jena.rdf.model.ModelFactory; +import com.hp.hpl.jena.rdf.model.Statement; +import com.hp.hpl.jena.rdf.model.StmtIterator; + +import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener; + +/** + * A ChangeListener that forwards events to a Jena ModelChangedListener + * @author bjl23 + * + */ +public class JenaChangeListener implements ChangeListener { + + private static final Log log = LogFactory.getLog(JenaChangeListener.class); + + private ModelChangedListener listener; + private Model m = ModelFactory.createDefaultModel(); + + public JenaChangeListener(ModelChangedListener listener) { + this.listener = listener; + } + + @Override + public void addedStatement(String serializedTriple, String graphURI) { + listener.addedStatement(parseTriple(serializedTriple)); + } + + @Override + public void removedStatement(String serializedTriple, String graphURI) { + listener.removedStatement(parseTriple(serializedTriple)); + } + + @Override + public void notifyEvent(String graphURI, Object event) { + listener.notifyEvent(m, event); + } + + // TODO avoid overhead of Model + private Statement parseTriple(String serializedTriple) { + try { + Model m = ModelFactory.createDefaultModel(); + m.read(new ByteArrayInputStream( + serializedTriple.getBytes("UTF-8")), null, "N3"); + StmtIterator sit = m.listStatements(); + if (!sit.hasNext()) { + throw new RuntimeException("no triple parsed from change event"); + } else { + Statement s = sit.nextStatement(); + if (sit.hasNext()) { + log.warn("More than one triple parsed from change event"); + } + return s; + } + } catch (UnsupportedEncodingException uee) { + throw new RuntimeException(uee); + } + } + +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java index 89afe54ed..4d2a84f98 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java @@ -4,12 +4,19 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena; import javax.servlet.ServletContext; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.rdf.model.ModelChangedListener; import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; +import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; public class ModelContext { + + private static final Log log = LogFactory.getLog(ModelContext.class); private static final String ONT_MODEL_SELECTOR = "ontModelSelector"; private static final String UNION_ONT_MODEL_SELECTOR = "unionOntModelSelector"; @@ -96,15 +103,23 @@ public class ModelContext { * Changes to application model */ public static void registerListenerForChanges(ServletContext ctx, ModelChangedListener ml){ - ModelContext.getJenaOntModel(ctx).register(ml); - ModelContext.getBaseOntModel(ctx).register(ml); - ModelContext.getInferenceOntModel(ctx).register(ml); - ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml); - ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml); - ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml); - ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml); + + try { + RDFServiceUtils.getRDFServiceFactory(ctx).registerListener( + new JenaChangeListener(ml)); + } catch (RDFServiceException e) { + log.error(e,e); + } - ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml); +// ModelContext.getJenaOntModel(ctx).register(ml); +// ModelContext.getBaseOntModel(ctx).register(ml); +// ModelContext.getInferenceOntModel(ctx).register(ml); +// ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml); +// ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml); +// ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml); +// ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml); + +// ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java index 3d7699121..3e0928cba 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java @@ -1,5 +1,7 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PipedInputStream; import java.io.PipedOutputStream; @@ -71,7 +73,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler { @Override public void add(Graph g, boolean arg1) { - log.info("adding graph"); + log.info("bulk addGraph()"); Model[] model = separateStatementsWithBlankNodes(g); addModel(model[1] /* nonBlankNodeModel */); // replace following call with different method @@ -112,46 +114,34 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler { @Override public void delete(Graph g) { - ChangeSet changeSet = graph.getRDFService().manufactureChangeSet(); - Model m = ModelFactory.createModelForGraph(g); - PipedOutputStream out = new PipedOutputStream(); - m.write(out, "N-TRIPLE"); - try { - changeSet.addRemoval(new PipedInputStream( - out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI()); - graph.getRDFService().changeSetUpdate(changeSet); - } catch (IOException ioe) { - throw new RuntimeException(ioe); - } catch (RDFServiceException rdfse) { - throw new RuntimeException(rdfse); - } + deleteModel(ModelFactory.createModelForGraph(g)); } public void addModel(Model model) { + log.info("bulk addModel()"); ChangeSet changeSet = graph.getRDFService().manufactureChangeSet(); - PipedOutputStream out = new PipedOutputStream(); - model.write(out, "N-TRIPLE"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.write(out, "N-TRIPLE"); + changeSet.addAddition(new ByteArrayInputStream( + out.toByteArray()), RDFService.ModelSerializationFormat.N3, + graph.getGraphURI()); try { - changeSet.addAddition(new PipedInputStream( - out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI()); graph.getRDFService().changeSetUpdate(changeSet); - } catch (IOException ioe) { - throw new RuntimeException(ioe); } catch (RDFServiceException rdfse) { throw new RuntimeException(rdfse); } } public void deleteModel(Model model) { + log.info("bulk addModel()"); ChangeSet changeSet = graph.getRDFService().manufactureChangeSet(); - PipedOutputStream out = new PipedOutputStream(); - model.write(out, "N-TRIPLE"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + model.write(out, "N-TRIPLE"); + changeSet.addRemoval(new ByteArrayInputStream( + out.toByteArray()), RDFService.ModelSerializationFormat.N3, + graph.getGraphURI()); try { - changeSet.addRemoval(new PipedInputStream( - out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI()); graph.getRDFService().changeSetUpdate(changeSet); - } catch (IOException ioe) { - throw new RuntimeException(ioe); } catch (RDFServiceException rdfse) { throw new RuntimeException(rdfse); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java index e834a2326..48d9dd37c 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java @@ -94,12 +94,6 @@ public class WebappDaoFactorySDBPrep implements Filter { WebappDaoFactory wadf = null; VitroRequest vreq = new VitroRequest((HttpServletRequest) request); - Enumeration headStrs = vreq.getHeaderNames(); - while (headStrs.hasMoreElements()) { - String head = headStrs.nextElement(); - log.info(head + " : " + vreq.getHeader(head)); - } - List langs = new ArrayList(); log.debug("Accept-Language: " + vreq.getHeader("Accept-Language")); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java index a2d062ffe..dc2cca163 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java @@ -6,6 +6,22 @@ public interface RDFServiceFactory { public RDFService getRDFService(); - public RDFService getRDFService(VitroRequest vreq); + /** + * Register a listener to listen to changes in any graph in + * the RDF store. Any RDFService objects returned by this factory should notify + * this listener of changes. + * + * @param changeListener - the change listener + */ + public void registerListener(ChangeListener changeListener) throws RDFServiceException; + + /** + * Unregister a listener from listening to changes in + * the RDF store. Any RDFService objects returned by this factory should notify + * this listener of changes. + * + * @param changeListener - the change listener + */ + public void unregisterListener(ChangeListener changeListener) throws RDFServiceException; } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java index d8f0b4267..2580100c9 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java @@ -1,7 +1,8 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.impl; -import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; +import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; /** @@ -21,10 +22,15 @@ public class RDFServiceFactorySingle implements RDFServiceFactory { public RDFService getRDFService() { return this.rdfService; } - + @Override - public RDFService getRDFService(VitroRequest vreq) { - return this.rdfService; + public void registerListener(ChangeListener listener) throws RDFServiceException { + this.rdfService.registerListener(listener); + } + + @Override + public void unregisterListener(ChangeListener listener) throws RDFServiceException { + this.rdfService.unregisterListener(listener); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java index 9ea36dcca..ea253a21e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java @@ -168,7 +168,7 @@ public abstract class RDFServiceImpl implements RDFService { protected static String getSerializationFormatString(RDFService.ModelSerializationFormat format) { switch (format) { case RDFXML: - return "RDFXML"; + return "RDF/XML"; case N3: return "N3"; default: diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java index 4bf574b23..71d0e79cc 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java @@ -6,6 +6,8 @@ import java.io.UnsupportedEncodingException; import javax.servlet.ServletContext; +import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; public class RDFServiceUtils { @@ -31,4 +33,9 @@ public class RDFServiceUtils { } } + public static RDFService getRDFService(VitroRequest vreq) { + return getRDFServiceFactory( + vreq.getSession().getServletContext()).getRDFService(); + } + } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java index ca055ed63..1e52016d7 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java @@ -94,7 +94,7 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService { } Dataset dataset = getDataset(conn); - boolean transaction = false; // conn.getTransactionHandler().transactionsSupported(); + boolean transaction = conn.getTransactionHandler().transactionsSupported(); try { if (transaction) { @@ -109,8 +109,8 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService { try { model.register(new ModelListener(modelChange.getGraphURI(), this)); if (modelChange.getOperation() == ModelChange.Operation.ADD) { - Model m = parseModel(modelChange); - model.add(m); + model.read(modelChange.getSerializedModel(), null, + getSerializationFormatString(modelChange.getSerializationFormat())); } else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) { model.remove(parseModel(modelChange)); removeBlankNodesWithSparqlUpdate(dataset, model, modelChange.getGraphURI()); @@ -156,6 +156,12 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService { StringBuffer patternBuff = new StringBuffer(); StmtIterator stmtIt = model.listStatements(); + + if (!stmtIt.hasNext()) { + stmtIt.close(); + return; + } + while(stmtIt.hasNext()) { Triple t = stmtIt.next().asTriple(); patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null)); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java index 160f4a67a..53c5e4f2d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java @@ -162,24 +162,15 @@ public class RDFServiceSetup extends JenaDataSourceSetupBase // ABox assertions Model aboxAssertions = dataset.getNamedModel( JenaDataSourceSetupBase.JENA_DB_MODEL); - Model listenableAboxAssertions = ModelFactory.createUnion( - aboxAssertions, ModelFactory.createDefaultModel()); baseOms.setABoxModel( ModelFactory.createOntologyModel( - OntModelSpec.OWL_MEM, listenableAboxAssertions)); + OntModelSpec.OWL_MEM, aboxAssertions)); // ABox inferences Model aboxInferences = dataset.getNamedModel( JenaDataSourceSetupBase.JENA_INF_MODEL); - Model listenableAboxInferences = ModelFactory.createUnion( - aboxInferences, ModelFactory.createDefaultModel()); inferenceOms.setABoxModel(ModelFactory.createOntologyModel( - OntModelSpec.OWL_MEM, listenableAboxInferences)); - - - // Since the TBox models are in memory, they do not have timeout issues - // like the like the ABox models do (and so don't need the extra step - // to make them listenable.) + OntModelSpec.OWL_MEM, aboxInferences)); // TBox assertions try {