litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3780 search indexer listening to RDF API and other RDF API related improvements

Browse source
This commit is contained in:
brianjlowe 2012-06-07 19:40:26 +00:00
parent 5d3ca126d7
commit cef7b583e2
12 changed files with 200 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

12
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java
View file

@ -21,6 +21,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDao;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroModelSource.ModelName;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
public class VitroRequest extends HttpServletRequestWrapper {
@ -41,6 +43,16 @@ public class VitroRequest extends HttpServletRequestWrapper {
this._req = _req;
}
public RDFService getRDFService() {
Object o = getAttribute("rdfService");
if (o instanceof RDFService) {
return (RDFService) o;
} else {
RDFService rdfService = RDFServiceUtils.getRDFService(this);
setAttribute("rdfService", rdfService);
return rdfService;
}
}
public void setWebappDaoFactory( WebappDaoFactory wdf){
setAttribute("webappDaoFactory",wdf);

40
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/RDFUploadController.java
View file

@ -4,6 +4,7 @@ package edu.cornell.mannlib.vitro.webapp.controller.jena;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
@ -37,6 +38,10 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
import edu.cornell.mannlib.vitro.webapp.dao.jena.event.BulkUpdateEvent;
import edu.cornell.mannlib.vitro.webapp.dao.jena.event.EditEvent;
import edu.cornell.mannlib.vitro.webapp.filestorage.uploadrequest.FileUploadServletRequest;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
public class RDFUploadController extends JenaIngestController {
@ -119,12 +124,17 @@ public class RDFUploadController extends JenaIngestController {
&& fileStreams.get("rdfStream").size() > 0 ) {
FileItem rdfStream = fileStreams.get("rdfStream").get(0);
try {
uploadModel.enterCriticalSection(Lock.WRITE);
try {
uploadModel.read(
rdfStream.getInputStream(), null, languageStr);
} finally {
uploadModel.leaveCriticalSection();
if (directRead) {
addUsingRDFService(rdfStream.getInputStream(), languageStr,
request.getRDFService());
} else {
uploadModel.enterCriticalSection(Lock.WRITE);
try {
uploadModel.read(
rdfStream.getInputStream(), null, languageStr);
} finally {
uploadModel.leaveCriticalSection();
}
}
uploadDesc = verb + " RDF from file " + rdfStream.getName();
} catch (IOException e) {
@ -198,6 +208,24 @@ public class RDFUploadController extends JenaIngestController {
}
}
private void addUsingRDFService(InputStream in, String languageStr,
RDFService rdfService) {
ChangeSet changeSet = rdfService.manufactureChangeSet();
RDFService.ModelSerializationFormat format =
("RDF/XML".equals(languageStr)
|| "RDF/XML-ABBREV".equals(languageStr))
? RDFService.ModelSerializationFormat.RDFXML
: RDFService.ModelSerializationFormat.N3;
changeSet.addAddition(in, format,
JenaDataSourceSetupBase.JENA_DB_MODEL);
try {
rdfService.changeSetUpdate(changeSet);
} catch (RDFServiceException rdfse) {
log.error(rdfse);
throw new RuntimeException(rdfse);
}
}
public void loadRDF(FileUploadServletRequest req,
VitroRequest request,
HttpServletResponse response)

69
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaChangeListener.java Normal file
View file

@ -0,0 +1,69 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.io.ByteArrayInputStream;
import java.io.UnsupportedEncodingException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
/**
* A ChangeListener that forwards events to a Jena ModelChangedListener
* @author bjl23
*
*/
public class JenaChangeListener implements ChangeListener {
private static final Log log = LogFactory.getLog(JenaChangeListener.class);
private ModelChangedListener listener;
private Model m = ModelFactory.createDefaultModel();
public JenaChangeListener(ModelChangedListener listener) {
this.listener = listener;
}
@Override
public void addedStatement(String serializedTriple, String graphURI) {
listener.addedStatement(parseTriple(serializedTriple));
}
@Override
public void removedStatement(String serializedTriple, String graphURI) {
listener.removedStatement(parseTriple(serializedTriple));
}
@Override
public void notifyEvent(String graphURI, Object event) {
listener.notifyEvent(m, event);
}
// TODO avoid overhead of Model
private Statement parseTriple(String serializedTriple) {
try {
Model m = ModelFactory.createDefaultModel();
m.read(new ByteArrayInputStream(
serializedTriple.getBytes("UTF-8")), null, "N3");
StmtIterator sit = m.listStatements();
if (!sit.hasNext()) {
throw new RuntimeException("no triple parsed from change event");
} else {
Statement s = sit.nextStatement();
if (sit.hasNext()) {
log.warn("More than one triple parsed from change event");
}
return s;
}
} catch (UnsupportedEncodingException uee) {
throw new RuntimeException(uee);
}
}
}

31
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java
View file

@ -4,13 +4,20 @@ package edu.cornell.mannlib.vitro.webapp.dao.jena;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
public class ModelContext {
private static final Log log = LogFactory.getLog(ModelContext.class);
private static final String ONT_MODEL_SELECTOR = "ontModelSelector";
private static final String UNION_ONT_MODEL_SELECTOR = "unionOntModelSelector";
private static final String BASE_ONT_MODEL_SELECTOR = "baseOntModelSelector";
@ -96,15 +103,23 @@ public class ModelContext {
* Changes to application model
*/
public static void registerListenerForChanges(ServletContext ctx, ModelChangedListener ml){
ModelContext.getJenaOntModel(ctx).register(ml);
ModelContext.getBaseOntModel(ctx).register(ml);
ModelContext.getInferenceOntModel(ctx).register(ml);
ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml);
ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml);
ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml);
ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml);
ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml);
try {
RDFServiceUtils.getRDFServiceFactory(ctx).registerListener(
new JenaChangeListener(ml));
} catch (RDFServiceException e) {
log.error(e,e);
}
// ModelContext.getJenaOntModel(ctx).register(ml);
// ModelContext.getBaseOntModel(ctx).register(ml);
// ModelContext.getInferenceOntModel(ctx).register(ml);
// ModelContext.getUnionOntModelSelector(ctx).getABoxModel().register(ml);
// ModelContext.getBaseOntModelSelector(ctx).getABoxModel().register(ml);
// ModelContext.getBaseOntModelSelector(ctx).getApplicationMetadataModel().register(ml);
// ModelContext.getInferenceOntModelSelector(ctx).getABoxModel().register(ml);
// ModelContext.getBaseOntModelSelector(ctx).getTBoxModel().register(ml);
}

38
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceGraphBulkUpdater.java
View file

@ -1,5 +1,7 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PipedInputStream;
import java.io.PipedOutputStream;
@ -71,7 +73,7 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
@Override
public void add(Graph g, boolean arg1) {
log.info("adding graph");
log.info("bulk addGraph()");
Model[] model = separateStatementsWithBlankNodes(g);
addModel(model[1] /* nonBlankNodeModel */);
// replace following call with different method
@ -112,46 +114,34 @@ public class RDFServiceGraphBulkUpdater extends SimpleBulkUpdateHandler {
@Override
public void delete(Graph g) {
ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
Model m = ModelFactory.createModelForGraph(g);
PipedOutputStream out = new PipedOutputStream();
m.write(out, "N-TRIPLE");
try {
changeSet.addRemoval(new PipedInputStream(
out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
graph.getRDFService().changeSetUpdate(changeSet);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
} catch (RDFServiceException rdfse) {
throw new RuntimeException(rdfse);
}
deleteModel(ModelFactory.createModelForGraph(g));
}
public void addModel(Model model) {
log.info("bulk addModel()");
ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
PipedOutputStream out = new PipedOutputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
model.write(out, "N-TRIPLE");
changeSet.addAddition(new ByteArrayInputStream(
out.toByteArray()), RDFService.ModelSerializationFormat.N3,
graph.getGraphURI());
try {
changeSet.addAddition(new PipedInputStream(
out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
graph.getRDFService().changeSetUpdate(changeSet);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
} catch (RDFServiceException rdfse) {
throw new RuntimeException(rdfse);
}
}
public void deleteModel(Model model) {
log.info("bulk addModel()");
ChangeSet changeSet = graph.getRDFService().manufactureChangeSet();
PipedOutputStream out = new PipedOutputStream();
ByteArrayOutputStream out = new ByteArrayOutputStream();
model.write(out, "N-TRIPLE");
changeSet.addRemoval(new ByteArrayInputStream(
out.toByteArray()), RDFService.ModelSerializationFormat.N3,
graph.getGraphURI());
try {
changeSet.addRemoval(new PipedInputStream(
out), RDFService.ModelSerializationFormat.N3, graph.getGraphURI());
graph.getRDFService().changeSetUpdate(changeSet);
} catch (IOException ioe) {
throw new RuntimeException(ioe);
} catch (RDFServiceException rdfse) {
throw new RuntimeException(rdfse);
}

6
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java
View file

@ -94,12 +94,6 @@ public class WebappDaoFactorySDBPrep implements Filter {
WebappDaoFactory wadf = null;
VitroRequest vreq = new VitroRequest((HttpServletRequest) request);
Enumeration<String> headStrs = vreq.getHeaderNames();
while (headStrs.hasMoreElements()) {
String head = headStrs.nextElement();
log.info(head + " : " + vreq.getHeader(head));
}
List<String> langs = new ArrayList<String>();
log.debug("Accept-Language: " + vreq.getHeader("Accept-Language"));

18
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java
View file

@ -6,6 +6,22 @@ public interface RDFServiceFactory {
public RDFService getRDFService();
public RDFService getRDFService(VitroRequest vreq);
/**
* Register a listener to listen to changes in any graph in
* the RDF store. Any RDFService objects returned by this factory should notify
* this listener of changes.
*
* @param changeListener - the change listener
*/
public void registerListener(ChangeListener changeListener) throws RDFServiceException;
/**
* Unregister a listener from listening to changes in
* the RDF store. Any RDFService objects returned by this factory should notify
* this listener of changes.
*
* @param changeListener - the change listener
*/
public void unregisterListener(ChangeListener changeListener) throws RDFServiceException;
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java
View file

@ -1,7 +1,8 @@
package edu.cornell.mannlib.vitro.webapp.rdfservice.impl;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
/**
@ -23,8 +24,13 @@ public class RDFServiceFactorySingle implements RDFServiceFactory {
}
@Override
public RDFService getRDFService(VitroRequest vreq) {
return this.rdfService;
public void registerListener(ChangeListener listener) throws RDFServiceException {
this.rdfService.registerListener(listener);
}
@Override
public void unregisterListener(ChangeListener listener) throws RDFServiceException {
this.rdfService.unregisterListener(listener);
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java
View file

@ -168,7 +168,7 @@ public abstract class RDFServiceImpl implements RDFService {
protected static String getSerializationFormatString(RDFService.ModelSerializationFormat format) {
switch (format) {
case RDFXML:
return "RDFXML";
return "RDF/XML";
case N3:
return "N3";
default:

7
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java
View file

@ -6,6 +6,8 @@ import java.io.UnsupportedEncodingException;
import javax.servlet.ServletContext;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
public class RDFServiceUtils {
@ -31,4 +33,9 @@ public class RDFServiceUtils {
}
}
public static RDFService getRDFService(VitroRequest vreq) {
return getRDFServiceFactory(
vreq.getSession().getServletContext()).getRDFService();
}
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java
View file

@ -94,7 +94,7 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
}
Dataset dataset = getDataset(conn);
boolean transaction = false; // conn.getTransactionHandler().transactionsSupported();
boolean transaction = conn.getTransactionHandler().transactionsSupported();
try {
if (transaction) {
@ -109,8 +109,8 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
try {
model.register(new ModelListener(modelChange.getGraphURI(), this));
if (modelChange.getOperation() == ModelChange.Operation.ADD) {
Model m = parseModel(modelChange);
model.add(m);
model.read(modelChange.getSerializedModel(), null,
getSerializationFormatString(modelChange.getSerializationFormat()));
} else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) {
model.remove(parseModel(modelChange));
removeBlankNodesWithSparqlUpdate(dataset, model, modelChange.getGraphURI());
@ -156,6 +156,12 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
StringBuffer patternBuff = new StringBuffer();
StmtIterator stmtIt = model.listStatements();
if (!stmtIt.hasNext()) {
stmtIt.close();
return;
}
while(stmtIt.hasNext()) {
Triple t = stmtIt.next().asTriple();
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));

13
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java
View file

@ -162,24 +162,15 @@ public class RDFServiceSetup extends JenaDataSourceSetupBase
// ABox assertions
Model aboxAssertions = dataset.getNamedModel(
JenaDataSourceSetupBase.JENA_DB_MODEL);
Model listenableAboxAssertions = ModelFactory.createUnion(
aboxAssertions, ModelFactory.createDefaultModel());
baseOms.setABoxModel(
ModelFactory.createOntologyModel(
OntModelSpec.OWL_MEM, listenableAboxAssertions));
OntModelSpec.OWL_MEM, aboxAssertions));
// ABox inferences
Model aboxInferences = dataset.getNamedModel(
JenaDataSourceSetupBase.JENA_INF_MODEL);
Model listenableAboxInferences = ModelFactory.createUnion(
aboxInferences, ModelFactory.createDefaultModel());
inferenceOms.setABoxModel(ModelFactory.createOntologyModel(
OntModelSpec.OWL_MEM, listenableAboxInferences));
// Since the TBox models are in memory, they do not have timeout issues
// like the like the ABox models do (and so don't need the extra step
// to make them listenable.)
OntModelSpec.OWL_MEM, aboxInferences));
// TBox assertions
try {