litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3779 bulk updates and blank node handling for SPARQL implementation of RDF API, and associated bug fixes

Browse source
This commit is contained in:
brianjlowe 2012-06-19 20:14:57 +00:00
parent 4b00e781b4
commit 70ad9831eb
15 changed files with 737 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/edit/ReorderController.java
View file

@ -106,7 +106,7 @@ public class ReorderController extends VitroAjaxController {
private void reorderIndividuals(String[] individualUris, VitroRequest vreq, String rankPredicate) {
//Testing new mechanism
OntModel writeModel = vreq.getJenaOntModel();
OntModel writeModel = vreq.getOntModelSelector().getABoxModel();
Model additions = ModelFactory.createDefaultModel();
Model retractions = ModelFactory.createDefaultModel();
Property rankPredicateProperty = ResourceFactory.createProperty(rankPredicate);

3
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/edit/VclassRetryController.java
View file

@ -49,7 +49,6 @@ public class VclassRetryController extends BaseEditController {
//create an EditProcessObject for this and put it in the session
EditProcessObject epo = super.createEpo(request);
epo.setDataAccessObject(request.getFullWebappDaoFactory().getVClassDao());
/*for testing*/
VClass testMask = new VClass();
@ -65,7 +64,7 @@ public class VclassRetryController extends BaseEditController {
action = epo.getAction();
}
VClassDao vcwDao = request.getFullWebappDaoFactory().getVClassDao();
VClassDao vcwDao = request.getAssertionsWebappDaoFactory().getVClassDao();
epo.setDataAccessObject(vcwDao);
VClassGroupDao cgDao = request.getFullWebappDaoFactory().getVClassGroupDao();
OntologyDao oDao = request.getFullWebappDaoFactory().getOntologyDao();

40
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/JenaBaseDao.java
View file

@ -1028,17 +1028,36 @@ public class JenaBaseDao extends JenaBaseDaoCon {
return directSubjectList;
}
/**
* Returns additions and retractions to perform
* @param ontRes
* @param ontModel
* @return Model[] where [0] is retractions and [1] is additions
*/
protected Model[] getSmartRemoval(OntResource ontRes, OntModel ontModel) {
Model[] changeSet = removeFromLists(ontRes, ontModel);
List<Statement> stmtForDependentRes = DependentResourceDeleteJena.getDependentResourceDeleteList(ontRes,ontModel);
changeSet[0].add(removeUsingDescribe(ontRes, ontModel));
changeSet[0].add(stmtForDependentRes);
return changeSet;
}
protected void smartRemove(OntResource ontRes, OntModel ontModel) {
removeFromLists(ontRes, ontModel);
List<Statement> stmtForDependentRes = DependentResourceDeleteJena.getDependentResourceDeleteList(ontRes,ontModel);
removeUsingDescribe(ontRes, ontModel);
ontModel.remove(stmtForDependentRes);
Model[] changes = getSmartRemoval(ontRes, ontModel);
ontModel.remove(changes[0]);
ontModel.add(changes[1]);
}
/**
* Removes a resource from any rdf:Lists in which it is a member
*/
private void removeFromLists(OntResource res, OntModel ontModel) {
private Model[] removeFromLists(OntResource res, OntModel ontModel) {
Model[] changeSet = new Model[2];
Model retractions = ModelFactory.createDefaultModel();
Model additions = ModelFactory.createDefaultModel();
changeSet[0] = retractions;
changeSet[1] = additions;
// Iterate through all of the list nodes this resource is attached to
Iterator<Resource> listNodeIt = ontModel.listSubjectsWithProperty(RDF.first, res);
while (listNodeIt.hasNext()) {
@ -1056,16 +1075,17 @@ public class JenaBaseDao extends JenaBaseDaoCon {
// if current node is list head
if (!nextNode.equals(RDF.nil)) {
// only repair the list if there is more than one node
ontModel.add(stmt.getSubject(), RDF.rest, nextNode);
additions.add(stmt.getSubject(), RDF.rest, nextNode);
}
} else {
ontModel.add(stmt.getSubject(), RDF.rest, nextNode);
additions.add(stmt.getSubject(), RDF.rest, nextNode);
}
}
}
//Remove any statements about this node
ontModel.remove(listNode, (Property) null, (RDFNode) null);
retractions.add(listNode, (Property) null, (RDFNode) null);
}
return changeSet;
}
public void removeRulesMentioningResource(Resource res, OntModel ontModel) {
@ -1093,10 +1113,10 @@ public class JenaBaseDao extends JenaBaseDaoCon {
// removes a resource and its bnode closure using ARQ's DESCRIBE semantics
// plus any incoming properties
private void removeUsingDescribe(OntResource ontRes, OntModel ontModel) {
private Model removeUsingDescribe(OntResource ontRes, OntModel ontModel) {
Model temp = describeResource(ontRes, ontModel);
temp.add(ontModel.listStatements((Resource) null, (Property) null, ontRes));
ontModel.remove(temp);
return temp;
}
private Model describeResource(Resource res, OntModel ontModel) {

15
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassDaoJena.java
View file

@ -38,6 +38,7 @@ import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.ResIterator;
@ -1026,15 +1027,23 @@ public class VClassDaoJena extends JenaBaseDao implements VClassDao {
try {
OntResource subclass = getOntClass(ontModel,c2c.getSubclassURI());
OntResource superclass = getOntClass(ontModel,c2c.getSuperclassURI());
Model removal = ModelFactory.createDefaultModel();
Model additions = ModelFactory.createDefaultModel(); // to repair any rdf:Lists
if ((subclass != null) && (superclass != null)) {
ontModel.removeAll(subclass, RDFS.subClassOf, superclass);
removal.add(ontModel.listStatements(subclass, RDFS.subClassOf, superclass));
}
if (subclass.isAnon()) {
smartRemove(subclass, getOntModel());
Model[] changeSet = getSmartRemoval(subclass, getOntModel());
removal.add(changeSet[0]);
additions.add(changeSet[1]);
}
if (superclass.isAnon()) {
smartRemove(superclass, getOntModel());
Model[] changeSet = getSmartRemoval(superclass, getOntModel());
removal.add(changeSet[0]);
additions.add(changeSet[1]);
}
ontModel.remove(removal);
ontModel.add(additions);
} finally {
ontModel.getBaseModel().notifyEvent(new EditEvent(getWebappDaoFactory().getUserURI(),false));
ontModel.leaveCriticalSection();

2
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/WebappDaoFactorySDB.java
View file

@ -147,7 +147,7 @@ public class WebappDaoFactorySDB extends WebappDaoFactoryJena {
@Override
public void close() {
super.close();
this.rdfService.close();
//this.rdfService.close();
}
private class ReconnectingDatasetFactory implements DatasetWrapperFactory {

2
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java
View file

@ -119,7 +119,7 @@ public class WebappDaoFactorySDBPrep implements Filter {
vreq.setAssertionsWebappDaoFactory(assertions);
vreq.setFullWebappDaoFactory(wadf);
vreq.setDataset(dataset);
vreq.setOntModelSelector(oms);
vreq.setOntModelSelector(baseOms);
vreq.setJenaOntModel(ModelFactory.createOntologyModel(
OntModelSpec.OWL_MEM, dataset.getDefaultModel()));

2
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java
View file

@ -2,8 +2,6 @@
package edu.cornell.mannlib.vitro.webapp.rdfservice;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
public interface RDFServiceFactory {
public RDFService getRDFService();

65
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/FilteredResultSet.java Normal file
View file

@ -0,0 +1,65 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.rdfservice.filter;
import java.util.Iterator;
import java.util.List;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.sparql.engine.binding.Binding;
public class FilteredResultSet implements ResultSet {
private Iterator<QuerySolution> solutIt;
private ResultSet originalResultSet;
private int rowNum = -1;
public FilteredResultSet (List<QuerySolution> solutions, ResultSet originalResultSet) {
this.solutIt = solutions.iterator();
this.originalResultSet = originalResultSet;
}
@Override
public void remove() {
throw new UnsupportedOperationException("Attempt to remove an element");
}
@Override
public Model getResourceModel() {
return originalResultSet.getResourceModel();
}
@Override
public List<String> getResultVars() {
return originalResultSet.getResultVars();
}
@Override
public int getRowNumber() {
return rowNum;
}
@Override
public boolean hasNext() {
return solutIt.hasNext();
}
@Override
public QuerySolution next() {
return nextSolution();
}
@Override
public Binding nextBinding() {
throw new UnsupportedOperationException("Can we ignore this?");
}
@Override
public QuerySolution nextSolution() {
rowNum++;
return solutIt.next();
}
}

246
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java Normal file
View file

@ -0,0 +1,246 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.rdfservice.filter;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.ResultSetFactory;
import com.hp.hpl.jena.query.ResultSetFormatter;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceImpl;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
public class SameAsFilteringRDFServiceFactory implements RDFServiceFactory {
private final static Log log = LogFactory.getLog(
SameAsFilteringRDFServiceFactory.class);
private RDFServiceFactory f;
private Model sameAsModel;
public SameAsFilteringRDFServiceFactory(RDFServiceFactory rdfServiceFactory) {
this.f = rdfServiceFactory;
try {
InputStream in = f.getRDFService().sparqlConstructQuery("CONSTRUCT { ?s <" + OWL.sameAs.getURI() + "> ?o } WHERE { ?s <" + OWL.sameAs.getURI() + "> ?o } ", ModelSerializationFormat.N3);
sameAsModel = RDFServiceUtils.parseModel(in, ModelSerializationFormat.N3);
} catch (RDFServiceException e) {
throw new RuntimeException(e);
}
}
@Override
public RDFService getRDFService() {
return new SameAsFilteringRDFService(f.getRDFService());
}
@Override
public void registerListener(ChangeListener changeListener) throws RDFServiceException {
f.registerListener(changeListener);
}
@Override
public void unregisterListener(ChangeListener changeListener) throws RDFServiceException {
f.registerListener(changeListener);
}
public class SameAsFilteringRDFService extends RDFServiceImpl implements RDFService {
private final Log log = LogFactory.getLog(SameAsFilteringRDFService.class);
private RDFService s;
public SameAsFilteringRDFService(RDFService rdfService) {
this.s = rdfService;
}
@Override
public InputStream sparqlConstructQuery(String query,
RDFService.ModelSerializationFormat resultFormat)
throws RDFServiceException {
Model m = RDFServiceUtils.parseModel(
s.sparqlConstructQuery(query, resultFormat), resultFormat);
Model filtered = ModelFactory.createDefaultModel();
StmtIterator stmtIt = m.listStatements();
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (!isRedundant(stmt)) {
filtered.add(stmt);
}
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
filtered.write(out, RDFServiceUtils.getSerializationFormatString(
resultFormat));
return new ByteArrayInputStream(out.toByteArray());
}
@Override
public InputStream sparqlSelectQuery(String query, ResultFormat resultFormat)
throws RDFServiceException {
ResultSet rs = ResultSetFactory.load(
s.sparqlSelectQuery(query, resultFormat),
RDFServiceUtils.getJenaResultSetFormat(resultFormat));
List<QuerySolution> solutions = new ArrayList<QuerySolution>();
while (rs.hasNext()) {
QuerySolution solution = rs.nextSolution();
if (!isRedundant(solution)) {
solutions.add(solution);
}
}
ResultSet resultSet = new FilteredResultSet(solutions, rs);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
switch (resultFormat) {
case CSV:
ResultSetFormatter.outputAsCSV(outputStream,resultSet);
break;
case TEXT:
ResultSetFormatter.out(outputStream,resultSet);
break;
case JSON:
ResultSetFormatter.outputAsJSON(outputStream, resultSet);
break;
case XML:
ResultSetFormatter.outputAsXML(outputStream, resultSet);
break;
default:
throw new RDFServiceException("unrecognized result format");
}
return new ByteArrayInputStream(outputStream.toByteArray());
}
private boolean isRedundant(Statement s) {
List<Resource> sameAsResources = getSameAsResources(s.getSubject());
if (sameAsResources.size() > 0 && !sameAsResources.get(0).equals(s.getSubject())) {
return true;
}
if (s.getObject().isLiteral() || s.getObject().isAnon()) {
return false;
}
sameAsResources = getSameAsResources(s.getObject().asResource());
if (sameAsResources.size() > 0 && !sameAsResources.get(0).equals(s.getObject().asResource())) {
return true;
}
return false;
}
private List<Resource> getSameAsResources(Resource resource) {
List<Resource> sameAsResources = new ArrayList<Resource>();
if (resource.isAnon()) {
return sameAsResources;
}
String queryStr = "SELECT DISTINCT ?s WHERE { <" + resource.getURI() + "> <" + OWL.sameAs.getURI() + "> ?s } ORDER BY ?s";
try {
Query query = QueryFactory.create(queryStr);
QueryExecution qe = QueryExecutionFactory.create(query, sameAsModel);
try {
ResultSet rs = qe.execSelect();
//ResultSet rs = JSONInput.fromJSON(s.sparqlSelectQuery(queryStr, ResultFormat.JSON));
while (rs.hasNext()) {
QuerySolution q = rs.next();
Resource res = q.getResource("s");
if (s != null) {
log.info("adding same as " + res.getURI());
sameAsResources.add(res);
}
}
} finally {
qe.close();
}
return sameAsResources;
} catch (/*RDFService*/Exception e) {
throw new RuntimeException(e);
}
}
private boolean isRedundant(QuerySolution q) {
Iterator<String> varIt = q.varNames();
while(varIt.hasNext()) {
String varName = varIt.next();
RDFNode n = q.get(varName);
if (n.isResource()) {
Resource r = n.asResource();
List<Resource> sames = getSameAsResources(r);
if (sames.size() > 0 && !sames.get(0).equals(r)) {
return true;
}
}
}
return false;
}
@Override
public boolean changeSetUpdate(ChangeSet changeSet)
throws RDFServiceException {
return s.changeSetUpdate(changeSet);
}
@Override
public InputStream sparqlDescribeQuery(String query,
ModelSerializationFormat resultFormat)
throws RDFServiceException {
Model m = RDFServiceUtils.parseModel(
s.sparqlConstructQuery(query, resultFormat), resultFormat);
Model filtered = ModelFactory.createDefaultModel();
StmtIterator stmtIt = m.listStatements();
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (!isRedundant(stmt)) {
filtered.add(stmt);
}
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
filtered.write(out, RDFServiceUtils.getSerializationFormatString(
resultFormat));
return new ByteArrayInputStream(out.toByteArray());
}
@Override
public boolean sparqlAskQuery(String query) throws RDFServiceException {
return s.sparqlAskQuery(query);
}
@Override
public List<String> getGraphURIs() throws RDFServiceException {
return s.getGraphURIs();
}
@Override
public void getGraphMetadata() throws RDFServiceException {
s.getGraphMetadata();
}
@Override
public void close() {
s.close();
}
}
}

11
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java
View file

@ -132,7 +132,11 @@ public abstract class RDFServiceImpl implements RDFService {
return new ChangeSetImpl();
}
protected void notifyListeners(Triple triple, ModelChange.Operation operation, String graphURI) {
// I switched the following two methods back to public so they could be
// used by the ListeningGraph, which is common to both implementations.
// This could probably be improved later. BJL
public void notifyListeners(Triple triple, ModelChange.Operation operation, String graphURI) {
Iterator<ChangeListener> iter = registeredListeners.iterator();
while (iter.hasNext()) {
@ -145,8 +149,7 @@ public abstract class RDFServiceImpl implements RDFService {
}
}
protected void notifyListenersOfEvent(Object event) {
public void notifyListenersOfEvent(Object event) {
Iterator<ChangeListener> iter = registeredListeners.iterator();
while (iter.hasNext()) {
@ -181,7 +184,7 @@ public abstract class RDFServiceImpl implements RDFService {
case RDFXML:
return "RDF/XML";
case N3:
return "N3";
return "TTL";
default:
log.error("unexpected format in getFormatString");
return null;

39
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceUtils.java
View file

@ -8,8 +8,14 @@ import java.io.UnsupportedEncodingException;
import javax.servlet.ServletContext;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.sparql.resultset.ResultSetFormat;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
public class RDFServiceUtils {
@ -35,6 +41,39 @@ public class RDFServiceUtils {
}
}
public static Model parseModel(InputStream in, ModelSerializationFormat format) {
Model model = ModelFactory.createDefaultModel();
model.read(in, null,
getSerializationFormatString(format));
return model;
}
public static ResultSetFormat getJenaResultSetFormat(ResultFormat resultFormat) {
switch(resultFormat) {
case JSON:
return ResultSetFormat.syntaxJSON;
case CSV:
return ResultSetFormat.syntaxCSV;
case XML:
return ResultSetFormat.syntaxXML;
case TEXT:
return ResultSetFormat.syntaxText;
default:
throw new RuntimeException("unsupported ResultFormat");
}
}
public static String getSerializationFormatString(RDFService.ModelSerializationFormat format) {
switch (format) {
case RDFXML:
return "RDF/XML";
case N3:
return "N3";
default:
throw new RuntimeException("unexpected format in getFormatString");
}
}
public static RDFService getRDFService(VitroRequest vreq) {
return getRDFServiceFactory(
vreq.getSession().getServletContext()).getRDFService();

17
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/ListeningGraph.java
View file

@ -32,12 +32,13 @@ import com.hp.hpl.jena.util.iterator.WrappedIterator;
import edu.cornell.mannlib.vitro.webapp.dao.jena.EmptyReifier;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceImpl;
public class ListeningGraph implements GraphWithPerform {
private static final Log log = LogFactory.getLog(ListeningGraph.class);
private RDFServiceSDB rdfServiceSDB;
private RDFServiceImpl rdfServiceImpl;
private String graphURI;
private BulkUpdateHandler bulkUpdateHandler;
@ -46,9 +47,9 @@ public class ListeningGraph implements GraphWithPerform {
private Reifier reifier = new EmptyReifier(this);
private QueryHandler queryHandler;
public ListeningGraph(String graphURI, RDFServiceSDB rdfServiceSDB) {
public ListeningGraph(String graphURI, RDFServiceImpl rdfServiceImpl) {
this.graphURI = graphURI;
this.rdfServiceSDB = rdfServiceSDB;
this.rdfServiceImpl = rdfServiceImpl;
}
@Override
@ -58,7 +59,10 @@ public class ListeningGraph implements GraphWithPerform {
@Override
public void performAdd(Triple triple) throws AddDeniedException {
this.rdfServiceSDB.notifyListeners(triple, ModelChange.Operation.ADD, graphURI);
if (log.isDebugEnabled()) {
log.debug("adding " + triple + " to " + graphURI);
}
this.rdfServiceImpl.notifyListeners(triple, ModelChange.Operation.ADD, graphURI);
}
@Override
@ -68,7 +72,10 @@ public class ListeningGraph implements GraphWithPerform {
@Override
public void performDelete(Triple triple) throws DeleteDeniedException {
this.rdfServiceSDB.notifyListeners(triple, ModelChange.Operation.REMOVE, graphURI);
if (log.isDebugEnabled()) {
log.debug("deleting " + triple + " from " + graphURI);
}
this.rdfServiceImpl.notifyListeners(triple, ModelChange.Operation.REMOVE, graphURI);
}
@Override

84
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sdb/RDFServiceSDB.java
View file

@ -197,6 +197,11 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
log.debug("removal model size " + model.size());
log.debug("blank node model size " + blankNodeModel.size());
if (blankNodeModel.size() == 1) {
log.warn("Deleting single triple with blank node: " + blankNodeModel);
log.warn("This likely indicates a problem; excessive data may be deleted.");
}
String rootFinder = "SELECT ?s WHERE { ?s ?p ?o OPTIONAL { ?ss ?pp ?s } FILTER (!bound(?ss)) }";
Query rootFinderQuery = QueryFactory.create(rootFinder);
QueryExecution qe = QueryExecutionFactory.create(rootFinderQuery, blankNodeModel);
@ -210,32 +215,36 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
QueryExecution qee = QueryExecutionFactory.create(treeFinderQuery, blankNodeModel);
try {
Model tree = qee.execDescribe();
StmtIterator sit = tree.listStatements(s, null, (RDFNode) null);
while (sit.hasNext()) {
Statement stmt = sit.nextStatement();
RDFNode n = stmt.getObject();
Model m2 = ModelFactory.createDefaultModel();
if (n.isResource()) {
Resource s2 = (Resource) n;
// now run yet another describe query
String smallerTree = makeDescribe(s2);
Query smallerTreeQuery = QueryFactory.create(smallerTree);
QueryExecution qe3 = QueryExecutionFactory.create(
smallerTreeQuery, tree);
try {
qe3.execDescribe(m2);
} finally {
qe3.close();
DataSource ds = DatasetFactory.create();
if (graphURI == null) {
ds.setDefaultModel(dataset.getDefaultModel());
} else {
ds.addNamedModel(graphURI, dataset.getNamedModel(graphURI));
}
if (s.isAnon()) {
removeUsingSparqlUpdate(ds, tree, graphURI);
} else {
StmtIterator sit = tree.listStatements(s, null, (RDFNode) null);
while (sit.hasNext()) {
Statement stmt = sit.nextStatement();
RDFNode n = stmt.getObject();
Model m2 = ModelFactory.createDefaultModel();
if (n.isResource()) {
Resource s2 = (Resource) n;
// now run yet another describe query
String smallerTree = makeDescribe(s2);
Query smallerTreeQuery = QueryFactory.create(smallerTree);
QueryExecution qe3 = QueryExecutionFactory.create(
smallerTreeQuery, tree);
try {
qe3.execDescribe(m2);
} finally {
qe3.close();
}
}
m2.add(stmt);
removeUsingSparqlUpdate(ds, m2, graphURI);
}
m2.add(stmt);
DataSource ds = DatasetFactory.create();
if (graphURI == null) {
ds.setDefaultModel(dataset.getDefaultModel());
} else {
ds.addNamedModel(graphURI, dataset.getNamedModel(graphURI));
}
removeUsingSparqlUpdate(ds, m2, graphURI);
}
} finally {
qee.close();
@ -279,12 +288,13 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
StringBuffer queryBuff = new StringBuffer();
queryBuff.append("CONSTRUCT { \n");
queryBuff.append(patternBuff);
addStatementPatterns(stmtIt, queryBuff, !WHERE_CLAUSE);
queryBuff.append("} WHERE { \n");
if (graphURI != null) {
queryBuff.append(" GRAPH <" + graphURI + "> { \n");
}
queryBuff.append(patternBuff);
stmtIt = model.listStatements();
addStatementPatterns(stmtIt, queryBuff, !WHERE_CLAUSE);
if (graphURI != null) {
queryBuff.append(" } \n");
}
@ -314,6 +324,28 @@ public class RDFServiceSDB extends RDFServiceImpl implements RDFService {
}
}
private static final boolean WHERE_CLAUSE = true;
private void addStatementPatterns(StmtIterator stmtIt, StringBuffer patternBuff, boolean whereClause) {
while(stmtIt.hasNext()) {
Triple t = stmtIt.next().asTriple();
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));
patternBuff.append(" ");
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getPredicate(), null));
patternBuff.append(" ");
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getObject(), null));
patternBuff.append(" .\n");
if (whereClause) {
if (t.getSubject().isBlank()) {
patternBuff.append(" FILTER(isBlank(" + SparqlGraph.sparqlNodeDelete(t.getSubject(), null)).append(")) \n");
}
if (t.getObject().isBlank()) {
patternBuff.append(" FILTER(isBlank(" + SparqlGraph.sparqlNodeDelete(t.getObject(), null)).append(")) \n");
}
}
}
}
private Model parseModel(ModelChange modelChange) {
Model model = ModelFactory.createDefaultModel();
model.read(modelChange.getSerializedModel(), null,

276
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java
View file

@ -5,6 +5,7 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sparql;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -27,13 +28,16 @@ import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.ResultSetFormatter;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange;
@ -41,6 +45,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.ChangeSetImpl;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceImpl;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sdb.ListeningGraph;
/*
* API to write, read, and update Vitro's RDF store, with support
@ -98,33 +103,62 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
*
* @return boolean - indicates whether the precondition was satisfied
*/
@Override
public boolean changeSetUpdate(ChangeSet changeSet) throws RDFServiceException {
@Override
public boolean changeSetUpdate(ChangeSet changeSet)
throws RDFServiceException {
if (changeSet.getPreconditionQuery() != null
if (changeSet.getPreconditionQuery() != null
&& !isPreconditionSatisfied(
changeSet.getPreconditionQuery(),
changeSet.getPreconditionQueryType())) {
return false;
}
Iterator<ModelChange> csIt = changeSet.getModelChanges().iterator();
try {
for (Object o : changeSet.getPreChangeEvents()) {
this.notifyListenersOfEvent(o);
}
while (csIt.hasNext()) {
Iterator<ModelChange> csIt = changeSet.getModelChanges().iterator();
while (csIt.hasNext()) {
ModelChange modelChange = csIt.next();
modelChange.getSerializedModel().mark(Integer.MAX_VALUE);
performChange(modelChange);
}
ModelChange modelChange = csIt.next();
// notify listeners of triple changes
csIt = changeSet.getModelChanges().iterator();
while (csIt.hasNext()) {
ModelChange modelChange = csIt.next();
modelChange.getSerializedModel().reset();
Model model = ModelFactory.createModelForGraph(
new ListeningGraph(modelChange.getGraphURI(), this));
if (modelChange.getOperation() == ModelChange.Operation.ADD) {
model.read(modelChange.getSerializedModel(), null,
getSerializationFormatString(
modelChange.getSerializationFormat()));
} else if (modelChange.getOperation() == ModelChange.Operation.REMOVE){
Model temp = ModelFactory.createDefaultModel();
temp.read(modelChange.getSerializedModel(), null,
getSerializationFormatString(
modelChange.getSerializationFormat()));
model.remove(temp);
} else {
log.error("Unsupported model change type " +
modelChange.getOperation().getClass().getName());
}
}
if (modelChange.getOperation() == ModelChange.Operation.ADD) {
performAdd(modelChange);
} else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) {
performRemove(modelChange);
} else {
log.error("unrecognized operation type");
}
}
for (Object o : changeSet.getPostChangeEvents()) {
this.notifyListenersOfEvent(o);
}
return true;
}
} catch (Exception e) {
log.error(e, e);
throw new RDFServiceException(e);
}
return true;
}
/**
* Performs a SPARQL construct query against the knowledge base. The query may have
@ -368,6 +402,43 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
}
}
public void addModel(Model model, String graphURI) {
verbModel(model, graphURI, "INSERT");
}
public void deleteModel(Model model, String graphURI) {
verbModel(model, graphURI, "DELETE");
}
private void verbModel(Model model, String graphURI, String verb) {
Model m = ModelFactory.createDefaultModel();
int testLimit = 1000;
StmtIterator stmtIt = model.listStatements();
int count = 0;
try {
while (stmtIt.hasNext()) {
count++;
m.add(stmtIt.nextStatement());
if (count % testLimit == 0 || !stmtIt.hasNext()) {
StringWriter sw = new StringWriter();
m.write(sw, "N-TRIPLE");
StringBuffer updateStringBuff = new StringBuffer();
updateStringBuff.append(verb + " DATA { " + ((graphURI != null) ? "GRAPH <" + graphURI + "> { " : "" ));
updateStringBuff.append(sw);
updateStringBuff.append(((graphURI != null) ? " } " : "") + " }");
String updateString = updateStringBuff.toString();
executeUpdate(updateString);
m.removeAll();
}
}
} finally {
stmtIt.close();
}
}
protected void addTriple(Triple t, String graphURI) {
StringBuffer updateString = new StringBuffer();
@ -438,32 +509,165 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
}
}
protected void performAdd(ModelChange modelChange) throws RDFServiceException {
private void performChange(ModelChange modelChange) {
Model model = parseModel(modelChange);
if (modelChange.getOperation() == ModelChange.Operation.ADD) {
addModel(model, modelChange.getGraphURI());
} else if (modelChange.getOperation() == ModelChange.Operation.REMOVE) {
deleteModel(model, modelChange.getGraphURI());
removeBlankNodesWithSparqlUpdate(model, modelChange.getGraphURI());
} else {
log.error("unrecognized operation type");
}
}
Model model = ModelFactory.createDefaultModel();
model.read(modelChange.getSerializedModel(),getSerializationFormatString(modelChange.getSerializationFormat()));
private void removeBlankNodesWithSparqlUpdate(Model model, String graphURI) {
List<Statement> blankNodeStatements = new ArrayList<Statement>();
StmtIterator stmtIt = model.listStatements();
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (stmt.getSubject().isAnon() || stmt.getObject().isAnon()) {
blankNodeStatements.add(stmt);
}
}
StmtIterator stmtIt = model.listStatements();
if(blankNodeStatements.size() == 0) {
return;
}
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.next();
Triple triple = new Triple(stmt.getSubject().asNode(), stmt.getPredicate().asNode(), stmt.getObject().asNode());
addTriple(triple, modelChange.getGraphURI());
}
}
Model blankNodeModel = ModelFactory.createDefaultModel();
blankNodeModel.add(blankNodeStatements);
protected void performRemove(ModelChange modelChange) throws RDFServiceException {
log.debug("removal model size " + model.size());
log.debug("blank node model size " + blankNodeModel.size());
Model model = ModelFactory.createDefaultModel();
model.read(modelChange.getSerializedModel(),getSerializationFormatString(modelChange.getSerializationFormat()));
if (blankNodeModel.size() == 1) {
log.warn("Deleting single triple with blank node: " + blankNodeModel);
log.warn("This likely indicates a problem; excessive data may be deleted.");
}
StmtIterator stmtIt = model.listStatements();
String rootFinder = "SELECT ?s WHERE { ?s ?p ?o OPTIONAL { ?ss ?pp ?s } FILTER (!bound(?ss)) }";
Query rootFinderQuery = QueryFactory.create(rootFinder);
QueryExecution qe = QueryExecutionFactory.create(rootFinderQuery, blankNodeModel);
try {
ResultSet rs = qe.execSelect();
while (rs.hasNext()) {
QuerySolution qs = rs.next();
com.hp.hpl.jena.rdf.model.Resource s = qs.getResource("s");
String treeFinder = makeDescribe(s);
Query treeFinderQuery = QueryFactory.create(treeFinder);
QueryExecution qee = QueryExecutionFactory.create(treeFinderQuery, blankNodeModel);
try {
Model tree = qee.execDescribe();
if (s.isAnon()) {
removeUsingSparqlUpdate(tree, graphURI);
} else {
StmtIterator sit = tree.listStatements(s, null, (RDFNode) null);
while (sit.hasNext()) {
Statement stmt = sit.nextStatement();
RDFNode n = stmt.getObject();
Model m2 = ModelFactory.createDefaultModel();
if (n.isResource()) {
com.hp.hpl.jena.rdf.model.Resource s2 =
(com.hp.hpl.jena.rdf.model.Resource) n;
// now run yet another describe query
String smallerTree = makeDescribe(s2);
Query smallerTreeQuery = QueryFactory.create(smallerTree);
QueryExecution qe3 = QueryExecutionFactory.create(
smallerTreeQuery, tree);
try {
qe3.execDescribe(m2);
} finally {
qe3.close();
}
}
m2.add(stmt);
removeUsingSparqlUpdate(m2, graphURI);
}
}
} finally {
qee.close();
}
}
} finally {
qe.close();
}
}
while (stmtIt.hasNext()) {
Statement stmt = stmtIt.next();
Triple triple = new Triple(stmt.getSubject().asNode(), stmt.getPredicate().asNode(), stmt.getObject().asNode());
removeTriple(triple, modelChange.getGraphURI());
}
}
private void removeUsingSparqlUpdate(Model model, String graphURI) {
StmtIterator stmtIt = model.listStatements();
if (!stmtIt.hasNext()) {
stmtIt.close();
return;
}
StringBuffer queryBuff = new StringBuffer();
queryBuff.append("DELETE { \n");
if (graphURI != null) {
queryBuff.append(" GRAPH <" + graphURI + "> { \n");
}
addStatementPatterns(stmtIt, queryBuff, !WHERE_CLAUSE);
if (graphURI != null) {
queryBuff.append(" } \n");
}
queryBuff.append("} WHERE { \n");
if (graphURI != null) {
queryBuff.append(" GRAPH <" + graphURI + "> { \n");
}
stmtIt = model.listStatements();
addStatementPatterns(stmtIt, queryBuff, WHERE_CLAUSE);
if (graphURI != null) {
queryBuff.append(" } \n");
}
queryBuff.append("} \n");
if(log.isDebugEnabled()) {
log.debug(queryBuff.toString());
}
executeUpdate(queryBuff.toString());
}
private static final boolean WHERE_CLAUSE = true;
private void addStatementPatterns(StmtIterator stmtIt, StringBuffer patternBuff, boolean whereClause) {
while(stmtIt.hasNext()) {
Triple t = stmtIt.next().asTriple();
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getSubject(), null));
patternBuff.append(" ");
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getPredicate(), null));
patternBuff.append(" ");
patternBuff.append(SparqlGraph.sparqlNodeDelete(t.getObject(), null));
patternBuff.append(" .\n");
if (whereClause) {
if (t.getSubject().isBlank()) {
patternBuff.append(" FILTER(isBlank(" + SparqlGraph.sparqlNodeDelete(t.getSubject(), null)).append(")) \n");
}
if (t.getObject().isBlank()) {
patternBuff.append(" FILTER(isBlank(" + SparqlGraph.sparqlNodeDelete(t.getObject(), null)).append(")) \n");
}
}
}
}
private String makeDescribe(com.hp.hpl.jena.rdf.model.Resource s) {
StringBuffer query = new StringBuffer("DESCRIBE <") ;
if (s.isAnon()) {
query.append("_:" + s.getId().toString());
} else {
query.append(s.getURI());
}
query.append(">");
return query.toString();
}
private Model parseModel(ModelChange modelChange) {
Model model = ModelFactory.createDefaultModel();
model.read(modelChange.getSerializedModel(), null,
getSerializationFormatString(modelChange.getSerializationFormat()));
return model;
}
}

7
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java
View file

@ -22,6 +22,7 @@ import com.hp.hpl.jena.sdb.util.StoreUtils;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.filter.SameAsFilteringRDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sdb.RDFServiceSDB;
@ -49,6 +50,11 @@ implements javax.servlet.ServletContextListener {
} else {
useSDB(ctx, ss);
}
//experimental
//RDFServiceFactory factory = RDFServiceUtils.getRDFServiceFactory(ctx);
//RDFServiceUtils.setRDFServiceFactory(ctx, new SameAsFilteringRDFServiceFactory(factory));
} catch (SQLException e) {
ss.fatal(this, "Exception in RDFServiceSetup", e);
}
@ -58,6 +64,7 @@ implements javax.servlet.ServletContextListener {
RDFService rdfService = new RDFServiceSparql(endpointURI);
RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySingle(rdfService);
RDFServiceUtils.setRDFServiceFactory(ctx, rdfServiceFactory);
log.info("Using endpoint at " + endpointURI);
}
private void useSDB(ServletContext ctx, StartupStatus ss) throws SQLException {