From ab60341355fd4e8d7305f9a91050ffafa3449881 Mon Sep 17 00:00:00 2001 From: Jim Blake Date: Tue, 17 Feb 2015 16:28:45 -0500 Subject: [PATCH] VIVO-907 Handle RDFService quirks for TDB and Virtuoso Modify signature for isEquivalentGraph() - might throw RDFServiceException Modify RDFServiceTDB to work around the nonNegativeIntegerBug Implement toString() for the entire RDFServiceImpl hierarchy Parameterize ContentTripleSourceSPARQL.createRDFService() so ContentTripleSourceVirtuoso can override it. Create ContentTripleSourceVirtuoso and RDFServiceVirtuoso, to work around the nonNegativeInteger bug and the "INSERT DATA" problem. --- .../vitro/webapp/rdfservice/RDFService.java | 3 +- .../filter/LanguageFilteringRDFService.java | 2 +- .../SameAsFilteringRDFServiceFactory.java | 2 +- .../impl/RDFServiceFactorySingle.java | 2 +- .../rdfservice/impl/RDFServiceImpl.java | 7 +- .../rdfservice/impl/jena/RDFServiceJena.java | 2 +- .../impl/jena/sdb/RDFServiceSDB.java | 6 -- .../impl/jena/tdb/RDFServiceTDB.java | 42 +++++++++- .../impl/logging/LoggingRDFService.java | 3 +- .../impl/sparql/RDFServiceSparql.java | 2 +- .../impl/virtuoso/RDFServiceVirtuoso.java | 83 +++++++++++++++++++ .../webapp/servlet/setup/FileGraphSetup.java | 3 +- .../sparql/ContentTripleSourceSPARQL.java | 20 +++-- .../virtuoso/ContentTripleSourceVirtuoso.java | 29 +++++++ 14 files changed, 179 insertions(+), 27 deletions(-) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/virtuoso/ContentTripleSourceVirtuoso.java diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java index c43a5a583..1df550e7e 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java @@ -169,7 +169,8 @@ public interface RDFService { * @param serializedGraph - the contents to be compared with the existing graph. May not be null. * @param serializationFormat - May not be null. */ - public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, ModelSerializationFormat serializationFormat); + public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, + ModelSerializationFormat serializationFormat) throws RDFServiceException; /** * Registers a listener to listen to changes in any graph in diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/LanguageFilteringRDFService.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/LanguageFilteringRDFService.java index b35660a8a..f7db0caf6 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/LanguageFilteringRDFService.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/LanguageFilteringRDFService.java @@ -338,7 +338,7 @@ public class LanguageFilteringRDFService implements RDFService { @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) throws RDFServiceException { return s.isEquivalentGraph(graphURI, serializedGraph, serializationFormat); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java index 222e11052..b7b5e3836 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java @@ -255,7 +255,7 @@ public class SameAsFilteringRDFServiceFactory implements RDFServiceFactory { @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) throws RDFServiceException { return s.isEquivalentGraph(graphURI, serializedGraph, serializationFormat); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java index 96b2a4233..ce1fedc08 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java @@ -128,7 +128,7 @@ public class RDFServiceFactorySingle implements RDFServiceFactory { @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) throws RDFServiceException { return s.isEquivalentGraph(graphURI, serializedGraph, serializationFormat); } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java index 2fafb20ce..58b17bc6b 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java @@ -11,7 +11,6 @@ import java.util.concurrent.CopyOnWriteArrayList; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import com.hp.hpl.jena.graph.Graph; import com.hp.hpl.jena.graph.Node; import com.hp.hpl.jena.graph.NodeFactory; import com.hp.hpl.jena.graph.Triple; @@ -32,6 +31,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; +import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; public abstract class RDFServiceImpl implements RDFService { @@ -286,5 +286,10 @@ public abstract class RDFServiceImpl implements RDFService { } return q; } + + @Override + public String toString() { + return ToString.simpleName(this) + "[" + ToString.hashHex(this) + "]"; + } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java index d39e9deb7..57ad6dd46 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java @@ -562,7 +562,7 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic */ @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) throws RDFServiceException { Model fileModel = RDFServiceUtils.parseModel(serializedGraph, serializationFormat); Model tripleStoreModel = new RDFServiceDataset(this).getNamedModel(graphURI); Model fromTripleStoreModel = ModelFactory.createDefaultModel().add(tripleStoreModel); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java index 40fb8edda..91b57d403 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java @@ -27,7 +27,6 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.RDFServiceJena; -import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; public class RDFServiceSDB extends RDFServiceJena implements RDFService { @@ -172,9 +171,4 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService { } } - @Override - public String toString() { - return "RDFServiceSDB[" + ToString.hashHex(this) + "]"; - } - } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/tdb/RDFServiceTDB.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/tdb/RDFServiceTDB.java index 307ff438c..f611b9c48 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/tdb/RDFServiceTDB.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/tdb/RDFServiceTDB.java @@ -2,6 +2,7 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.tdb; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -10,6 +11,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -22,7 +24,6 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.DatasetWrapper; import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.RDFServiceJena; -import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; /** * An implementation that is based on Jena TDB. @@ -177,8 +178,43 @@ public class RDFServiceTDB extends RDFServiceJena { } } + /** + * TDB has a bug: if given a literal of type xsd:nonNegativeInteger, it + * stores a literal of type xsd:integer. + * + * To determine whether this serialized graph is equivalent to what's in + * TDB, we need to do the same. + */ @Override - public String toString() { - return "RDFServiceTDB[" + ToString.hashHex(this) + "]"; + public boolean isEquivalentGraph(String graphURI, + InputStream serializedGraph, + ModelSerializationFormat serializationFormat) + throws RDFServiceException { + return super.isEquivalentGraph(graphURI, + adjustForNonNegativeIntegers(serializedGraph), + serializationFormat); + } + + /** + * Convert all of the references to "nonNegativeInteger" to "integer" in + * this serialized graph. + * + * This isn't rigorous: it could fail if another property contained the text + * "nonNegativeInteger" in its name, or if that text were used as part of a + * string literal. If that happens before this TDB bug is fixed, we'll need + * to improve this method. + * + * It also isn't scalable: if we wanted real scalability, we would write to + * a temporary file as we converted. + */ + private InputStream adjustForNonNegativeIntegers(InputStream serializedGraph) + throws RDFServiceException { + try { + String raw = IOUtils.toString(serializedGraph, "UTF-8"); + String modified = raw.replace("nonNegativeInteger", "integer"); + return new ByteArrayInputStream(modified.getBytes("UTF-8")); + } catch (IOException e) { + throw new RDFServiceException(e); + } } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/logging/LoggingRDFService.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/logging/LoggingRDFService.java index 0988862c9..092d3e351 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/logging/LoggingRDFService.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/logging/LoggingRDFService.java @@ -86,7 +86,8 @@ public class LoggingRDFService implements RDFService { @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) + throws RDFServiceException { try (RDFServiceLogger l = new RDFServiceLogger(graphURI)) { return innerService.isEquivalentGraph(graphURI, serializedGraph, serializationFormat); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java index 2b3dd8520..2c6743fed 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java @@ -849,7 +849,7 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService { */ @Override public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph, - ModelSerializationFormat serializationFormat) { + ModelSerializationFormat serializationFormat) throws RDFServiceException { Model fileModel = RDFServiceUtils.parseModel(serializedGraph, serializationFormat); Model tripleStoreModel = new RDFServiceDataset(this).getNamedModel(graphURI); Model fromTripleStoreModel = ModelFactory.createDefaultModel().add(tripleStoreModel); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java new file mode 100644 index 000000000..ddca7a6c4 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java @@ -0,0 +1,83 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.virtuoso; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; + +import org.apache.commons.io.IOUtils; + +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; +import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sparql.RDFServiceSparql; + +/** + * For now, at least, it is just like an RDFServiceSparql except for: + * + * A small change in the syntax of an UPDATE request. + * + * Allow for the nonNegativeInteger bug when checking to see whether a graph has + * changed. + */ +public class RDFServiceVirtuoso extends RDFServiceSparql { + + public RDFServiceVirtuoso(String readEndpointURI, String updateEndpointURI, + String defaultWriteGraphURI) { + super(readEndpointURI, updateEndpointURI, defaultWriteGraphURI); + } + + public RDFServiceVirtuoso(String readEndpointURI, String updateEndpointURI) { + super(readEndpointURI, updateEndpointURI); + } + + public RDFServiceVirtuoso(String endpointURI) { + super(endpointURI); + } + + @Override + protected void executeUpdate(String updateString) + throws RDFServiceException { + super.executeUpdate(updateString.replace("INSERT DATA", "INSERT")); + } + + /** + * Virtuoso has a bug, which it shares with TDB: if given a literal of type + * xsd:nonNegativeInteger, it stores a literal of type xsd:integer. + * + * To determine whether this serialized graph is equivalent to what is + * already in Virtuoso, we need to do the same. + */ + @Override + public boolean isEquivalentGraph(String graphURI, + InputStream serializedGraph, + ModelSerializationFormat serializationFormat) + throws RDFServiceException { + return super.isEquivalentGraph(graphURI, + adjustForNonNegativeIntegers(serializedGraph), + serializationFormat); + } + + /** + * Convert all of the references to "nonNegativeInteger" to "integer" in + * this serialized graph. + * + * This isn't rigorous: it could fail if another property contained the text + * "nonNegativeInteger" in its name, or if that text were used as part of a + * string literal. If that happens before this Virtuoso bug is fixed, we'll + * need to improve this method. + * + * It also isn't scalable: if we wanted real scalability, we would write to + * a temporary file as we converted. + */ + private InputStream adjustForNonNegativeIntegers(InputStream serializedGraph) + throws RDFServiceException { + try { + String raw = IOUtils.toString(serializedGraph, "UTF-8"); + String modified = raw.replace("nonNegativeInteger", "integer"); + return new ByteArrayInputStream(modified.getBytes("UTF-8")); + } catch (IOException e) { + throw new RDFServiceException(e); + } + } + +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java index 1d8fbb58e..bf51f2b85 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/FileGraphSetup.java @@ -40,6 +40,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelAccess; import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelNames; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ModelSerializationFormat; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; @@ -206,7 +207,7 @@ public class FileGraphSetup implements ServletContextListener { * Otherwise, if a graph with the given name is in the DB and is isomorphic with * the graph that was read from the files system, then do nothing. */ - public boolean updateGraphInDB(RDFService rdfService, Model fileModel, String type, Path path) { + public boolean updateGraphInDB(RDFService rdfService, Model fileModel, String type, Path path) throws RDFServiceException { String graphURI = pathToURI(path,type); ByteArrayOutputStream buffer = new ByteArrayOutputStream(); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/sparql/ContentTripleSourceSPARQL.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/sparql/ContentTripleSourceSPARQL.java index 4d63510ea..1e25b12f2 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/sparql/ContentTripleSourceSPARQL.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/sparql/ContentTripleSourceSPARQL.java @@ -35,7 +35,7 @@ import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; public class ContentTripleSourceSPARQL extends ContentTripleSource { private String endpointURI; private String updateEndpointURI; // Optional - + private RDFService rdfService; private RDFServiceFactory rdfServiceFactory; private Dataset dataset; @@ -73,20 +73,22 @@ public class ContentTripleSourceSPARQL extends ContentTripleSource { @Override public void startup(Application application, ComponentStartupStatus ss) { - this.rdfServiceFactory = createRDFServiceFactory(createRDFService(ss)); + this.rdfServiceFactory = createRDFServiceFactory(createRDFService(ss, + endpointURI, updateEndpointURI)); this.rdfService = this.rdfServiceFactory.getRDFService(); this.dataset = createDataset(); this.modelMaker = createModelMaker(); } - private RDFService createRDFService(ComponentStartupStatus ss) { - if (updateEndpointURI == null) { - ss.info("Using endpoint at " + endpointURI); - return new RDFServiceSparql(endpointURI); + protected RDFService createRDFService(ComponentStartupStatus ss, + String endpoint, String updateEndpoint) { + if (updateEndpoint == null) { + ss.info("Using endpoint at " + endpoint); + return new RDFServiceSparql(endpoint); } else { - ss.info("Using read endpoint at " + endpointURI - + " and update endpoint at " + updateEndpointURI); - return new RDFServiceSparql(endpointURI, updateEndpointURI); + ss.info("Using read endpoint at " + endpoint + + " and update endpoint at " + updateEndpoint); + return new RDFServiceSparql(endpoint, updateEndpoint); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/virtuoso/ContentTripleSourceVirtuoso.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/virtuoso/ContentTripleSourceVirtuoso.java new file mode 100644 index 000000000..5e3b8e3e5 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/triplesource/impl/virtuoso/ContentTripleSourceVirtuoso.java @@ -0,0 +1,29 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.triplesource.impl.virtuoso; + +import edu.cornell.mannlib.vitro.webapp.modules.ComponentStartupStatus; +import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; +import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.virtuoso.RDFServiceVirtuoso; +import edu.cornell.mannlib.vitro.webapp.triplesource.impl.sparql.ContentTripleSourceSPARQL; + +/** + * So far, it's just like a ContentTripleSourceSPARQL but it uses an instance of + * RDFServiceVirtuoso. + */ +public class ContentTripleSourceVirtuoso extends ContentTripleSourceSPARQL { + + @Override + protected RDFService createRDFService(ComponentStartupStatus ss, + String endpoint, String updateEndpoint) { + if (updateEndpoint == null) { + ss.info("Using endpoint at " + endpoint); + return new RDFServiceVirtuoso(endpoint); + } else { + ss.info("Using read endpoint at " + endpoint + + " and update endpoint at " + updateEndpoint); + return new RDFServiceVirtuoso(endpoint, updateEndpoint); + } + } + +}