Allow SPRAQL endpoint to have credentials on read as well as write; update Virtuoso normalisation of types

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java

@@ -21,14 +21,22 @@ import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpResponse;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.AuthenticationException;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.DefaultHttpClient;
+import org.apache.http.impl.auth.BasicScheme;
-import org.apache.http.impl.conn.PoolingClientConnectionManager;
+import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.protocol.BasicHttpContext;
+import org.apache.http.protocol.HttpContext;
 import org.apache.http.util.EntityUtils;
 import org.apache.jena.riot.RDFDataMgr;
 
@@ -95,10 +103,12 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 		this.updateEndpointURI = updateEndpointURI;
 		httpClient = HttpClientFactory.getHttpClient();
 
+		if (RDFServiceSparql.class.getName().equals(this.getClass().getName())) {
 			testConnection();
 		}
+	}
 
-    private void testConnection() {
+	protected void testConnection() {
 		try {
 			this.sparqlSelectQuery(
 					"SELECT ?s WHERE { ?s a " +
@@ -302,7 +312,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 		try {
 			HttpGet meth = new HttpGet(new URIBuilder(readEndpointURI).addParameter("query", queryStr).build());
 			meth.addHeader("Accept", "application/sparql-results+xml");
-			HttpResponse response = httpClient.execute(meth);
+			HttpContext context = getContext(meth);
+			HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
 			try {
 				int statusCode = response.getStatusLine().getStatusCode();
 				if (statusCode > 399) {
@@ -351,7 +362,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 		try {
 			HttpGet meth = new HttpGet(new URIBuilder(readEndpointURI).addParameter("query", queryStr).build());
 			meth.addHeader("Accept", "application/sparql-results+xml");
-			HttpResponse response = httpClient.execute(meth);
+			HttpContext context = getContext(meth);
+			HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
 			try {
 				int statusCode = response.getStatusLine().getStatusCode();
 				if (statusCode > 399) {
@@ -507,7 +519,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 			HttpPost meth = new HttpPost(updateEndpointURI);
 			meth.addHeader("Content-Type", "application/x-www-form-urlencoded");
 			meth.setEntity(new UrlEncodedFormEntity(Arrays.asList(new BasicNameValuePair("update", updateString))));
-			HttpResponse response = httpClient.execute(meth);
+			HttpContext context = getContext(meth);
+			HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
 			try {
 				int statusCode = response.getStatusLine().getStatusCode();
 				if (statusCode > 399) {
@@ -748,27 +761,18 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 		}
 
 		StringBuffer queryBuff = new StringBuffer();
-        queryBuff.append("DELETE { \n");
 		if (graphURI != null) {
-            queryBuff.append("    GRAPH <" + graphURI + "> { \n");
+			queryBuff.append("WITH <" + graphURI + "> \n");
 		}
+		queryBuff.append("DELETE { \n");
 		List<Statement> stmts = stmtIt.toList();
 		sort(stmts);
 		addStatementPatterns(stmts, queryBuff, !WHERE_CLAUSE);
-        if (graphURI != null) {
-            queryBuff.append("    } \n");
-        }
 		queryBuff.append("} WHERE { \n");
-        if (graphURI != null) {
-            queryBuff.append("    GRAPH <" + graphURI + "> { \n");
-        }
 		stmtIt = model.listStatements();
 		stmts = stmtIt.toList();
 		sort(stmts);
 		addStatementPatterns(stmts, queryBuff, WHERE_CLAUSE);
-        if (graphURI != null) {
-            queryBuff.append("    } \n");
-        }
 		queryBuff.append("} \n");
 
 		if(log.isDebugEnabled()) {
@@ -900,4 +904,27 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
 		return fileModel.isIsomorphicWith(fromTripleStoreModel);
 	}
 
+	protected HttpContext getContext(HttpRequestBase request) {
+		UsernamePasswordCredentials credentials = getCredentials();
+		if (credentials != null) {
+			try {
+				request.addHeader(new BasicScheme().authenticate(credentials, request, null));
+
+				CredentialsProvider provider = new BasicCredentialsProvider();
+				provider.setCredentials(AuthScope.ANY, getCredentials());
+
+				BasicHttpContext context = new BasicHttpContext();
+				context.setAttribute(ClientContext.CREDS_PROVIDER, provider);
+				return context;
+			} catch (AuthenticationException e) {
+				log.error("Unable to set credentials");
+			}
+		}
+
+		return null;
+	}
+
+	protected UsernamePasswordCredentials getCredentials() {
+		return null;
+	}
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java

@@ -6,20 +6,26 @@ import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
 
-import org.apache.commons.httpclient.HttpMethod;
+import com.hp.hpl.jena.rdf.model.Model;
+import com.hp.hpl.jena.rdf.model.ModelFactory;
+import com.hp.hpl.jena.rdf.model.Property;
+import com.hp.hpl.jena.rdf.model.RDFNode;
+import com.hp.hpl.jena.rdf.model.Resource;
+import com.hp.hpl.jena.rdf.model.Selector;
+import com.hp.hpl.jena.rdf.model.Statement;
+import com.hp.hpl.jena.rdf.model.StmtIterator;
+import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpResponse;
-import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
-import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.protocol.ClientContext;
 import org.apache.http.entity.StringEntity;
-import org.apache.http.impl.client.BasicCredentialsProvider;
-import org.apache.http.protocol.BasicHttpContext;
 import org.apache.http.protocol.HttpContext;
 
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
@@ -53,10 +59,10 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
 	private final String password;
 
 	public RDFServiceVirtuoso(String baseURI, String username, String password) {
-		super(figureReadEndpointUri(baseURI), figureUpdateEndpointUri(baseURI,
+		super(figureReadEndpointUri(baseURI), figureUpdateEndpointUri(baseURI, username));
-				username));
 		this.username = username;
 		this.password = password;
+		testConnection();
 	}
 
 	private static String figureReadEndpointUri(String baseUri) {
@@ -81,8 +87,8 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
 
 		try {
 			HttpPost request = createHttpRequest(updateString);
-			HttpResponse response = httpClient.execute(
+			HttpContext context = getContext(request);
-					request, createHttpContext());
+			HttpResponse response = context != null ? httpClient.execute(request, context) : httpClient.execute(request);
 			try {
 				int statusCode = response.getStatusLine().getStatusCode();
 				if (statusCode > 399) {
@@ -129,18 +135,29 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
 		return meth;
 	}
 
-	/**
+	protected UsernamePasswordCredentials getCredentials() {
-	 * We need an HttpContext that will provide username and password in
+		if (username != null && password != null) {
-	 * response to a basic authentication challenge.
+			return new UsernamePasswordCredentials(username, password);
-	 */
+		}
-	private HttpContext createHttpContext() {
-		CredentialsProvider provider = new BasicCredentialsProvider();
-		provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(
-				username, password));
 
-		BasicHttpContext context = new BasicHttpContext();
+		return null;
-		context.setAttribute(ClientContext.CREDS_PROVIDER, provider);
+	}
-		return context;
+
+	private static boolean isNumeric(String typeUri) {
+		return typeUri != null && (typeUri.endsWith("decimal") ||
+				typeUri.endsWith("int") ||
+				typeUri.endsWith("integer") ||
+				typeUri.endsWith("float") ||
+				typeUri.endsWith("long") ||
+				typeUri.endsWith("negativeInteger") ||
+				typeUri.endsWith("nonNegativeInteger") ||
+				typeUri.endsWith("nonPositiveInteger") ||
+				typeUri.endsWith("positiveInteger") ||
+				typeUri.endsWith("short") ||
+				typeUri.endsWith("unsignedLong") ||
+				typeUri.endsWith("unsignedInt") ||
+				typeUri.endsWith("unsignedShort") ||
+				typeUri.endsWith("unsignedByte"));
 	}
 
 	/**
@@ -150,14 +167,84 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
 	 * To determine whether this serialized graph is equivalent to what is
 	 * already in Virtuoso, we need to do the same.
 	 */
+	public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph,
+									 ModelSerializationFormat serializationFormat) throws RDFServiceException {
+		Model fileModel = RDFServiceUtils.parseModel(serializedGraph, serializationFormat);
+		Model tripleStoreModel = new RDFServiceDataset(this).getNamedModel(graphURI);
+		Model fromTripleStoreModel = ModelFactory.createDefaultModel().add(tripleStoreModel);
+
+		// Compare the models
+		Model difference = fileModel.difference(fromTripleStoreModel);
+
+		// If there is a difference
+		if (difference.size() > 0) {
+			// First, normalize the numeric values, as Virtuoso likes to mess with the datatypes
+			// Iterate over the differences
+			StmtIterator stmtIterator = difference.listStatements();
+			while (stmtIterator.hasNext()) {
+				final Statement stmt = stmtIterator.next();
+				final RDFNode subject = stmt.getSubject();
+				final Property predicate = stmt.getPredicate();
+				final RDFNode object = stmt.getObject();
+
+				// If the object is a numeric literal
+				if (object.isLiteral() &&  isNumeric(object.asLiteral().getDatatypeURI())) {
+					// Find a matching statement in the triple store, based on normalized numeric values
+					StmtIterator matching = fromTripleStoreModel.listStatements(new Selector() {
 						@Override
-	public boolean isEquivalentGraph(String graphURI,
+						public boolean test(Statement statement) {
-			InputStream serializedGraph,
+							RDFNode objectToMatch = statement.getObject();
-			ModelSerializationFormat serializationFormat)
+
-			throws RDFServiceException {
+							// Both values are numeric, so compare them as parsed doubles
-		return super.isEquivalentGraph(graphURI,
+							if (objectToMatch.isLiteral()) {
-				adjustForNonNegativeIntegers(serializedGraph),
+								String num1 = object.asLiteral().getString();
-				serializationFormat);
+								String num2 = objectToMatch.asLiteral().getString();
+
+								return Double.parseDouble(num1) == Double.parseDouble(num2);
+							}
+
+							return false;
+						}
+
+						@Override
+						public boolean isSimple() {
+							return false;
+						}
+
+						@Override
+						public Resource getSubject() {
+							return subject.asResource();
+						}
+
+						@Override
+						public Property getPredicate() {
+							return predicate;
+						}
+
+						@Override
+						public RDFNode getObject() {
+							return null;
+						}
+					});
+
+					// For every matching statement
+					// Rewrite the object as the one in the file model (they are the same, just differ in datatype)
+					List<Statement> toModify = new ArrayList<Statement>();
+					while (matching.hasNext()) {
+						toModify.add(matching.next());
+					}
+
+					for (Statement stmtToModify : toModify) {
+						stmtToModify.changeObject(object);
+					}
+				}
+			}
+
+			// Now we've normalized the datatypes, check the graphs are isomorphic
+			return fileModel.isIsomorphicWith(fromTripleStoreModel);
+		}
+
+		return true;
 	}
 
 	/**

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java

@@ -12,6 +12,7 @@ import java.net.UnknownHostException;
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
 
+import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpException;
@@ -24,6 +25,7 @@ import org.apache.http.impl.client.DefaultHttpClient;
 import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
 import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
+import org.apache.http.util.EntityUtils;
 
 /**
  * Spin off a thread that will try to connect to Solr.
@@ -207,7 +209,7 @@ public class SolrSmokeTest implements ServletContextListener {
 		private static final long SLEEP_INTERVAL = 20000; // 20 seconds
 
 		private final URL solrUrl;
-		private final HttpClient httpClient = new DefaultHttpClient();
+		private final HttpClient httpClient = HttpClientFactory.getHttpClient();
 
 		private int statusCode;
 
@@ -238,8 +240,12 @@ public class SolrSmokeTest implements ServletContextListener {
 				HttpGet method = new HttpGet(solrUrl.toExternalForm());
 				SolrSmokeTest.log.debug("Trying to connect to Solr");
 				HttpResponse response = httpClient.execute(method);
+				try {
 					statusCode = response.getStatusLine().getStatusCode();
 					SolrSmokeTest.log.debug("HTTP status was " + statusCode);
+				} finally {
+					EntityUtils.consume(response.getEntity());
+				}
 			} catch (SocketTimeoutException e) {
 				// Catch the exception so we can retry this.
 				// Save the status so we know why we failed.
@@ -274,7 +280,7 @@ public class SolrSmokeTest implements ServletContextListener {
 	 */
 	private static class SolrPinger {
 		private final URL solrUrl;
-		private final HttpClient httpClient = new DefaultHttpClient();
+		private final HttpClient httpClient = HttpClientFactory.getHttpClient();
 
 		public SolrPinger(URL solrUrl) {
 			this.solrUrl = solrUrl;
@@ -286,11 +292,15 @@ public class SolrSmokeTest implements ServletContextListener {
 						+ "/admin/ping");
 				SolrSmokeTest.log.debug("Trying to ping Solr");
 				HttpResponse response = httpClient.execute(method);
+				try {
 					SolrSmokeTest.log.debug("Finished pinging Solr");
 					int statusCode = response.getStatusLine().getStatusCode();
 					if (statusCode != HttpStatus.SC_OK) {
 						throw new SolrProblemException(statusCode);
 					}
+				} finally {
+					EntityUtils.consume(response.getEntity());
+				}
 			} catch (IOException e) {
 				throw new SolrProblemException(e);
 			}

webapp/src/edu/ucsf/vitro/opensocial/OpenSocialSmokeTests.java

@@ -18,12 +18,14 @@ import javax.servlet.ServletContext;
 import javax.servlet.ServletContextEvent;
 import javax.servlet.ServletContextListener;
 
+import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory;
 import org.apache.commons.dbcp.BasicDataSource;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 
@@ -349,7 +351,7 @@ public class OpenSocialSmokeTests implements ServletContextListener {
 
 		private final String shindigBaseUrl;
 		private final String shindigTestUrl;
-		private final DefaultHttpClient httpClient = new DefaultHttpClient();
+		private final HttpClient httpClient = HttpClientFactory.getHttpClient();
 
 		private int statusCode = Integer.MIN_VALUE;