litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Allow SPRAQL endpoint to have credentials on read as well as write; update Virtuoso normalisation of types

Browse source
This commit is contained in:
grahamtriggs 2015-11-04 11:03:07 +00:00
parent 817e90716c
commit 15bc18b69a
4 changed files with 731 additions and 605 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

61
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/sparql/RDFServiceSparql.java
View file

@ -21,14 +21,22 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.AuthenticationException;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.conn.PoolingClientConnectionManager;
import org.apache.http.impl.auth.BasicScheme;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.apache.jena.riot.RDFDataMgr;
@ -95,10 +103,12 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
this.updateEndpointURI = updateEndpointURI;
httpClient = HttpClientFactory.getHttpClient();
if (RDFServiceSparql.class.getName().equals(this.getClass().getName())) {
testConnection();
}
}
private void testConnection() {
protected void testConnection() {
try {
this.sparqlSelectQuery(
"SELECT ?s WHERE { ?s a " +
@ -302,7 +312,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
try {
HttpGet meth = new HttpGet(new URIBuilder(readEndpointURI).addParameter("query", queryStr).build());
meth.addHeader("Accept", "application/sparql-results+xml");
HttpResponse response = httpClient.execute(meth);
HttpContext context = getContext(meth);
HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
try {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode > 399) {
@ -351,7 +362,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
try {
HttpGet meth = new HttpGet(new URIBuilder(readEndpointURI).addParameter("query", queryStr).build());
meth.addHeader("Accept", "application/sparql-results+xml");
HttpResponse response = httpClient.execute(meth);
HttpContext context = getContext(meth);
HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
try {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode > 399) {
@ -507,7 +519,8 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
HttpPost meth = new HttpPost(updateEndpointURI);
meth.addHeader("Content-Type", "application/x-www-form-urlencoded");
meth.setEntity(new UrlEncodedFormEntity(Arrays.asList(new BasicNameValuePair("update", updateString))));
HttpResponse response = httpClient.execute(meth);
HttpContext context = getContext(meth);
HttpResponse response = context != null ? httpClient.execute(meth, context) : httpClient.execute(meth);
try {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode > 399) {
@ -748,27 +761,18 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
}
StringBuffer queryBuff = new StringBuffer();
queryBuff.append("DELETE { \n");
if (graphURI != null) {
queryBuff.append(" GRAPH <" + graphURI + "> { \n");
queryBuff.append("WITH <" + graphURI + "> \n");
}
queryBuff.append("DELETE { \n");
List<Statement> stmts = stmtIt.toList();
sort(stmts);
addStatementPatterns(stmts, queryBuff, !WHERE_CLAUSE);
if (graphURI != null) {
queryBuff.append(" } \n");
}
queryBuff.append("} WHERE { \n");
if (graphURI != null) {
queryBuff.append(" GRAPH <" + graphURI + "> { \n");
}
stmtIt = model.listStatements();
stmts = stmtIt.toList();
sort(stmts);
addStatementPatterns(stmts, queryBuff, WHERE_CLAUSE);
if (graphURI != null) {
queryBuff.append(" } \n");
}
queryBuff.append("} \n");
if(log.isDebugEnabled()) {
@ -900,4 +904,27 @@ public class RDFServiceSparql extends RDFServiceImpl implements RDFService {
return fileModel.isIsomorphicWith(fromTripleStoreModel);
}
protected HttpContext getContext(HttpRequestBase request) {
UsernamePasswordCredentials credentials = getCredentials();
if (credentials != null) {
try {
request.addHeader(new BasicScheme().authenticate(credentials, request, null));
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(AuthScope.ANY, getCredentials());
BasicHttpContext context = new BasicHttpContext();
context.setAttribute(ClientContext.CREDS_PROVIDER, provider);
return context;
} catch (AuthenticationException e) {
log.error("Unable to set credentials");
}
}
return null;
}
protected UsernamePasswordCredentials getCredentials() {
return null;
}
}

143
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/virtuoso/RDFServiceVirtuoso.java
View file

@ -6,20 +6,26 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.httpclient.HttpMethod;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.Selector;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceDataset;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpResponse;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.protocol.ClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.protocol.BasicHttpContext;
import org.apache.http.protocol.HttpContext;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
@ -53,10 +59,10 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
private final String password;
public RDFServiceVirtuoso(String baseURI, String username, String password) {
super(figureReadEndpointUri(baseURI), figureUpdateEndpointUri(baseURI,
username));
super(figureReadEndpointUri(baseURI), figureUpdateEndpointUri(baseURI, username));
this.username = username;
this.password = password;
testConnection();
}
private static String figureReadEndpointUri(String baseUri) {
@ -81,8 +87,8 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
try {
HttpPost request = createHttpRequest(updateString);
HttpResponse response = httpClient.execute(
request, createHttpContext());
HttpContext context = getContext(request);
HttpResponse response = context != null ? httpClient.execute(request, context) : httpClient.execute(request);
try {
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode > 399) {
@ -129,18 +135,29 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
return meth;
}
/**
* We need an HttpContext that will provide username and password in
* response to a basic authentication challenge.
*/
private HttpContext createHttpContext() {
CredentialsProvider provider = new BasicCredentialsProvider();
provider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(
username, password));
protected UsernamePasswordCredentials getCredentials() {
if (username != null && password != null) {
return new UsernamePasswordCredentials(username, password);
}
BasicHttpContext context = new BasicHttpContext();
context.setAttribute(ClientContext.CREDS_PROVIDER, provider);
return context;
return null;
}
private static boolean isNumeric(String typeUri) {
return typeUri != null && (typeUri.endsWith("decimal") ||
typeUri.endsWith("int") ||
typeUri.endsWith("integer") ||
typeUri.endsWith("float") ||
typeUri.endsWith("long") ||
typeUri.endsWith("negativeInteger") ||
typeUri.endsWith("nonNegativeInteger") ||
typeUri.endsWith("nonPositiveInteger") ||
typeUri.endsWith("positiveInteger") ||
typeUri.endsWith("short") ||
typeUri.endsWith("unsignedLong") ||
typeUri.endsWith("unsignedInt") ||
typeUri.endsWith("unsignedShort") ||
typeUri.endsWith("unsignedByte"));
}
/**
@ -150,14 +167,84 @@ public class RDFServiceVirtuoso extends RDFServiceSparql {
* To determine whether this serialized graph is equivalent to what is
* already in Virtuoso, we need to do the same.
*/
public boolean isEquivalentGraph(String graphURI, InputStream serializedGraph,
ModelSerializationFormat serializationFormat) throws RDFServiceException {
Model fileModel = RDFServiceUtils.parseModel(serializedGraph, serializationFormat);
Model tripleStoreModel = new RDFServiceDataset(this).getNamedModel(graphURI);
Model fromTripleStoreModel = ModelFactory.createDefaultModel().add(tripleStoreModel);
// Compare the models
Model difference = fileModel.difference(fromTripleStoreModel);
// If there is a difference
if (difference.size() > 0) {
// First, normalize the numeric values, as Virtuoso likes to mess with the datatypes
// Iterate over the differences
StmtIterator stmtIterator = difference.listStatements();
while (stmtIterator.hasNext()) {
final Statement stmt = stmtIterator.next();
final RDFNode subject = stmt.getSubject();
final Property predicate = stmt.getPredicate();
final RDFNode object = stmt.getObject();
// If the object is a numeric literal
if (object.isLiteral() && isNumeric(object.asLiteral().getDatatypeURI())) {
// Find a matching statement in the triple store, based on normalized numeric values
StmtIterator matching = fromTripleStoreModel.listStatements(new Selector() {
@Override
public boolean isEquivalentGraph(String graphURI,
InputStream serializedGraph,
ModelSerializationFormat serializationFormat)
throws RDFServiceException {
return super.isEquivalentGraph(graphURI,
adjustForNonNegativeIntegers(serializedGraph),
serializationFormat);
public boolean test(Statement statement) {
RDFNode objectToMatch = statement.getObject();
// Both values are numeric, so compare them as parsed doubles
if (objectToMatch.isLiteral()) {
String num1 = object.asLiteral().getString();
String num2 = objectToMatch.asLiteral().getString();
return Double.parseDouble(num1) == Double.parseDouble(num2);
}
return false;
}
@Override
public boolean isSimple() {
return false;
}
@Override
public Resource getSubject() {
return subject.asResource();
}
@Override
public Property getPredicate() {
return predicate;
}
@Override
public RDFNode getObject() {
return null;
}
});
// For every matching statement
// Rewrite the object as the one in the file model (they are the same, just differ in datatype)
List<Statement> toModify = new ArrayList<Statement>();
while (matching.hasNext()) {
toModify.add(matching.next());
}
for (Statement stmtToModify : toModify) {
stmtToModify.changeObject(object);
}
}
}
// Now we've normalized the datatypes, check the graphs are isomorphic
return fileModel.isIsomorphicWith(fromTripleStoreModel);
}
return true;
}
/**

14
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/SolrSmokeTest.java
View file

@ -12,6 +12,7 @@ import java.net.UnknownHostException;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpException;
@ -24,6 +25,7 @@ import org.apache.http.impl.client.DefaultHttpClient;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
import org.apache.http.util.EntityUtils;
/**
* Spin off a thread that will try to connect to Solr.
@ -207,7 +209,7 @@ public class SolrSmokeTest implements ServletContextListener {
private static final long SLEEP_INTERVAL = 20000; // 20 seconds
private final URL solrUrl;
private final HttpClient httpClient = new DefaultHttpClient();
private final HttpClient httpClient = HttpClientFactory.getHttpClient();
private int statusCode;
@ -238,8 +240,12 @@ public class SolrSmokeTest implements ServletContextListener {
HttpGet method = new HttpGet(solrUrl.toExternalForm());
SolrSmokeTest.log.debug("Trying to connect to Solr");
HttpResponse response = httpClient.execute(method);
try {
statusCode = response.getStatusLine().getStatusCode();
SolrSmokeTest.log.debug("HTTP status was " + statusCode);
} finally {
EntityUtils.consume(response.getEntity());
}
} catch (SocketTimeoutException e) {
// Catch the exception so we can retry this.
// Save the status so we know why we failed.
@ -274,7 +280,7 @@ public class SolrSmokeTest implements ServletContextListener {
*/
private static class SolrPinger {
private final URL solrUrl;
private final HttpClient httpClient = new DefaultHttpClient();
private final HttpClient httpClient = HttpClientFactory.getHttpClient();
public SolrPinger(URL solrUrl) {
this.solrUrl = solrUrl;
@ -286,11 +292,15 @@ public class SolrSmokeTest implements ServletContextListener {
+ "/admin/ping");
SolrSmokeTest.log.debug("Trying to ping Solr");
HttpResponse response = httpClient.execute(method);
try {
SolrSmokeTest.log.debug("Finished pinging Solr");
int statusCode = response.getStatusLine().getStatusCode();
if (statusCode != HttpStatus.SC_OK) {
throw new SolrProblemException(statusCode);
}
} finally {
EntityUtils.consume(response.getEntity());
}
} catch (IOException e) {
throw new SolrProblemException(e);
}

4
webapp/src/edu/ucsf/vitro/opensocial/OpenSocialSmokeTests.java
View file

@ -18,12 +18,14 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import edu.cornell.mannlib.vitro.webapp.utils.http.HttpClientFactory;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
@ -349,7 +351,7 @@ public class OpenSocialSmokeTests implements ServletContextListener {
private final String shindigBaseUrl;
private final String shindigTestUrl;
private final DefaultHttpClient httpClient = new DefaultHttpClient();
private final HttpClient httpClient = HttpClientFactory.getHttpClient();
private int statusCode = Integer.MIN_VALUE;