litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3951 performance improvements for SPARQL with OPTIONAL on query page and NIHVIVO-3953 support for older SPARQL and ARQ syntaxes

Browse source
This commit is contained in:
brianjlowe 2012-09-06 18:44:45 +00:00
parent 5d950625c2
commit 4fde9b8166
12 changed files with 220 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SparqlQueryServlet.java
View file

@ -127,7 +127,7 @@ public class SparqlQueryServlet extends BaseEditController {
}
executeQuery(response, resultFormatParam, rdfResultFormatParam,
queryParam, vreq.getRDFService());
queryParam, vreq.getUnfilteredRDFService());
return;
}

15
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java
View file

@ -54,10 +54,25 @@ public class VitroRequest extends HttpServletRequestWrapper {
}
}
public RDFService getUnfilteredRDFService() {
Object o = getAttribute("unfilteredRDFService");
if (o instanceof RDFService) {
return (RDFService) o;
} else {
RDFService rdfService = RDFServiceUtils.getRDFService(this);
setAttribute("unfilteredRDFService", rdfService);
return rdfService;
}
}
public void setRDFService(RDFService rdfService) {
setAttribute("rdfService", rdfService);
}
public void setUnfilteredRDFService(RDFService rdfService) {
setAttribute("unfilteredRDFService", rdfService);
}
public void setWebappDaoFactory( WebappDaoFactory wdf){
setAttribute("webappDaoFactory",wdf);
}

11
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/VitroRequestPrep.java
View file

@ -174,10 +174,13 @@ public class VitroRequestPrep implements Filter {
}
ServletContext ctx = vreq.getSession().getServletContext();
vreq.setUnfilteredWebappDaoFactory(new WebappDaoFactorySDB(
RDFServiceUtils.getRDFServiceFactory(ctx).getRDFService(),
ModelContext.getUnionOntModelSelector(
ctx)));
if (vreq.getUnfilteredWebappDaoFactory() == null) {
vreq.setUnfilteredWebappDaoFactory(new WebappDaoFactorySDB(
RDFServiceUtils.getRDFServiceFactory(ctx).getRDFService(),
ModelContext.getUnionOntModelSelector(
ctx)));
}
req.setAttribute("VitroRequestPrep.setup", new Integer(1));
chain.doFilter(req, response);

12
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java
View file

@ -105,12 +105,17 @@ public class WebappDaoFactorySDBPrep implements Filter {
config.setPreferredLanguages(langs);
RDFServiceFactory factory = RDFServiceUtils.getRDFServiceFactory(_ctx);
RDFService rdfService = factory.getRDFService();
//RDFService rdfService = factory.getRDFService();
RDFService unfilteredRDFService = factory.getShortTermRDFService();
RDFService rdfService = null;
if (!"false".equals(
ConfigurationProperties.getBean(vreq).getProperty(
"RDFService.languageFilter", "true"))) {
rdfService = new LanguageFilteringRDFService(rdfService, langs);
rdfService = new LanguageFilteringRDFService(unfilteredRDFService, langs);
} else {
rdfService = unfilteredRDFService;
}
Dataset dataset = new RDFServiceDataset(rdfService);
@ -118,9 +123,12 @@ public class WebappDaoFactorySDBPrep implements Filter {
WebappDaoFactory assertions = new WebappDaoFactorySDB(
rdfService, baseOms, config, SDBDatasetMode.ASSERTIONS_ONLY);
vreq.setRDFService(rdfService);
vreq.setUnfilteredRDFService(unfilteredRDFService);
vreq.setWebappDaoFactory(wadf);
vreq.setAssertionsWebappDaoFactory(assertions);
vreq.setFullWebappDaoFactory(wadf);
vreq.setUnfilteredWebappDaoFactory(new WebappDaoFactorySDB(
rdfService, ModelContext.getUnionOntModelSelector(_ctx)));
vreq.setDataset(dataset);
vreq.setOntModelSelector(baseOms);

17
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java
View file

@ -9,6 +9,23 @@ public interface RDFServiceFactory {
*/
public RDFService getRDFService();
/**
* Returns an instance of RDFService that may not support being left idle
* for long periods of time. RDFService instances returned by this method
* should be immediately used and closed, not stored in (for example) session
* or context attributes.
*
* This method exists to enable performance improvements resulting from a
* lack of need to handle database connection or other service timeouts and
* reconnects.
*
* The results provided by RDFService instances returned by this method must
* be identical to those provided by instances returned by getRDFService().
*
* @return RDFService - an RDFService instance
*/
public RDFService getShortTermRDFService();
/**
* Registers a listener to listen to changes in any graph in
* the RDF store. Any RDFService objects returned by this factory

5
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java
View file

@ -59,6 +59,11 @@ public class SameAsFilteringRDFServiceFactory implements RDFServiceFactory {
return new SameAsFilteringRDFService(f.getRDFService());
}
@Override
public RDFService getShortTermRDFService() {
return new SameAsFilteringRDFService(f.getShortTermRDFService());
}
@Override
public void registerListener(ChangeListener changeListener) throws RDFServiceException {
f.registerListener(changeListener);

5
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java
View file

@ -29,6 +29,11 @@ public class RDFServiceFactorySingle implements RDFServiceFactory {
return this.rdfService;
}
@Override
public RDFService getShortTermRDFService() {
return this.rdfService;
}
@Override
public void registerListener(ChangeListener listener) throws RDFServiceException {
this.rdfService.registerListener(listener);

4
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java
View file

@ -91,6 +91,10 @@ public abstract class RDFServiceImpl implements RDFService {
registeredListeners.remove(changeListener);
}
public synchronized List<ChangeListener> getRegisteredListeners() {
return this.registeredListeners;
}
@Override
public ChangeSet manufactureChangeSet() {
return new ChangeSetImpl();

38
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java
View file

@ -6,6 +6,7 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@ -20,9 +21,11 @@ import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QueryParseException;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.ResultSetFormatter;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.RDFNode;
@ -239,8 +242,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
DatasetWrapper dw = getDatasetWrapper();
try {
Dataset d = dw.getDataset();
Query q = QueryFactory.create(query);
QueryExecution qe = QueryExecutionFactory.create(q, d);
Query q = createQuery(query);
QueryExecution qe = createQueryExecution(query, q, d);
ByteArrayOutputStream serializedModel = new ByteArrayOutputStream();
try {
// TODO pipe this
@ -278,8 +281,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
DatasetWrapper dw = getDatasetWrapper();
try {
Dataset d = dw.getDataset();
Query q = QueryFactory.create(query);
QueryExecution qe = QueryExecutionFactory.create(q, d);
Query q = createQuery(query);
QueryExecution qe = createQueryExecution(query, q, d);
try {
ResultSet resultSet = qe.execSelect();
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@ -314,8 +317,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
DatasetWrapper dw = getDatasetWrapper();
try {
Dataset d = dw.getDataset();
Query q = QueryFactory.create(query);
QueryExecution qe = QueryExecutionFactory.create(q, d);
Query q = createQuery(query);
QueryExecution qe = createQueryExecution(query, q, d);
try {
return qe.execAsk();
} finally {
@ -356,4 +359,27 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
super.notifyListeners(triple, operation, graphURI);
}
protected Query createQuery(String queryString) {
List<Syntax> syntaxes = Arrays.asList(
Syntax.defaultQuerySyntax, Syntax.syntaxSPARQL_11,
Syntax.syntaxSPARQL_10, Syntax.syntaxSPARQL, Syntax.syntaxARQ);
Query q = null;
Iterator<Syntax> syntaxIt = syntaxes.iterator();
while (q == null) {
Syntax syntax = syntaxIt.next();
try {
q = QueryFactory.create(queryString, syntax);
} catch (QueryParseException e) {
if (!syntaxIt.hasNext()) {
throw(e);
}
}
}
return q;
}
protected QueryExecution createQueryExecution(String queryString, Query q, Dataset d) {
return QueryExecutionFactory.create(q, d);
}
}

63
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceFactorySDB.java Normal file
View file

@ -0,0 +1,63 @@
package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb;
import java.sql.SQLException;
import javax.sql.DataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.sdb.StoreDesc;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
public class RDFServiceFactorySDB implements RDFServiceFactory {
private final static Log log = LogFactory.getLog(RDFServiceFactorySDB.class);
private DataSource ds;
private StoreDesc storeDesc;
private RDFService longTermRDFService;
public RDFServiceFactorySDB(DataSource dataSource, StoreDesc storeDesc) {
this.ds = dataSource;
this.storeDesc = storeDesc;
this.longTermRDFService = new RDFServiceSDB(dataSource, storeDesc);
}
@Override
public RDFService getRDFService() {
return this.longTermRDFService;
}
@Override
public RDFService getShortTermRDFService() {
try {
RDFService rdfService = new RDFServiceSDB(ds.getConnection(), storeDesc);
for (ChangeListener cl : ((RDFServiceSDB) longTermRDFService)
.getRegisteredListeners() ) {
rdfService.registerListener(cl);
}
return rdfService;
} catch (Exception e) {
log.error(e,e);
throw new RuntimeException(e);
}
}
@Override
public void registerListener(ChangeListener changeListener)
throws RDFServiceException {
this.longTermRDFService.registerListener(changeListener);
}
@Override
public void unregisterListener(ChangeListener changeListener)
throws RDFServiceException {
this.longTermRDFService.registerListener(changeListener);
}
}

58
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java
View file

@ -3,17 +3,22 @@
package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb;
import java.io.ByteArrayInputStream;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Iterator;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.sql.DataSource;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.sdb.SDBFactory;
@ -23,6 +28,7 @@ import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.dao.jena.DatasetWrapper;
import edu.cornell.mannlib.vitro.webapp.dao.jena.StaticDatasetFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
@ -36,15 +42,27 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
private DataSource ds;
private StoreDesc storeDesc;
private Connection conn;
private StaticDatasetFactory staticDatasetFactory;
public RDFServiceSDB(DataSource dataSource, StoreDesc storeDesc) {
this.ds = dataSource;
this.storeDesc = storeDesc;
}
public RDFServiceSDB(Connection conn, StoreDesc storeDesc) {
this.conn = conn;
this.storeDesc = storeDesc;
this.staticDatasetFactory = new StaticDatasetFactory(getDataset(
new SDBConnection(conn)));
}
@Override
protected DatasetWrapper getDatasetWrapper() {
try {
if (staticDatasetFactory != null) {
return staticDatasetFactory.getDatasetWrapper();
}
SDBConnection conn = new SDBConnection(ds.getConnection());
return new DatasetWrapper(getDataset(conn), conn);
} catch (SQLException sqle) {
@ -66,7 +84,7 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
SDBConnection conn = null;
try {
conn = new SDBConnection(ds.getConnection());
conn = new SDBConnection(getConnection());
} catch (SQLException sqle) {
log.error(sqle, sqle);
throw new RDFServiceException(sqle);
@ -135,9 +153,45 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
return true;
}
protected Connection getConnection() throws SQLException {
return (conn != null) ? conn : ds.getConnection();
}
protected Dataset getDataset(SDBConnection conn) {
Store store = SDBFactory.connectStore(conn, storeDesc);
store.getLoader().setUseThreading(false);
return SDBFactory.connectDataset(store);
}
private static final Pattern OPTIONAL_PATTERN = Pattern.compile("optional", Pattern.CASE_INSENSITIVE);
private static final Pattern GRAPH_PATTERN = Pattern.compile("graph", Pattern.CASE_INSENSITIVE);
@Override
protected QueryExecution createQueryExecution(String queryString, Query q, Dataset d) {
// query performance with OPTIONAL can be dramatically improved on SDB by
// using the default model (union model) instead of the dataset, so long as
// we're not querying particular named graphs
Matcher optional = OPTIONAL_PATTERN.matcher(queryString);
Matcher graph = GRAPH_PATTERN.matcher(queryString);
if (optional.find() && !graph.find()) {
return QueryExecutionFactory.create(q, d.getDefaultModel());
} else {
return QueryExecutionFactory.create(q, d);
}
}
@Override
public void close() {
if (conn != null) {
try {
conn.close();
} catch (SQLException e) {
log.error(e,e);
}
}
}
}

9
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java
View file

@ -7,7 +7,6 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.sql.DataSource;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -25,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb.RDFServiceSDB;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb.RDFServiceFactorySDB;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sparql.RDFServiceSparql;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
@ -105,8 +104,10 @@ implements javax.servlet.ServletContextListener {
setupSDB(ctx, store);
}
RDFService rdfService = new RDFServiceSDB(ds, storeDesc);
RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySingle(rdfService);
//RDFService rdfService = new RDFServiceSDB(ds, storeDesc);
//RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySingle(rdfService);
RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySDB(ds, storeDesc);
RDFServiceUtils.setRDFServiceFactory(ctx, rdfServiceFactory);
log.info("SDB store ready for use");