NIHVIVO-3951 performance improvements for SPARQL with OPTIONAL on query page and NIHVIVO-3953 support for older SPARQL and ARQ syntaxes

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SparqlQueryServlet.java

@@ -127,7 +127,7 @@ public class SparqlQueryServlet extends BaseEditController {
         }
 
         executeQuery(response, resultFormatParam, rdfResultFormatParam, 
-                queryParam, vreq.getRDFService()); 
+                queryParam, vreq.getUnfilteredRDFService()); 
         return;
     }
     

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/VitroRequest.java

@@ -54,10 +54,25 @@ public class VitroRequest extends HttpServletRequestWrapper {
         }
     }
     
+    public RDFService getUnfilteredRDFService() {
+        Object o = getAttribute("unfilteredRDFService");
+        if (o instanceof RDFService) {
+            return (RDFService) o;
+        } else {
+            RDFService rdfService = RDFServiceUtils.getRDFService(this);
+            setAttribute("unfilteredRDFService", rdfService);
+            return rdfService;
+        }
+    }
+    
     public void setRDFService(RDFService rdfService) {
         setAttribute("rdfService", rdfService);
     }
     
+    public void setUnfilteredRDFService(RDFService rdfService) {
+        setAttribute("unfilteredRDFService", rdfService);
+    }
+    
     public void setWebappDaoFactory( WebappDaoFactory wdf){
         setAttribute("webappDaoFactory",wdf);
     }

webapp/src/edu/cornell/mannlib/vitro/webapp/filters/VitroRequestPrep.java

@@ -174,10 +174,13 @@ public class VitroRequestPrep implements Filter {
         }
         
         ServletContext ctx = vreq.getSession().getServletContext();
+        
+        if (vreq.getUnfilteredWebappDaoFactory() == null) {
             vreq.setUnfilteredWebappDaoFactory(new WebappDaoFactorySDB(
                     RDFServiceUtils.getRDFServiceFactory(ctx).getRDFService(),
                     ModelContext.getUnionOntModelSelector(
                             ctx)));
+        }
         
         req.setAttribute("VitroRequestPrep.setup", new Integer(1));
         chain.doFilter(req, response);

webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java

@@ -105,12 +105,17 @@ public class WebappDaoFactorySDBPrep implements Filter {
         config.setPreferredLanguages(langs);
 		
 		RDFServiceFactory factory = RDFServiceUtils.getRDFServiceFactory(_ctx);
-		RDFService rdfService = factory.getRDFService();
+		
+		//RDFService rdfService = factory.getRDFService();
+		RDFService unfilteredRDFService = factory.getShortTermRDFService();
+		RDFService rdfService = null;
 		
 		if (!"false".equals(
 		        ConfigurationProperties.getBean(vreq).getProperty(
 		                "RDFService.languageFilter", "true"))) {
-		    rdfService = new LanguageFilteringRDFService(rdfService, langs);
+		    rdfService = new LanguageFilteringRDFService(unfilteredRDFService, langs);
+		} else {
+		    rdfService = unfilteredRDFService;
 		}
 		
 		Dataset dataset = new RDFServiceDataset(rdfService);
@@ -118,9 +123,12 @@ public class WebappDaoFactorySDBPrep implements Filter {
 	    WebappDaoFactory assertions = new WebappDaoFactorySDB(
 	            rdfService, baseOms, config, SDBDatasetMode.ASSERTIONS_ONLY);
 	    vreq.setRDFService(rdfService);
+	    vreq.setUnfilteredRDFService(unfilteredRDFService);
 		vreq.setWebappDaoFactory(wadf);
 		vreq.setAssertionsWebappDaoFactory(assertions);
 		vreq.setFullWebappDaoFactory(wadf);
+        vreq.setUnfilteredWebappDaoFactory(new WebappDaoFactorySDB(
+                rdfService, ModelContext.getUnionOntModelSelector(_ctx)));
 		vreq.setDataset(dataset);
 		vreq.setOntModelSelector(baseOms);
 		

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFServiceFactory.java

@@ -9,6 +9,23 @@ public interface RDFServiceFactory {
      */
     public RDFService getRDFService();
     
+    /**
+     * Returns an instance of RDFService that may not support being left idle
+     * for long periods of time.  RDFService instances returned by this method
+     * should be immediately used and closed, not stored in (for example) session
+     * or context attributes.
+     * 
+     * This method exists to enable performance improvements resulting from a
+     * lack of need to handle database connection or other service timeouts and
+     * reconnects.
+     * 
+     * The results provided by RDFService instances returned by this method must 
+     * be identical to those provided by instances returned by getRDFService().  
+     *   
+     * @return RDFService - an RDFService instance
+     */
+    public RDFService getShortTermRDFService();
+    
     /**
      * Registers a listener to listen to changes in any graph in
      * the RDF store.  Any RDFService objects returned by this factory

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/SameAsFilteringRDFServiceFactory.java

@@ -59,6 +59,11 @@ public class SameAsFilteringRDFServiceFactory implements RDFServiceFactory {
         return new SameAsFilteringRDFService(f.getRDFService());
     }
     
+    @Override
+    public RDFService getShortTermRDFService() {
+        return new SameAsFilteringRDFService(f.getShortTermRDFService());
+    }
+    
     @Override
     public void registerListener(ChangeListener changeListener) throws RDFServiceException {
         f.registerListener(changeListener);

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java

@@ -29,6 +29,11 @@ public class RDFServiceFactorySingle implements RDFServiceFactory {
         return this.rdfService;
     }
     
+    @Override 
+    public RDFService getShortTermRDFService() {
+        return this.rdfService;
+    }
+    
     @Override
     public void registerListener(ChangeListener listener) throws RDFServiceException {
         this.rdfService.registerListener(listener);

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java

@@ -91,6 +91,10 @@ public abstract class RDFServiceImpl implements RDFService {
 		registeredListeners.remove(changeListener);
 	}
 
+	public synchronized List<ChangeListener> getRegisteredListeners() {
+	    return this.registeredListeners;
+	}
+	
 	@Override
 	public ChangeSet manufactureChangeSet() {
 		return new ChangeSetImpl();

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java

@@ -6,6 +6,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 
@@ -20,9 +21,11 @@ import com.hp.hpl.jena.query.Query;
 import com.hp.hpl.jena.query.QueryExecution;
 import com.hp.hpl.jena.query.QueryExecutionFactory;
 import com.hp.hpl.jena.query.QueryFactory;
+import com.hp.hpl.jena.query.QueryParseException;
 import com.hp.hpl.jena.query.QuerySolution;
 import com.hp.hpl.jena.query.ResultSet;
 import com.hp.hpl.jena.query.ResultSetFormatter;
+import com.hp.hpl.jena.query.Syntax;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
 import com.hp.hpl.jena.rdf.model.RDFNode;
@@ -239,8 +242,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         DatasetWrapper dw = getDatasetWrapper();
         try {
             Dataset d = dw.getDataset();
-            Query q = QueryFactory.create(query);
+            Query q = createQuery(query);
-            QueryExecution qe = QueryExecutionFactory.create(q, d);
+            QueryExecution qe = createQueryExecution(query, q, d);
             ByteArrayOutputStream serializedModel = new ByteArrayOutputStream();
             try {
                 // TODO pipe this
@@ -278,8 +281,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         DatasetWrapper dw = getDatasetWrapper();
         try {
             Dataset d = dw.getDataset();
-            Query q = QueryFactory.create(query);
+            Query q = createQuery(query);
-            QueryExecution qe = QueryExecutionFactory.create(q, d);
+            QueryExecution qe = createQueryExecution(query, q, d);
             try {
                 ResultSet resultSet = qe.execSelect();
                 ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); 
@@ -314,8 +317,8 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         DatasetWrapper dw = getDatasetWrapper();
         try {
             Dataset d = dw.getDataset();
-            Query q = QueryFactory.create(query);
+            Query q = createQuery(query);
-            QueryExecution qe = QueryExecutionFactory.create(q, d);
+            QueryExecution qe = createQueryExecution(query, q, d);
             try {
                 return qe.execAsk();
             } finally {
@@ -356,4 +359,27 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
         super.notifyListeners(triple, operation, graphURI);
     }
     
+    protected Query createQuery(String queryString) {
+        List<Syntax> syntaxes = Arrays.asList(
+                Syntax.defaultQuerySyntax, Syntax.syntaxSPARQL_11, 
+                Syntax.syntaxSPARQL_10, Syntax.syntaxSPARQL, Syntax.syntaxARQ);
+        Query q = null;
+        Iterator<Syntax> syntaxIt = syntaxes.iterator(); 
+        while (q == null) {
+            Syntax syntax = syntaxIt.next();
+            try {
+               q = QueryFactory.create(queryString, syntax);  
+            } catch (QueryParseException e) {
+               if (!syntaxIt.hasNext()) {
+                   throw(e);
+               }
+            }
+        }
+        return q;
+    }
+    
+    protected QueryExecution createQueryExecution(String queryString, Query q, Dataset d) {
+        return QueryExecutionFactory.create(q, d);
+    }
+    
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceFactorySDB.java

@@ -0,0 +1,63 @@
+package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb;
+
+import java.sql.SQLException;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import com.hp.hpl.jena.sdb.StoreDesc;
+
+import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
+
+public class RDFServiceFactorySDB implements RDFServiceFactory {
+
+    private final static Log log = LogFactory.getLog(RDFServiceFactorySDB.class);
+    
+    private DataSource ds;
+    private StoreDesc storeDesc;
+    private RDFService longTermRDFService;
+    
+    public RDFServiceFactorySDB(DataSource dataSource, StoreDesc storeDesc) {
+        this.ds = dataSource;
+        this.storeDesc = storeDesc;
+        this.longTermRDFService = new RDFServiceSDB(dataSource, storeDesc);
+    }
+    
+    @Override
+    public RDFService getRDFService() {
+        return this.longTermRDFService;
+    }
+
+    @Override
+    public RDFService getShortTermRDFService() {
+        try {
+            RDFService rdfService = new RDFServiceSDB(ds.getConnection(), storeDesc);
+            for (ChangeListener cl : ((RDFServiceSDB) longTermRDFService)
+                    .getRegisteredListeners() ) {
+                rdfService.registerListener(cl);    
+            }
+            return rdfService;
+        } catch (Exception e) {
+            log.error(e,e);
+            throw new RuntimeException(e);
+        }
+    } 
+
+    @Override
+    public void registerListener(ChangeListener changeListener)
+            throws RDFServiceException {
+        this.longTermRDFService.registerListener(changeListener);
+    }
+
+    @Override
+    public void unregisterListener(ChangeListener changeListener)
+            throws RDFServiceException {
+        this.longTermRDFService.registerListener(changeListener);
+    }
+
+}

webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java

@@ -3,17 +3,22 @@
 package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb;
 
 import java.io.ByteArrayInputStream;
+import java.sql.Connection;
 import java.sql.SQLException;
 import java.util.Iterator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import javax.sql.DataSource;
 
-import org.apache.commons.dbcp.BasicDataSource;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import com.hp.hpl.jena.query.Dataset;
+import com.hp.hpl.jena.query.Query;
+import com.hp.hpl.jena.query.QueryExecution;
+import com.hp.hpl.jena.query.QueryExecutionFactory;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelFactory;
 import com.hp.hpl.jena.sdb.SDBFactory;
@@ -23,6 +28,7 @@ import com.hp.hpl.jena.sdb.sql.SDBConnection;
 import com.hp.hpl.jena.shared.Lock;
 
 import edu.cornell.mannlib.vitro.webapp.dao.jena.DatasetWrapper;
+import edu.cornell.mannlib.vitro.webapp.dao.jena.StaticDatasetFactory;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.ModelChange;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
@@ -36,15 +42,27 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
     
     private DataSource ds;
     private StoreDesc storeDesc;
+    private Connection conn;
+    private StaticDatasetFactory staticDatasetFactory;
     
     public RDFServiceSDB(DataSource dataSource, StoreDesc storeDesc) {
         this.ds = dataSource;
         this.storeDesc = storeDesc;
     }
     
+    public RDFServiceSDB(Connection conn, StoreDesc storeDesc) {
+        this.conn = conn;
+        this.storeDesc = storeDesc;
+        this.staticDatasetFactory = new StaticDatasetFactory(getDataset(
+                new SDBConnection(conn)));
+    }
+    
     @Override
     protected DatasetWrapper getDatasetWrapper() {
         try {
+            if (staticDatasetFactory != null) {
+                return staticDatasetFactory.getDatasetWrapper();
+            }
             SDBConnection conn = new SDBConnection(ds.getConnection());
             return new DatasetWrapper(getDataset(conn), conn);
         } catch (SQLException sqle) {
@@ -66,7 +84,7 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
             
         SDBConnection conn = null;
         try {
-            conn = new SDBConnection(ds.getConnection());
+            conn = new SDBConnection(getConnection());
         } catch (SQLException sqle) {
             log.error(sqle, sqle);
             throw new RDFServiceException(sqle);
@@ -135,9 +153,45 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
         return true;
     }  
     
+    protected Connection getConnection() throws SQLException {
+        return (conn != null) ? conn : ds.getConnection();
+    }
+    
     protected Dataset getDataset(SDBConnection conn) {
         Store store = SDBFactory.connectStore(conn, storeDesc);
         store.getLoader().setUseThreading(false);
         return SDBFactory.connectDataset(store);
     }
+    
+    private static final Pattern OPTIONAL_PATTERN = Pattern.compile("optional", Pattern.CASE_INSENSITIVE);
+    
+    private static final Pattern GRAPH_PATTERN = Pattern.compile("graph", Pattern.CASE_INSENSITIVE);
+    
+    @Override
+    protected QueryExecution createQueryExecution(String queryString, Query q, Dataset d) {
+        // query performance with OPTIONAL can be dramatically improved on SDB by 
+        // using the default model (union model) instead of the dataset, so long as 
+        // we're not querying particular named graphs
+                
+        Matcher optional = OPTIONAL_PATTERN.matcher(queryString);
+        Matcher graph = GRAPH_PATTERN.matcher(queryString);
+        
+        if (optional.find() && !graph.find()) { 
+            return QueryExecutionFactory.create(q, d.getDefaultModel());
+        } else {
+            return QueryExecutionFactory.create(q, d); 
+        }       
+    }
+    
+    @Override 
+    public void close() {
+        if (conn != null) {
+            try {
+                conn.close();
+            } catch (SQLException e) {
+                log.error(e,e);
+            }
+        }
+    }
+    
 }

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFServiceSetup.java

@@ -7,7 +7,6 @@ import javax.servlet.ServletContext;
 import javax.servlet.ServletContextEvent;
 import javax.sql.DataSource;
 
-import org.apache.commons.dbcp.BasicDataSource;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -25,7 +24,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
-import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb.RDFServiceSDB;
+import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb.RDFServiceFactorySDB;
 import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.sparql.RDFServiceSparql;
 import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
 
@@ -105,8 +104,10 @@ implements javax.servlet.ServletContextListener {
             setupSDB(ctx, store);
         }
         
-        RDFService rdfService = new RDFServiceSDB(ds, storeDesc);
+        //RDFService rdfService = new RDFServiceSDB(ds, storeDesc);
-        RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySingle(rdfService);
+        //RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySingle(rdfService);
+        
+        RDFServiceFactory rdfServiceFactory = new RDFServiceFactorySDB(ds, storeDesc);
         RDFServiceUtils.setRDFServiceFactory(ctx, rdfServiceFactory);
         
         log.info("SDB store ready for use");