NIHVIVO-2074 reconnecting SDB models for ingest tools

webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/JenaIngestController.java

@@ -8,6 +8,7 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.sql.SQLException;
 import java.text.Collator;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -27,6 +28,7 @@ import javax.servlet.ServletContextEvent;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.commons.dbcp.BasicDataSource;
 import org.apache.commons.lang.StringEscapeUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -352,7 +354,11 @@ public class JenaIngestController extends BaseEditController {
 			String jdbcUrl = vreq.getParameter("jdbcUrl");
 			String tripleStore = vreq.getParameter("tripleStore");
 			if (jdbcUrl != null) {
-				doConnectDB(vreq);
+			    try {
+			        doConnectDB(vreq);
+			    } catch (SQLException sqle) {
+			        throw new RuntimeException("Unable to connect to DB", sqle);
+			    }
 				if ("SDB".equals(tripleStore)) {
 		        	getServletContext().setAttribute("modelT", "sdb");
 		 		    getServletContext().setAttribute("info", "SDB models");
@@ -884,7 +890,7 @@ public class JenaIngestController extends BaseEditController {
         return tempModel.size();     
 	}
 	
-	public void doConnectDB(VitroRequest vreq) {
+	public void doConnectDB(VitroRequest vreq) throws SQLException {
 		String jdbcUrl = vreq.getParameter("jdbcUrl");
 		String username = vreq.getParameter("username");
 		String password = vreq.getParameter("password");
@@ -896,40 +902,48 @@ public class JenaIngestController extends BaseEditController {
 			jdbcUrl += "useUnicode=yes&characterEncoding=utf8";
 		}
 		dbTypeObj = DatabaseType.fetch(dbType);
-        loadDriver(dbTypeObj);
+        String driver = loadDriver(dbTypeObj);
 		System.out.println("Connecting to DB at "+jdbcUrl);
 		StoreDesc storeDesc = new StoreDesc(LayoutType.LayoutTripleNodesHash,dbTypeObj) ; 
-    	SDBConnection conn = new SDBConnection(jdbcUrl, username, password) ; 
+    	BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
-    	Store store = SDBFactory.connectStore(conn, storeDesc);
+    	        driver, jdbcUrl, username, password);
-    	VitroJenaSDBModelMaker vsmm = new VitroJenaSDBModelMaker(store);
+    	try {
-    	VitroJenaModelMaker vjmm = new VitroJenaModelMaker(jdbcUrl, username, password, dbType);
+    	    VitroJenaSDBModelMaker vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
-    	getServletContext().setAttribute("vitroJenaSDBModelMaker", vsmm);
+    	  	VitroJenaModelMaker vjmm = new VitroJenaModelMaker(jdbcUrl, username, password, dbType);
-    	getServletContext().setAttribute("vitroJenaModelMaker", vjmm);
+        	getServletContext().setAttribute("vitroJenaSDBModelMaker", vsmm);
-    	if("SDB".equals(tripleStore))
+        	getServletContext().setAttribute("vitroJenaModelMaker", vjmm);
-    		vreq.getSession().setAttribute("vitroJenaModelMaker",vsmm);
+        	if("SDB".equals(tripleStore))
-    	else
+        		vreq.getSession().setAttribute("vitroJenaModelMaker",vsmm);
-    		vreq.getSession().setAttribute("vitroJenaModelMaker",vjmm);
+        	else
+        		vreq.getSession().setAttribute("vitroJenaModelMaker",vjmm);
+    	} catch (SQLException sqle) {
+            throw new RuntimeException("Unable to create SDB ModelMaker", sqle);
+        }
 	}
 	
 	
-	private void loadDriver(DatabaseType dbType) {
+	private String loadDriver(DatabaseType dbType) {
-        if (DatabaseType.MySQL.equals(dbType)) {
+	    String driverName = JDBC.getDriver(dbType);
-            JDBC.loadDriverMySQL();
+	    JDBC.loadDriver(driverName);
-        } else if (DatabaseType.DB2.equals(dbType)) {
+	    return driverName;
-            JDBC.loadDriverDB2();
+	    
-        } else if (DatabaseType.Derby.equals(dbType)) {
+//        if (DatabaseType.MySQL.equals(dbType)) {
-            JDBC.loadDriverDerby();
+//            JDBC.loadDriverMySQL();
-        } else if (DatabaseType.H2.equals(dbType)) {
+//        } else if (DatabaseType.DB2.equals(dbType)) {
-            JDBC.loadDriverH2();
+//            JDBC.loadDriverDB2();
-        } else if (DatabaseType.HSQLDB.equals(dbType)) {
+//        } else if (DatabaseType.Derby.equals(dbType)) {
-            JDBC.loadDriverHSQL();
+//            JDBC.loadDriverDerby();
-        } else if (DatabaseType.Oracle.equals(dbType)) {
+//        } else if (DatabaseType.H2.equals(dbType)) {
-            JDBC.loadDriverOracle();
+//            JDBC.loadDriverH2();
-        } else if (DatabaseType.PostgreSQL.equals(dbType)) {
+//        } else if (DatabaseType.HSQLDB.equals(dbType)) {
-            JDBC.loadDriverPGSQL();
+//            JDBC.loadDriverHSQL();
-        } else if (DatabaseType.SQLServer.equals(dbType)) {
+//        } else if (DatabaseType.Oracle.equals(dbType)) {
-            JDBC.loadDriverSQLServer();
+//            JDBC.loadDriverOracle();
-        }
+//        } else if (DatabaseType.PostgreSQL.equals(dbType)) {
+//            JDBC.loadDriverPGSQL();
+//        } else if (DatabaseType.SQLServer.equals(dbType)) {
+//            JDBC.loadDriverSQLServer();
+//        }
 	}
 	
 	/*public void doExecuteCsv2Rdf(VitroRequest vreq) {

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/PropertyGroupDaoJena.java

@@ -137,12 +137,17 @@ public class PropertyGroupDaoJena extends JenaBaseDao implements PropertyGroupDa
                         getOntModelSelector().getApplicationMetadataModel(), 
                         getOntModelSelector().getFullModel()));
         
+        WebappDaoFactory wadfForURIGeneration = null;
         try {
-            groupURI = (new WebappDaoFactoryJena(unionForURIGeneration)).
+            wadfForURIGeneration = new WebappDaoFactoryJena(
-                            getIndividualDao().insertNewIndividual(groupInd);
+                    unionForURIGeneration);
+            groupURI = wadfForURIGeneration
+                    .getIndividualDao().insertNewIndividual(groupInd);
     	} catch (InsertException ie) {
     		throw new RuntimeException(InsertException.class.getName() + 
     		        "Unable to insert property group " + groupURI, ie);
+    	} finally {
+    	    wadfForURIGeneration.close();
     	}
     	
     	if (groupURI != null) {

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupDaoJena.java

@@ -31,6 +31,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.InsertException;
 import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
 import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
 import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
+import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
 import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
 
 public class VClassGroupDaoJena extends JenaBaseDao implements VClassGroupDao {
@@ -226,11 +227,16 @@ public class VClassGroupDaoJena extends JenaBaseDao implements VClassGroupDao {
     	                getOntModelSelector().getApplicationMetadataModel(), 
     	                getOntModelSelector().getFullModel()));
     	
+    	WebappDaoFactory wadfForURIGeneration = null;
     	try {
-    		groupURI = (new WebappDaoFactoryJena(unionForURIGeneration)).
+    	    wadfForURIGeneration = new WebappDaoFactoryJena(
-                            getIndividualDao().insertNewIndividual(groupInd);
+    	            unionForURIGeneration);
+    		groupURI = wadfForURIGeneration
+                    .getIndividualDao().insertNewIndividual(groupInd);
     	} catch (InsertException ie) {
     		throw new RuntimeException(InsertException.class.getName() + "Unable to insert class group "+groupURI, ie);
+    	} finally {
+    	    wadfForURIGeneration.close();
     	}
     	
     	if (groupURI != null) {

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VitroJenaSDBModelMaker.java

@@ -2,6 +2,8 @@
 
 package edu.cornell.mannlib.vitro.webapp.dao.jena;
 
+import java.sql.Connection;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -11,8 +13,15 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
+import org.apache.commons.dbcp.BasicDataSource;
+
 import com.hp.hpl.jena.graph.GraphMaker;
 import com.hp.hpl.jena.graph.Node;
+import com.hp.hpl.jena.query.Dataset;
+import com.hp.hpl.jena.query.Query;
+import com.hp.hpl.jena.query.QueryExecution;
+import com.hp.hpl.jena.query.QueryExecutionFactory;
+import com.hp.hpl.jena.query.QueryFactory;
 import com.hp.hpl.jena.rdf.model.Literal;
 import com.hp.hpl.jena.rdf.model.Model;
 import com.hp.hpl.jena.rdf.model.ModelMaker;
@@ -23,67 +32,143 @@ import com.hp.hpl.jena.rdf.model.Resource;
 import com.hp.hpl.jena.rdf.model.ResourceFactory;
 import com.hp.hpl.jena.rdf.model.Statement;
 import com.hp.hpl.jena.rdf.model.StmtIterator;
+import com.hp.hpl.jena.sdb.SDB;
 import com.hp.hpl.jena.sdb.SDBFactory;
 import com.hp.hpl.jena.sdb.Store;
+import com.hp.hpl.jena.sdb.StoreDesc;
+import com.hp.hpl.jena.sdb.sql.SDBConnection;
 import com.hp.hpl.jena.sdb.util.StoreUtils;
 import com.hp.hpl.jena.util.iterator.ExtendedIterator;
 import com.hp.hpl.jena.util.iterator.WrappedIterator;
 import com.hp.hpl.jena.vocabulary.OWL;
 import com.hp.hpl.jena.vocabulary.RDF;
+import com.hp.hpl.jena.vocabulary.RDFS;
 import com.ibm.icu.text.Collator;
 
 public class VitroJenaSDBModelMaker implements ModelMaker {
 
 	// TODO: need to rethink the inheritance/interfaces here
 	
-	private Store store = null;
+	private StoreDesc storeDesc = null;
+	private BasicDataSource bds = null;
+	private SDBConnection conn = null;
 	
 	public static final String METADATA_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
 	public static final String HAS_NAMED_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel";
 	
-	private Model metadataModel;
 	private Resource sdbResource; // a resource representing the SDB database 
 	
-	public VitroJenaSDBModelMaker(Store store) {
+	public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds) throws SQLException {
-		this.store = store;
-		try {
-			Model meta = getModel(METADATA_MODEL_URI);
-			// Test query to see if the database has been initialized
-			meta.listStatements(null, RDF.type, OWL.Nothing); 
-		} catch (Exception e) {
-			// initialize the store
-			store.getTableFormatter().create();
-        	store.getTableFormatter().truncate();
-		}
 		
-		this.metadataModel = getModel(METADATA_MODEL_URI);
+	    this.storeDesc = storeDesc;
+	    this.bds = bds;
+	    Store store = getStore();
+    	try {
+    		if (!StoreUtils.isFormatted(store)) {
+    			// initialize the store
+    			store.getTableFormatter().create();
+            	store.getTableFormatter().truncate();
+    		}
     		
-		if (metadataModel.size()==0) {
+    		Model metadataModel = getMetadataModel();
-			// set up the model name metadata to avoid expensive calls to listNames()
+    		
-			Resource sdbRes = metadataModel.createResource(); 
+    		if (metadataModel.size()==0) {
-			this.sdbResource = sdbRes;
+    			// set up the model name metadata to avoid expensive calls to listNames()
-			Iterator nameIt = SDBFactory.connectDataset(store).listNames();
+    			Resource sdbRes = metadataModel.createResource(); 
-			while (nameIt.hasNext()) {
+    			this.sdbResource = sdbRes;
-				String name = (String) nameIt.next();
+    			Iterator nameIt = SDBFactory.connectDataset(store).listNames();
-				metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name);
+    			while (nameIt.hasNext()) {
-			}
+    				String name = (String) nameIt.next();
-		} else {
+    				metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name);
-			StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null);
+    			}
-			if (stmtIt.hasNext()) {
+    		} else {
-				Statement stmt = stmtIt.nextStatement();
+    			StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null);
-				sdbResource = stmt.getSubject();
+    			if (stmtIt.hasNext()) {
-			}
+    				Statement stmt = stmtIt.nextStatement();
-			stmtIt.close();
+    				sdbResource = stmt.getSubject();
-		}
+    			}
+    			stmtIt.close();
+    		}
+	    } finally {
+	        store.close();
+	    }
+	}
+	
+	private static final int MAX_TRIES = 10;
+	
+	private Store getStore() {
+	    Store store = null;
+	    boolean goodStore = false;
+	    boolean badConn = false;
+	    int tries = 0;
+	    while (!goodStore && tries < MAX_TRIES) {
+	        tries++;
+	        if (conn == null || badConn) {
+	            try {
+                    conn = new SDBConnection(bds.getConnection());
+                    badConn = false;
+                } catch (SQLException sqle) {
+                    throw new RuntimeException(
+                            "Unable to get SQL connection", sqle);
+                }
+	        }
+	        store = SDBFactory.connectStore(conn, storeDesc);
+	        if (!isWorking(store)) {
+	            if (conn != null) {
+	                conn.close();
+	                badConn = true;
+	            }
+	            
+	        } else {
+	            goodStore = true;
+	        }
+	    }
+	    if (store == null) {
+	        throw new RuntimeException(
+	                "Unable to connect to SDB store after " + 
+	                MAX_TRIES + " attempts");
+	    }
+	    return store;
+	}
+	
+	Model getMetadataModel() {
+	    return getModel(METADATA_MODEL_URI);
+	}
+	
+	private boolean isWorking(Store store) {
+	    Dataset d = SDBFactory.connectDataset(store);
+	    try {
+	        String validationQuery = "ASK { <" + RDFS.Resource.getURI() + "> " +
+	                                 "   <" + RDFS.isDefinedBy.getURI() + "> " +
+	                                 "   <" + RDFS.Resource.getURI() + "> }";
+	        Query q = QueryFactory.create(validationQuery);
+	        QueryExecution qe = QueryExecutionFactory.create(q, d);
+	        try {
+	            qe.execAsk();
+	            return true;
+	        } catch (Exception e) {
+	            e.printStackTrace();
+	            return false;
+	        } finally {
+	            qe.close();
+	        }
+	    } finally {
+	        d.close();
+	    }
 	}
 	
 	public void close() {
-		store.close();
+		getStore().close();
 	}
 
 	public Model createModel(String arg0) {
-		Model model = SDBFactory.connectNamedModel(store, arg0);
+		Model model = SDBFactory.connectNamedModel(getStore(), arg0);
-		metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
+		Model metadataModel = getMetadataModel();
+		try {
+		    metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
+		} finally {
+		    metadataModel.close();
+		}
 		return model;
 	}
 
@@ -97,22 +182,30 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
 	}
 
 	public boolean hasModel(String arg0) {
-		StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
+	    Model metadataModel = getMetadataModel();
-		try {
+	    try {
-			return stmtIt.hasNext();
+    		StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
-		} finally {
+    		try {
-			if (stmtIt != null) {
+    			return stmtIt.hasNext();
-				stmtIt.close();
+    		} finally {
-			}
+    			if (stmtIt != null) {
-		}
+    				stmtIt.close();
+    			}
+    		}
+	    } finally {
+	        metadataModel.close();
+	    }
 	}
 
 	public ExtendedIterator listModels() {
 		ArrayList<String> metaNameList = new ArrayList<String>();
 		ArrayList<String> storeNameList = new ArrayList<String>();
 		ArrayList<String> unionNameList = new ArrayList<String>();
+		
+		Model metadataModel = getMetadataModel();
+		
 		Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(metadataModel.getProperty(HAS_NAMED_MODEL_URI));
-		Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(store);
+		Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore());
 		Node node = null;
 		RDFNode rdfNode = null;
 		
@@ -230,17 +323,26 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
 	    		
 	    }while(metaString!=null || storeString!=null);
 		
+	    if (metadataModel != null) {
+	        metadataModel.close();
+	    }
+	    
 		return WrappedIterator.create(unionNameList.iterator());
 	}
 
 	public Model openModel(String arg0, boolean arg1) {
-		return SDBFactory.connectNamedModel(store,arg0);
+		return SDBFactory.connectNamedModel(getStore(),arg0);
 	}
 
 	public void removeModel(String arg0) {
 		Model m = getModel(arg0);
 		m.removeAll(null,null,null);
-		metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
+		Model metadataModel = getMetadataModel();
+		try {
+		    metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
+		} finally {
+		    metadataModel.close();
+		}
 	}
 
 	public Model addDescription(Model arg0, Resource arg1) {
@@ -260,11 +362,11 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
 	}
 
 	public Model openModel() {
-		return SDBFactory.connectDefaultModel(store);
+		return SDBFactory.connectDefaultModel(getStore());
 	}
 
 	public Model createDefaultModel() {
-		return SDBFactory.connectDefaultModel(store);
+		return SDBFactory.connectDefaultModel(getStore());
 	}
 
 	public Model createFreshModel() {
@@ -275,26 +377,26 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
 	 * @deprecated
 	 */
 	public Model createModel() {
-		return SDBFactory.connectDefaultModel(store);
+		return SDBFactory.connectDefaultModel(getStore());
 	}
 
 	/**
 	 * @deprecated
 	 */
 	public Model getModel() {
-		return SDBFactory.connectDefaultModel(store);
+		return SDBFactory.connectDefaultModel(getStore());
 	}
 
 	public Model openModel(String arg0) {
-		return SDBFactory.connectDefaultModel(store);
+		return SDBFactory.connectDefaultModel(getStore());
 	}
 
 	public Model openModelIfPresent(String arg0) {
-		return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(store,arg0) : null;
+		return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(getStore(),arg0) : null;
 	}
 
 	public Model getModel(String arg0) {
-		return SDBFactory.connectNamedModel(store, arg0);
+		return SDBFactory.connectNamedModel(getStore(), arg0);
 	}
 
 	public Model getModel(String arg0, ModelReader arg1) {

webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/WebappDaoFactorySDB.java

@@ -174,8 +174,6 @@ public class WebappDaoFactorySDB extends WebappDaoFactoryJena {
 	    
 	    private BasicDataSource _bds;
 	    private StoreDesc _storeDesc;
-	    private Dataset _dataset;
-	    private Connection _conn;
 	    
 	    public ReconnectingDatasetFactory(BasicDataSource bds, StoreDesc storeDesc) {
 	        _bds = bds;
@@ -184,15 +182,11 @@ public class WebappDaoFactorySDB extends WebappDaoFactoryJena {
 	    
 	    public DatasetWrapper getDatasetWrapper() {
 	        try {
-	            if ((_dataset != null) && (_conn != null) && (!_conn.isClosed())) {
+                Connection sqlConn = _bds.getConnection();
-	                return new DatasetWrapper(_dataset);
+                SDBConnection conn = new SDBConnection(sqlConn) ;
-	            } else {
+                Store store = SDBFactory.connectStore(conn, _storeDesc);
-	                _conn = _bds.getConnection();
+                Dataset dataset = SDBFactory.connectDataset(store);
-                    SDBConnection conn = new SDBConnection(_conn) ;
+                return new DatasetWrapper(dataset, conn);
-                    Store store = SDBFactory.connectStore(conn, _storeDesc);
-                    _dataset = SDBFactory.connectDataset(store);
-                    return new DatasetWrapper(_dataset);
-	            }
             } catch (SQLException sqe) {
                 throw new RuntimeException("Unable to connect to database", sqe);
             }

webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java

@@ -3,6 +3,7 @@
 package edu.cornell.mannlib.vitro.webapp.servlet.setup;
 
 import java.io.InputStream;
+import java.sql.SQLException;
 import java.util.Set;
 
 import javax.servlet.ServletContext;
@@ -400,10 +401,13 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
     	else if(TripleStoreType.SDB.equals(type)){
     		StoreDesc storeDesc = new StoreDesc(
     		        LayoutType.LayoutTripleNodesHash, DatabaseType.MySQL);
-    		SDBConnection sdbConn = new SDBConnection(
+    	    BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
-    		        jdbcUrl, username, password);
+    	            DB_DRIVER_CLASS_NAME, jdbcUrl, username, password);
-    		Store store = SDBFactory.connectStore(sdbConn, storeDesc);
+    	    try {
-    		vsmm = new VitroJenaSDBModelMaker(store);
+    	        vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
+    	    } catch (SQLException sqle) {
+    	        log.error("Unable to set up SDB ModelMaker", sqle);
+    	    }
     	}
     	
 		return;