litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-2074 reconnecting SDB models for ingest tools

Browse source
This commit is contained in:
bjl23 2011-02-08 20:49:11 +00:00
parent 1dbd15c042
commit b3c13118d9
6 changed files with 229 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/JenaIngestController.java
View file

@ -8,6 +8,7 @@ import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.sql.SQLException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collections;
@ -27,6 +28,7 @@ import javax.servlet.ServletContextEvent;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@ -352,7 +354,11 @@ public class JenaIngestController extends BaseEditController {
String jdbcUrl = vreq.getParameter("jdbcUrl");
String tripleStore = vreq.getParameter("tripleStore");
if (jdbcUrl != null) {
doConnectDB(vreq);
try {
doConnectDB(vreq);
} catch (SQLException sqle) {
throw new RuntimeException("Unable to connect to DB", sqle);
}
if ("SDB".equals(tripleStore)) {
getServletContext().setAttribute("modelT", "sdb");
getServletContext().setAttribute("info", "SDB models");
@ -884,7 +890,7 @@ public class JenaIngestController extends BaseEditController {
return tempModel.size();
}
public void doConnectDB(VitroRequest vreq) {
public void doConnectDB(VitroRequest vreq) throws SQLException {
String jdbcUrl = vreq.getParameter("jdbcUrl");
String username = vreq.getParameter("username");
String password = vreq.getParameter("password");
@ -896,40 +902,48 @@ public class JenaIngestController extends BaseEditController {
jdbcUrl += "useUnicode=yes&characterEncoding=utf8";
}
dbTypeObj = DatabaseType.fetch(dbType);
loadDriver(dbTypeObj);
String driver = loadDriver(dbTypeObj);
System.out.println("Connecting to DB at "+jdbcUrl);
StoreDesc storeDesc = new StoreDesc(LayoutType.LayoutTripleNodesHash,dbTypeObj) ;
SDBConnection conn = new SDBConnection(jdbcUrl, username, password) ;
Store store = SDBFactory.connectStore(conn, storeDesc);
VitroJenaSDBModelMaker vsmm = new VitroJenaSDBModelMaker(store);
VitroJenaModelMaker vjmm = new VitroJenaModelMaker(jdbcUrl, username, password, dbType);
getServletContext().setAttribute("vitroJenaSDBModelMaker", vsmm);
getServletContext().setAttribute("vitroJenaModelMaker", vjmm);
if("SDB".equals(tripleStore))
vreq.getSession().setAttribute("vitroJenaModelMaker",vsmm);
else
vreq.getSession().setAttribute("vitroJenaModelMaker",vjmm);
StoreDesc storeDesc = new StoreDesc(LayoutType.LayoutTripleNodesHash,dbTypeObj) ;
BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
driver, jdbcUrl, username, password);
try {
VitroJenaSDBModelMaker vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
VitroJenaModelMaker vjmm = new VitroJenaModelMaker(jdbcUrl, username, password, dbType);
getServletContext().setAttribute("vitroJenaSDBModelMaker", vsmm);
getServletContext().setAttribute("vitroJenaModelMaker", vjmm);
if("SDB".equals(tripleStore))
vreq.getSession().setAttribute("vitroJenaModelMaker",vsmm);
else
vreq.getSession().setAttribute("vitroJenaModelMaker",vjmm);
} catch (SQLException sqle) {
throw new RuntimeException("Unable to create SDB ModelMaker", sqle);
}
}
private void loadDriver(DatabaseType dbType) {
if (DatabaseType.MySQL.equals(dbType)) {
JDBC.loadDriverMySQL();
} else if (DatabaseType.DB2.equals(dbType)) {
JDBC.loadDriverDB2();
} else if (DatabaseType.Derby.equals(dbType)) {
JDBC.loadDriverDerby();
} else if (DatabaseType.H2.equals(dbType)) {
JDBC.loadDriverH2();
} else if (DatabaseType.HSQLDB.equals(dbType)) {
JDBC.loadDriverHSQL();
} else if (DatabaseType.Oracle.equals(dbType)) {
JDBC.loadDriverOracle();
} else if (DatabaseType.PostgreSQL.equals(dbType)) {
JDBC.loadDriverPGSQL();
} else if (DatabaseType.SQLServer.equals(dbType)) {
JDBC.loadDriverSQLServer();
}
private String loadDriver(DatabaseType dbType) {
String driverName = JDBC.getDriver(dbType);
JDBC.loadDriver(driverName);
return driverName;
// if (DatabaseType.MySQL.equals(dbType)) {
// JDBC.loadDriverMySQL();
// } else if (DatabaseType.DB2.equals(dbType)) {
// JDBC.loadDriverDB2();
// } else if (DatabaseType.Derby.equals(dbType)) {
// JDBC.loadDriverDerby();
// } else if (DatabaseType.H2.equals(dbType)) {
// JDBC.loadDriverH2();
// } else if (DatabaseType.HSQLDB.equals(dbType)) {
// JDBC.loadDriverHSQL();
// } else if (DatabaseType.Oracle.equals(dbType)) {
// JDBC.loadDriverOracle();
// } else if (DatabaseType.PostgreSQL.equals(dbType)) {
// JDBC.loadDriverPGSQL();
// } else if (DatabaseType.SQLServer.equals(dbType)) {
// JDBC.loadDriverSQLServer();
// }
}
/*public void doExecuteCsv2Rdf(VitroRequest vreq) {

9
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/PropertyGroupDaoJena.java
View file

@ -137,12 +137,17 @@ public class PropertyGroupDaoJena extends JenaBaseDao implements PropertyGroupDa
getOntModelSelector().getApplicationMetadataModel(),
getOntModelSelector().getFullModel()));
WebappDaoFactory wadfForURIGeneration = null;
try {
groupURI = (new WebappDaoFactoryJena(unionForURIGeneration)).
getIndividualDao().insertNewIndividual(groupInd);
wadfForURIGeneration = new WebappDaoFactoryJena(
unionForURIGeneration);
groupURI = wadfForURIGeneration
.getIndividualDao().insertNewIndividual(groupInd);
} catch (InsertException ie) {
throw new RuntimeException(InsertException.class.getName() +
"Unable to insert property group " + groupURI, ie);
} finally {
wadfForURIGeneration.close();
}
if (groupURI != null) {

10
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupDaoJena.java
View file

@ -31,6 +31,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.InsertException;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
public class VClassGroupDaoJena extends JenaBaseDao implements VClassGroupDao {
@ -226,11 +227,16 @@ public class VClassGroupDaoJena extends JenaBaseDao implements VClassGroupDao {
getOntModelSelector().getApplicationMetadataModel(),
getOntModelSelector().getFullModel()));
WebappDaoFactory wadfForURIGeneration = null;
try {
groupURI = (new WebappDaoFactoryJena(unionForURIGeneration)).
getIndividualDao().insertNewIndividual(groupInd);
wadfForURIGeneration = new WebappDaoFactoryJena(
unionForURIGeneration);
groupURI = wadfForURIGeneration
.getIndividualDao().insertNewIndividual(groupInd);
} catch (InsertException ie) {
throw new RuntimeException(InsertException.class.getName() + "Unable to insert class group "+groupURI, ie);
} finally {
wadfForURIGeneration.close();
}
if (groupURI != null) {

208
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VitroJenaSDBModelMaker.java
View file

@ -2,6 +2,8 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@ -11,8 +13,15 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.commons.dbcp.BasicDataSource;
import com.hp.hpl.jena.graph.GraphMaker;
import com.hp.hpl.jena.graph.Node;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelMaker;
@ -23,67 +32,143 @@ import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.sdb.SDB;
import com.hp.hpl.jena.sdb.SDBFactory;
import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc;
import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.sdb.util.StoreUtils;
import com.hp.hpl.jena.util.iterator.ExtendedIterator;
import com.hp.hpl.jena.util.iterator.WrappedIterator;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import com.ibm.icu.text.Collator;
public class VitroJenaSDBModelMaker implements ModelMaker {
// TODO: need to rethink the inheritance/interfaces here
private Store store = null;
private StoreDesc storeDesc = null;
private BasicDataSource bds = null;
private SDBConnection conn = null;
public static final String METADATA_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
public static final String HAS_NAMED_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel";
private Model metadataModel;
private Resource sdbResource; // a resource representing the SDB database
public VitroJenaSDBModelMaker(Store store) {
this.store = store;
try {
Model meta = getModel(METADATA_MODEL_URI);
// Test query to see if the database has been initialized
meta.listStatements(null, RDF.type, OWL.Nothing);
} catch (Exception e) {
// initialize the store
store.getTableFormatter().create();
store.getTableFormatter().truncate();
}
public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds) throws SQLException {
this.metadataModel = getModel(METADATA_MODEL_URI);
if (metadataModel.size()==0) {
// set up the model name metadata to avoid expensive calls to listNames()
Resource sdbRes = metadataModel.createResource();
this.sdbResource = sdbRes;
Iterator nameIt = SDBFactory.connectDataset(store).listNames();
while (nameIt.hasNext()) {
String name = (String) nameIt.next();
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name);
}
} else {
StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null);
if (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
sdbResource = stmt.getSubject();
}
stmtIt.close();
}
this.storeDesc = storeDesc;
this.bds = bds;
Store store = getStore();
try {
if (!StoreUtils.isFormatted(store)) {
// initialize the store
store.getTableFormatter().create();
store.getTableFormatter().truncate();
}
Model metadataModel = getMetadataModel();
if (metadataModel.size()==0) {
// set up the model name metadata to avoid expensive calls to listNames()
Resource sdbRes = metadataModel.createResource();
this.sdbResource = sdbRes;
Iterator nameIt = SDBFactory.connectDataset(store).listNames();
while (nameIt.hasNext()) {
String name = (String) nameIt.next();
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name);
}
} else {
StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null);
if (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
sdbResource = stmt.getSubject();
}
stmtIt.close();
}
} finally {
store.close();
}
}
private static final int MAX_TRIES = 10;
private Store getStore() {
Store store = null;
boolean goodStore = false;
boolean badConn = false;
int tries = 0;
while (!goodStore && tries < MAX_TRIES) {
tries++;
if (conn == null || badConn) {
try {
conn = new SDBConnection(bds.getConnection());
badConn = false;
} catch (SQLException sqle) {
throw new RuntimeException(
"Unable to get SQL connection", sqle);
}
}
store = SDBFactory.connectStore(conn, storeDesc);
if (!isWorking(store)) {
if (conn != null) {
conn.close();
badConn = true;
}
} else {
goodStore = true;
}
}
if (store == null) {
throw new RuntimeException(
"Unable to connect to SDB store after " +
MAX_TRIES + " attempts");
}
return store;
}
Model getMetadataModel() {
return getModel(METADATA_MODEL_URI);
}
private boolean isWorking(Store store) {
Dataset d = SDBFactory.connectDataset(store);
try {
String validationQuery = "ASK { <" + RDFS.Resource.getURI() + "> " +
" <" + RDFS.isDefinedBy.getURI() + "> " +
" <" + RDFS.Resource.getURI() + "> }";
Query q = QueryFactory.create(validationQuery);
QueryExecution qe = QueryExecutionFactory.create(q, d);
try {
qe.execAsk();
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
} finally {
qe.close();
}
} finally {
d.close();
}
}
public void close() {
store.close();
getStore().close();
}
public Model createModel(String arg0) {
Model model = SDBFactory.connectNamedModel(store, arg0);
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
Model model = SDBFactory.connectNamedModel(getStore(), arg0);
Model metadataModel = getMetadataModel();
try {
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
} finally {
metadataModel.close();
}
return model;
}
@ -97,22 +182,30 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
public boolean hasModel(String arg0) {
StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
try {
return stmtIt.hasNext();
} finally {
if (stmtIt != null) {
stmtIt.close();
}
}
Model metadataModel = getMetadataModel();
try {
StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
try {
return stmtIt.hasNext();
} finally {
if (stmtIt != null) {
stmtIt.close();
}
}
} finally {
metadataModel.close();
}
}
public ExtendedIterator listModels() {
ArrayList<String> metaNameList = new ArrayList<String>();
ArrayList<String> storeNameList = new ArrayList<String>();
ArrayList<String> unionNameList = new ArrayList<String>();
Model metadataModel = getMetadataModel();
Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(metadataModel.getProperty(HAS_NAMED_MODEL_URI));
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(store);
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore());
Node node = null;
RDFNode rdfNode = null;
@ -230,17 +323,26 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}while(metaString!=null || storeString!=null);
if (metadataModel != null) {
metadataModel.close();
}
return WrappedIterator.create(unionNameList.iterator());
}
public Model openModel(String arg0, boolean arg1) {
return SDBFactory.connectNamedModel(store,arg0);
return SDBFactory.connectNamedModel(getStore(),arg0);
}
public void removeModel(String arg0) {
Model m = getModel(arg0);
m.removeAll(null,null,null);
metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
Model metadataModel = getMetadataModel();
try {
metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
} finally {
metadataModel.close();
}
}
public Model addDescription(Model arg0, Resource arg1) {
@ -260,11 +362,11 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
public Model openModel() {
return SDBFactory.connectDefaultModel(store);
return SDBFactory.connectDefaultModel(getStore());
}
public Model createDefaultModel() {
return SDBFactory.connectDefaultModel(store);
return SDBFactory.connectDefaultModel(getStore());
}
public Model createFreshModel() {
@ -275,26 +377,26 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
* @deprecated
*/
public Model createModel() {
return SDBFactory.connectDefaultModel(store);
return SDBFactory.connectDefaultModel(getStore());
}
/**
* @deprecated
*/
public Model getModel() {
return SDBFactory.connectDefaultModel(store);
return SDBFactory.connectDefaultModel(getStore());
}
public Model openModel(String arg0) {
return SDBFactory.connectDefaultModel(store);
return SDBFactory.connectDefaultModel(getStore());
}
public Model openModelIfPresent(String arg0) {
return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(store,arg0) : null;
return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(getStore(),arg0) : null;
}
public Model getModel(String arg0) {
return SDBFactory.connectNamedModel(store, arg0);
return SDBFactory.connectNamedModel(getStore(), arg0);
}
public Model getModel(String arg0, ModelReader arg1) {

16
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/WebappDaoFactorySDB.java
View file

@ -174,8 +174,6 @@ public class WebappDaoFactorySDB extends WebappDaoFactoryJena {
private BasicDataSource _bds;
private StoreDesc _storeDesc;
private Dataset _dataset;
private Connection _conn;
public ReconnectingDatasetFactory(BasicDataSource bds, StoreDesc storeDesc) {
_bds = bds;
@ -184,15 +182,11 @@ public class WebappDaoFactorySDB extends WebappDaoFactoryJena {
public DatasetWrapper getDatasetWrapper() {
try {
if ((_dataset != null) && (_conn != null) && (!_conn.isClosed())) {
return new DatasetWrapper(_dataset);
} else {
_conn = _bds.getConnection();
SDBConnection conn = new SDBConnection(_conn) ;
Store store = SDBFactory.connectStore(conn, _storeDesc);
_dataset = SDBFactory.connectDataset(store);
return new DatasetWrapper(_dataset);
}
Connection sqlConn = _bds.getConnection();
SDBConnection conn = new SDBConnection(sqlConn) ;
Store store = SDBFactory.connectStore(conn, _storeDesc);
Dataset dataset = SDBFactory.connectDataset(store);
return new DatasetWrapper(dataset, conn);
} catch (SQLException sqe) {
throw new RuntimeException("Unable to connect to database", sqe);
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -3,6 +3,7 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.InputStream;
import java.sql.SQLException;
import java.util.Set;
import javax.servlet.ServletContext;
@ -400,10 +401,13 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
else if(TripleStoreType.SDB.equals(type)){
StoreDesc storeDesc = new StoreDesc(
LayoutType.LayoutTripleNodesHash, DatabaseType.MySQL);
SDBConnection sdbConn = new SDBConnection(
jdbcUrl, username, password);
Store store = SDBFactory.connectStore(sdbConn, storeDesc);
vsmm = new VitroJenaSDBModelMaker(store);
BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
DB_DRIVER_CLASS_NAME, jdbcUrl, username, password);
try {
vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
} catch (SQLException sqle) {
log.error("Unable to set up SDB ModelMaker", sqle);
}
}
return;