NIHVIVO-3440 rework of db connection management for VitroJenaSDBModelMaker

This commit is contained in:
brianjlowe 2011-12-07 18:48:47 +00:00
parent 989f0d9f97
commit ae85e5c648
6 changed files with 471 additions and 486 deletions

View file

@ -0,0 +1,46 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class SDBGraphConnectionGenerator {
private final static Log log = LogFactory.getLog(
SDBGraphConnectionGenerator.class);
private BasicDataSource ds = null;
private Connection connection = null;
public SDBGraphConnectionGenerator(BasicDataSource dataSource) {
this.ds = dataSource;
}
public Connection generateConnection() throws SQLException {
if ( this.connection == null ) {
this.connection = ds.getConnection();
} else if ( this.connection.isClosed() ) {
try {
this.connection.close();
} catch (SQLException e) {
// The connection will throw an "Already closed"
// SQLException that we need to catch. We need to
// make this extra call to .close() in order to make
// sure that the connection is returned to the pool.
// This depends on the particular behavior of version
// 1.4 of the Apache Commons connection pool library.
// Earlier versions threw the exception right away,
// making this impossible. Future versions may do the
// same.
}
this.connection = ds.getConnection();
}
return connection;
}
}

View file

@ -17,14 +17,21 @@ public class SDBGraphGenerator implements SQLGraphGenerator {
private static final Log log = LogFactory.getLog(SDBGraphGenerator.class.getName());
private BasicDataSource ds;
private SDBGraphConnectionGenerator connGen;
private Connection connection;
private StoreDesc storeDesc;
private String graphID;
public SDBGraphGenerator(BasicDataSource dataSource, StoreDesc storeDesc,
String graphID) {
this.ds = dataSource;
this.connGen = new SDBGraphConnectionGenerator(dataSource);
this.storeDesc = storeDesc;
this.graphID = graphID;
}
public SDBGraphGenerator(SDBGraphConnectionGenerator connectionGenerator,
StoreDesc storeDesc, String graphID) {
this.connGen = connectionGenerator;
this.storeDesc = storeDesc;
this.graphID = graphID;
}
@ -39,24 +46,7 @@ public class SDBGraphGenerator implements SQLGraphGenerator {
public Graph generateGraph() {
try {
if ( this.connection == null ) {
this.connection = ds.getConnection();
} else if ( this.connection.isClosed() ) {
try {
this.connection.close();
} catch (SQLException e) {
// The connection will throw an "Already closed"
// SQLException that we need to catch. We need to
// make this extra call to .close() in order to make
// sure that the connection is returned to the pool.
// This depends on the particular behavior of version
// 1.4 of the Apache Commons connection pool library.
// Earlier versions threw the exception right away,
// making this impossible. Future versions may do the
// same.
}
this.connection = ds.getConnection();
}
this.connection = connGen.generateConnection();
Store store = SDBFactory.connectStore(connection, storeDesc);
return SDBFactory.connectNamedGraph(store, graphID);
} catch (SQLException e) {

View file

@ -2,18 +2,18 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.sql.Connection;
import java.sql.SQLException;
import java.text.Collator;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.graph.Graph;
import com.hp.hpl.jena.graph.GraphMaker;
@ -28,13 +28,10 @@ import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ModelMaker;
import com.hp.hpl.jena.rdf.model.ModelReader;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.sdb.SDB;
import com.hp.hpl.jena.sdb.SDBFactory;
import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc;
@ -42,50 +39,35 @@ import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.sdb.util.StoreUtils;
import com.hp.hpl.jena.util.iterator.ExtendedIterator;
import com.hp.hpl.jena.util.iterator.WrappedIterator;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import com.ibm.icu.text.Collator;
public class VitroJenaSDBModelMaker implements ModelMaker {
// TODO: need to rethink the inheritance/interfaces here
private final static Log log = LogFactory.getLog(VitroJenaSDBModelMaker.class);
private StoreDesc storeDesc = null;
private BasicDataSource bds = null;
private SDBConnection conn = null;
private SDBGraphConnectionGenerator connGen = null;
public static final String METADATA_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
public static final String HAS_NAMED_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel";
public static final String METADATA_MODEL_URI =
"http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
public static final String HAS_NAMED_MODEL_URI =
"http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel";
private Resource sdbResource; // a resource representing the SDB database
public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds) throws SQLException {
public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds)
throws SQLException {
this.storeDesc = storeDesc;
this.bds = bds;
connGen = new SDBGraphConnectionGenerator(bds);
Store store = getStore();
try {
Model metadataModel = getMetadataModel();
if (metadataModel.size()==0) {
// set up the model name metadata to avoid expensive calls to listNames()
Resource sdbRes = metadataModel.createResource();
this.sdbResource = sdbRes;
Iterator nameIt = SDBFactory.connectDataset(store).listNames();
while (nameIt.hasNext()) {
String name = (String) nameIt.next();
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name);
}
} else {
StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null);
if (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
sdbResource = stmt.getSubject();
}
stmtIt.close();
}
setUpMetadata(store);
} finally {
store.close();
}
@ -101,7 +83,7 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
tries++;
if (conn == null) {
try {
conn = new SDBConnection(bds.getConnection());
conn = new SDBConnection(connGen.generateConnection());
} catch (SQLException sqle) {
throw new RuntimeException(
"Unable to get SQL connection", sqle);
@ -145,6 +127,33 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
return getModel(METADATA_MODEL_URI);
}
private void setUpMetadata(Store store) {
Model metadataModel = getMetadataModel();
if (metadataModel.size() == 0) {
// set up the model name metadata to avoid expensive calls to
// listNames()
Resource sdbRes = metadataModel.createResource();
this.sdbResource = sdbRes;
Iterator nameIt = SDBFactory.connectDataset(store).listNames();
while (nameIt.hasNext()) {
String name = (String) nameIt.next();
metadataModel.add(sdbResource,metadataModel.getProperty(
HAS_NAMED_MODEL_URI),name);
}
} else {
StmtIterator stmtIt = metadataModel.listStatements(
(Resource) null, metadataModel.getProperty(
HAS_NAMED_MODEL_URI),(RDFNode) null);
if (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
sdbResource = stmt.getSubject();
}
stmtIt.close();
}
}
private boolean isWorking(Store store) {
Dataset d = SDBFactory.connectDataset(store);
try {
@ -175,7 +184,9 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
Model model = getModel(modelName);
Model metadataModel = getMetadataModel();
try {
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI), modelName);
metadataModel.add(
sdbResource,metadataModel.getProperty(
HAS_NAMED_MODEL_URI), modelName);
} finally {
metadataModel.close();
}
@ -188,13 +199,16 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
public GraphMaker getGraphMaker() {
throw new UnsupportedOperationException("GraphMaker not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"GraphMaker not supported by " + this.getClass().getName());
}
public boolean hasModel(String arg0) {
Model metadataModel = getMetadataModel();
try {
StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0);
StmtIterator stmtIt = metadataModel.listStatements(
sdbResource, metadataModel.getProperty(
HAS_NAMED_MODEL_URI), arg0);
try {
return stmtIt.hasNext();
} finally {
@ -207,137 +221,38 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
}
public ExtendedIterator listModels() {
ArrayList<String> metaNameList = new ArrayList<String>();
ArrayList<String> storeNameList = new ArrayList<String>();
ArrayList<String> unionNameList = new ArrayList<String>();
public ExtendedIterator<String> listModels() {
Model metadataModel = getMetadataModel();
Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(metadataModel.getProperty(HAS_NAMED_MODEL_URI));
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore());
Node node = null;
RDFNode rdfNode = null;
// implement comparator to sort the lists
class sortList implements Comparator<String>{
Collator collator = Collator.getInstance();
int compareResult;
public int compare(String str1, String str2){
compareResult = collator.compare(str1, str2);
if(compareResult > 0)
return 1;
else if(compareResult < 0)
return -1;
else
return 0;
}
}
// put the names into the lists.
while (metadataNameIt.hasNext()) {
rdfNode = metadataNameIt.next();
if (rdfNode.isLiteral()) {
metaNameList.add( ((Literal)rdfNode).getLexicalForm());
}
}
while (storeNameIt.hasNext()){
node = storeNameIt.next();
storeNameList.add(node.getURI());
}
// sort the lists
if(metaNameList.size()!=0)
Collections.sort(metaNameList, new sortList());
if(storeNameList.size()!=0)
Collections.sort(storeNameList, new sortList());
// code to merge the lists.
Collator collator = Collator.getInstance();
int check = 0;
Iterator<String> metaItr = metaNameList.iterator();
Iterator<String> storeItr = storeNameList.iterator();
String metaString = null;
String storeString = null;
do{
if(metaString != null && storeString !=null){
check = collator.compare(metaString, storeString);
}
else if(metaString!=null && storeString == null){
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
continue;
}
else if(metaString==null && storeString!=null){
unionNameList.add(storeString);
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
continue;
}
else{
if(metaItr.hasNext()){
metaString = metaItr.next();
}
if(storeItr.hasNext()){
storeString = storeItr.next();
}
if(metaString!=null && storeString !=null)
check = collator.compare(metaString, storeString);
else
continue;
}
if(check > 0){
unionNameList.add(storeString);
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
}
else if(check < 0){
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
}
else{
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
}
}while(metaString!=null || storeString!=null);
if (metadataModel != null) {
try {
return listModelNames(metadataModel);
} finally {
metadataModel.close();
}
}
return WrappedIterator.create(unionNameList.iterator());
private ExtendedIterator<String> listModelNames(Model metadataModel) {
Set<String> modelNameSet = new HashSet<String>();
Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(
metadataModel.getProperty(HAS_NAMED_MODEL_URI));
while (metadataNameIt.hasNext()) {
RDFNode rdfNode = metadataNameIt.next();
if (rdfNode.isLiteral()) {
modelNameSet.add(((Literal) rdfNode).getLexicalForm());
}
}
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore());
while (storeNameIt.hasNext()){
Node node = storeNameIt.next();
modelNameSet.add(node.getURI());
}
List<String> modelNameList = new ArrayList<String>();
modelNameList.addAll(modelNameSet);
Collections.sort(modelNameList, Collator.getInstance());
return WrappedIterator.create(modelNameList.iterator());
}
public Model openModel(String arg0, boolean arg1) {
@ -349,26 +264,31 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
m.removeAll(null,null,null);
Model metadataModel = getMetadataModel();
try {
metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
metadataModel.remove(sdbResource, metadataModel.getProperty(
HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
} finally {
metadataModel.close();
}
}
public Model addDescription(Model arg0, Resource arg1) {
throw new UnsupportedOperationException("addDescription not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"addDescription not supported by " + this.getClass().getName());
}
public Model createModelOver(String arg0) {
throw new UnsupportedOperationException("createModelOver not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"createModelOver not supported by " + this.getClass().getName());
}
public Model getDescription() {
throw new UnsupportedOperationException("createModelOver not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"createModelOver not supported by " + this.getClass().getName());
}
public Model getDescription(Resource arg0) {
throw new UnsupportedOperationException("getDescription not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"getDescription not supported by "+this.getClass().getName());
}
public Model openModel() {
@ -380,7 +300,8 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
public Model createFreshModel() {
throw new UnsupportedOperationException("createFreshModel not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"createFreshModel not supported by " + this.getClass().getName());
}
/**
@ -402,19 +323,23 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
}
public Model openModelIfPresent(String arg0) {
return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(getStore(),arg0) : null;
return (this.hasModel(arg0))
? SDBFactory.connectNamedModel(getStore(),arg0)
: null;
}
public Model getModel(String modelName) {
SDBGraphGenerator graphGen = new SDBGraphGenerator(
bds, storeDesc, modelName);
connGen, storeDesc, modelName);
Graph g = new RegeneratingGraph(
SDBFactory.connectNamedGraph(getStore(), modelName), graphGen);
return ModelFactory.createModelForGraph(g);
}
public Model getModel(String arg0, ModelReader arg1) {
throw new UnsupportedOperationException("getModel(String,ModelReader) not supported by "+this.getClass().getName());
throw new UnsupportedOperationException(
"getModel(String, ModelReader) not supported by " +
this.getClass().getName());
}
}

View file

@ -107,8 +107,11 @@ public class JenaDataSourceSetup extends JenaDataSourceSetupBase
StoreDesc storeDesc = makeStoreDesc(ctx);
setApplicationStoreDesc(storeDesc, ctx);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx);
this.setApplicationDataSource(bds, ctx);
BasicDataSource bds = getApplicationDataSource(ctx);
if (bds == null) {
bds = makeDataSourceFromConfigurationProperties(ctx);
setApplicationDataSource(bds, ctx);
}
Store store = connectStore(bds, storeDesc);
setApplicationStore(store, ctx);

View file

@ -39,12 +39,19 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
private static final Log log = LogFactory.getLog(
JenaDataSourceSetupBase.class);
protected final static String MAX_ACTIVE_PROPERTY =
"VitroConnection.DataSource.pool.maxActive";
protected final static String MAX_IDLE_PROPERTY =
"VitroConnection.DataSource.pool.maxIdle";
protected final static int DEFAULT_MAXWAIT = 10000, // ms
DEFAULT_MAXACTIVE = 40,
MINIMUM_MAXACTIVE = 20,
DEFAULT_MAXIDLE = 10,
DEFAULT_TIMEBETWEENEVICTIONS = 3 * 1000, // ms
DEFAULT_TIMEBETWEENEVICTIONS = 180 * 1000, // ms
DEFAULT_TESTSPEREVICTION = DEFAULT_MAXACTIVE,
DEFAULT_MINEVICTIONIDLETIME = 3 * 1000; // ms
DEFAULT_MINEVICTIONIDLETIME = 180 * 1000; // ms
protected final static boolean DEFAULT_TESTONBORROW = true,
DEFAULT_TESTONRETURN = true, DEFAULT_TESTWHILEIDLE = true;
@ -214,10 +221,18 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
ds.setPassword(password);
int maxActiveInt = DEFAULT_MAXACTIVE;
String maxActiveStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.pool.maxActive");
MAX_ACTIVE_PROPERTY);
if (!StringUtils.isEmpty(maxActiveStr)) {
try {
maxActiveInt = Integer.parseInt(maxActiveStr);
int maxActiveIntFromConfigProperties = Integer.parseInt(maxActiveStr);
if (maxActiveIntFromConfigProperties < MINIMUM_MAXACTIVE) {
log.warn("Specified value for " + MAX_ACTIVE_PROPERTY +
" is too low. Using minimum value of " +
MINIMUM_MAXACTIVE);
maxActiveInt = MINIMUM_MAXACTIVE;
} else {
maxActiveInt = maxActiveIntFromConfigProperties;
}
} catch (NumberFormatException nfe) {
log.error("Unable to parse connection pool maxActive setting "
+ maxActiveStr + " as an integer");
@ -227,7 +242,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
? maxActiveInt / 4
: DEFAULT_MAXIDLE;
String maxIdleStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.pool.maxIdle");
MAX_IDLE_PROPERTY);
if (!StringUtils.isEmpty(maxIdleStr)) {
try {
maxIdleInt = Integer.parseInt(maxIdleStr);
@ -245,6 +260,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
ds.setMinEvictableIdleTimeMillis(DEFAULT_MINEVICTIONIDLETIME);
ds.setNumTestsPerEvictionRun(maxActiveInt);
ds.setTimeBetweenEvictionRunsMillis(DEFAULT_TIMEBETWEENEVICTIONS);
ds.setInitialSize(ds.getMaxActive() / 10);
try {
ds.getConnection().close();
@ -388,13 +404,14 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
if (TripleStoreType.RDB.equals(type)){
vjmm = new VitroJenaModelMaker(
jdbcUrl, username, password, dbtypeStr, ctx);
}
else if(TripleStoreType.SDB.equals(type)){
} else if (TripleStoreType.SDB.equals(type)) {
StoreDesc storeDesc = new StoreDesc(
LayoutType.LayoutTripleNodesHash, DatabaseType.fetch(dbtypeStr));
BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
getDbDriverClassName(ctx), jdbcUrl, username, password, ctx);
bds.setMaxActive(4); // for now, the SDB model makers should not use more
// than a small handful of connections
bds.setMaxIdle(2);
try {
vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
} catch (SQLException sqle) {

View file

@ -9,6 +9,7 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.joda.time.DateTime;
@ -40,9 +41,12 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
// we do not want to fetch imports when we wrap Models in OntModels
OntDocumentManager.getInstance().setProcessImports(false);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx);
setApplicationDataSource(bds, ctx);
// user accounts Model
try {
Model userAccountsDbModel = makeDBModelFromConfigurationProperties(
Model userAccountsDbModel = makeDBModel(bds,
JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (userAccountsDbModel.size() == 0) {
firstStartup = true;
@ -66,7 +70,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
// display, editing and navigation Model
try {
Model displayDbModel = makeDBModelFromConfigurationProperties(
Model displayDbModel = makeDBModel(bds,
JENA_DISPLAY_METADATA_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (displayDbModel.size() == 0) {
readOntologyFilesInPathSet(APPPATH, ctx,displayDbModel);
@ -85,7 +89,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
//display tbox - currently reading in every time
try {
Model displayTboxModel = makeDBModelFromConfigurationProperties(
Model displayTboxModel = makeDBModel(bds,
JENA_DISPLAY_TBOX_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time?
@ -103,7 +107,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
//Display Display model, currently empty, create if doesn't exist but no files to load
try {
Model displayDisplayModel = makeDBModelFromConfigurationProperties(
Model displayDisplayModel = makeDBModel(bds,
JENA_DISPLAY_DISPLAY_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time?