litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-3440 rework of db connection management for VitroJenaSDBModelMaker

Browse source
This commit is contained in:
brianjlowe 2011-12-07 18:48:47 +00:00
parent 989f0d9f97
commit ae85e5c648
6 changed files with 471 additions and 486 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SDBGraphConnectionGenerator.java Normal file
View file

@ -0,0 +1,46 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.sql.Connection;
import java.sql.SQLException;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
public class SDBGraphConnectionGenerator {
private final static Log log = LogFactory.getLog(
SDBGraphConnectionGenerator.class);
private BasicDataSource ds = null;
private Connection connection = null;
public SDBGraphConnectionGenerator(BasicDataSource dataSource) {
this.ds = dataSource;
}
public Connection generateConnection() throws SQLException {
if ( this.connection == null ) {
this.connection = ds.getConnection();
} else if ( this.connection.isClosed() ) {
try {
this.connection.close();
} catch (SQLException e) {
// The connection will throw an "Already closed"
// SQLException that we need to catch. We need to
// make this extra call to .close() in order to make
// sure that the connection is returned to the pool.
// This depends on the particular behavior of version
// 1.4 of the Apache Commons connection pool library.
// Earlier versions threw the exception right away,
// making this impossible. Future versions may do the
// same.
}
this.connection = ds.getConnection();
}
return connection;
}
}

32
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/SDBGraphGenerator.java
View file

@ -17,14 +17,21 @@ public class SDBGraphGenerator implements SQLGraphGenerator {
private static final Log log = LogFactory.getLog(SDBGraphGenerator.class.getName()); private static final Log log = LogFactory.getLog(SDBGraphGenerator.class.getName());
private BasicDataSource ds; private SDBGraphConnectionGenerator connGen;
private Connection connection; private Connection connection;
private StoreDesc storeDesc; private StoreDesc storeDesc;
private String graphID; private String graphID;
public SDBGraphGenerator(BasicDataSource dataSource, StoreDesc storeDesc, public SDBGraphGenerator(BasicDataSource dataSource, StoreDesc storeDesc,
String graphID) { String graphID) {
this.ds = dataSource; this.connGen = new SDBGraphConnectionGenerator(dataSource);
this.storeDesc = storeDesc;
this.graphID = graphID;
}
public SDBGraphGenerator(SDBGraphConnectionGenerator connectionGenerator,
StoreDesc storeDesc, String graphID) {
this.connGen = connectionGenerator;
this.storeDesc = storeDesc; this.storeDesc = storeDesc;
this.graphID = graphID; this.graphID = graphID;
} }
@ -39,24 +46,7 @@ public class SDBGraphGenerator implements SQLGraphGenerator {
public Graph generateGraph() { public Graph generateGraph() {
try { try {
if ( this.connection == null ) { this.connection = connGen.generateConnection();
this.connection = ds.getConnection();
} else if ( this.connection.isClosed() ) {
try {
this.connection.close();
} catch (SQLException e) {
// The connection will throw an "Already closed"
// SQLException that we need to catch. We need to
// make this extra call to .close() in order to make
// sure that the connection is returned to the pool.
// This depends on the particular behavior of version
// 1.4 of the Apache Commons connection pool library.
// Earlier versions threw the exception right away,
// making this impossible. Future versions may do the
// same.
}
this.connection = ds.getConnection();
}
Store store = SDBFactory.connectStore(connection, storeDesc); Store store = SDBFactory.connectStore(connection, storeDesc);
return SDBFactory.connectNamedGraph(store, graphID); return SDBFactory.connectNamedGraph(store, graphID);
} catch (SQLException e) { } catch (SQLException e) {

597
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VitroJenaSDBModelMaker.java
View file

@ -2,18 +2,18 @@
package edu.cornell.mannlib.vitro.webapp.dao.jena; package edu.cornell.mannlib.vitro.webapp.dao.jena;
import java.sql.Connection;
import java.sql.SQLException; import java.sql.SQLException;
import java.text.Collator;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Set; import java.util.Set;
import org.apache.commons.dbcp.BasicDataSource; import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.graph.Graph; import com.hp.hpl.jena.graph.Graph;
import com.hp.hpl.jena.graph.GraphMaker; import com.hp.hpl.jena.graph.GraphMaker;
@ -28,13 +28,10 @@ import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ModelMaker; import com.hp.hpl.jena.rdf.model.ModelMaker;
import com.hp.hpl.jena.rdf.model.ModelReader; import com.hp.hpl.jena.rdf.model.ModelReader;
import com.hp.hpl.jena.rdf.model.Property;
import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource; import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement; import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator; import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.sdb.SDB;
import com.hp.hpl.jena.sdb.SDBFactory; import com.hp.hpl.jena.sdb.SDBFactory;
import com.hp.hpl.jena.sdb.Store; import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc; import com.hp.hpl.jena.sdb.StoreDesc;
@ -42,74 +39,59 @@ import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.sdb.util.StoreUtils; import com.hp.hpl.jena.sdb.util.StoreUtils;
import com.hp.hpl.jena.util.iterator.ExtendedIterator; import com.hp.hpl.jena.util.iterator.ExtendedIterator;
import com.hp.hpl.jena.util.iterator.WrappedIterator; import com.hp.hpl.jena.util.iterator.WrappedIterator;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS; import com.hp.hpl.jena.vocabulary.RDFS;
import com.ibm.icu.text.Collator;
public class VitroJenaSDBModelMaker implements ModelMaker { public class VitroJenaSDBModelMaker implements ModelMaker {
// TODO: need to rethink the inheritance/interfaces here private final static Log log = LogFactory.getLog(VitroJenaSDBModelMaker.class);
private StoreDesc storeDesc = null; private StoreDesc storeDesc = null;
private BasicDataSource bds = null; private BasicDataSource bds = null;
private SDBConnection conn = null; private SDBConnection conn = null;
private SDBGraphConnectionGenerator connGen = null;
public static final String METADATA_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
public static final String HAS_NAMED_MODEL_URI = "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel"; public static final String METADATA_MODEL_URI =
"http://vitro.mannlib.cornell.edu/ns/vitro/sdb/metadata";
private Resource sdbResource; // a resource representing the SDB database
public static final String HAS_NAMED_MODEL_URI =
public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds) throws SQLException { "http://vitro.mannlib.cornell.edu/ns/vitro/sdb/hasNamedModel";
this.storeDesc = storeDesc; private Resource sdbResource; // a resource representing the SDB database
this.bds = bds;
Store store = getStore(); public VitroJenaSDBModelMaker(StoreDesc storeDesc, BasicDataSource bds)
try { throws SQLException {
Model metadataModel = getMetadataModel(); this.storeDesc = storeDesc;
this.bds = bds;
if (metadataModel.size()==0) { connGen = new SDBGraphConnectionGenerator(bds);
// set up the model name metadata to avoid expensive calls to listNames()
Resource sdbRes = metadataModel.createResource(); Store store = getStore();
this.sdbResource = sdbRes; try {
Iterator nameIt = SDBFactory.connectDataset(store).listNames(); setUpMetadata(store);
while (nameIt.hasNext()) { } finally {
String name = (String) nameIt.next(); store.close();
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),name); }
} }
} else {
StmtIterator stmtIt = metadataModel.listStatements((Resource)null, metadataModel.getProperty(HAS_NAMED_MODEL_URI),(RDFNode)null); private static final int MAX_TRIES = 10;
if (stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement(); private Store getStore() {
sdbResource = stmt.getSubject(); Store store = null;
} boolean goodStore = false;
stmtIt.close(); int tries = 0;
} while (!goodStore && tries < MAX_TRIES) {
} finally { tries++;
store.close(); if (conn == null) {
} try {
} conn = new SDBConnection(connGen.generateConnection());
private static final int MAX_TRIES = 10;
private Store getStore() {
Store store = null;
boolean goodStore = false;
int tries = 0;
while (!goodStore && tries < MAX_TRIES) {
tries++;
if (conn == null) {
try {
conn = new SDBConnection(bds.getConnection());
} catch (SQLException sqle) { } catch (SQLException sqle) {
throw new RuntimeException( throw new RuntimeException(
"Unable to get SQL connection", sqle); "Unable to get SQL connection", sqle);
} }
} }
store = SDBFactory.connectStore(conn, storeDesc); store = SDBFactory.connectStore(conn, storeDesc);
try { try {
if (!StoreUtils.isFormatted(store)) { if (!StoreUtils.isFormatted(store)) {
// initialize the store // initialize the store
try { try {
store.getTableFormatter().create(); store.getTableFormatter().create();
@ -120,301 +102,244 @@ public class VitroJenaSDBModelMaker implements ModelMaker {
"VitroJenaSDBModelMaker", e); "VitroJenaSDBModelMaker", e);
} }
} }
} catch (SQLException sqle) { } catch (SQLException sqle) {
conn.close(); conn.close();
conn = null; conn = null;
} }
if (conn != null) { if (conn != null) {
if (isWorking(store)) { if (isWorking(store)) {
goodStore = true; goodStore = true;
} else { } else {
conn.close(); conn.close();
conn = null; conn = null;
} }
} }
} }
if (store == null) { if (store == null) {
throw new RuntimeException( throw new RuntimeException(
"Unable to connect to SDB store after " + "Unable to connect to SDB store after " +
MAX_TRIES + " attempts"); MAX_TRIES + " attempts");
} }
return store; return store;
} }
Model getMetadataModel() { Model getMetadataModel() {
return getModel(METADATA_MODEL_URI); return getModel(METADATA_MODEL_URI);
} }
private boolean isWorking(Store store) { private void setUpMetadata(Store store) {
Dataset d = SDBFactory.connectDataset(store); Model metadataModel = getMetadataModel();
try {
String validationQuery = "ASK { <" + RDFS.Resource.getURI() + "> " + if (metadataModel.size() == 0) {
" <" + RDFS.isDefinedBy.getURI() + "> " + // set up the model name metadata to avoid expensive calls to
" <" + RDFS.Resource.getURI() + "> }"; // listNames()
Query q = QueryFactory.create(validationQuery); Resource sdbRes = metadataModel.createResource();
QueryExecution qe = QueryExecutionFactory.create(q, d); this.sdbResource = sdbRes;
try { Iterator nameIt = SDBFactory.connectDataset(store).listNames();
qe.execAsk(); while (nameIt.hasNext()) {
return true; String name = (String) nameIt.next();
} catch (Exception e) { metadataModel.add(sdbResource,metadataModel.getProperty(
e.printStackTrace(); HAS_NAMED_MODEL_URI),name);
return false; }
} finally { } else {
qe.close(); StmtIterator stmtIt = metadataModel.listStatements(
} (Resource) null, metadataModel.getProperty(
} finally { HAS_NAMED_MODEL_URI),(RDFNode) null);
d.close(); if (stmtIt.hasNext()) {
} Statement stmt = stmtIt.nextStatement();
} sdbResource = stmt.getSubject();
}
public void close() { stmtIt.close();
getStore().close(); }
}
public Model createModel(String modelName) { }
Model model = getModel(modelName);
Model metadataModel = getMetadataModel(); private boolean isWorking(Store store) {
try { Dataset d = SDBFactory.connectDataset(store);
metadataModel.add(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI), modelName); try {
} finally { String validationQuery = "ASK { <" + RDFS.Resource.getURI() + "> " +
metadataModel.close(); " <" + RDFS.isDefinedBy.getURI() + "> " +
} " <" + RDFS.Resource.getURI() + "> }";
return model; Query q = QueryFactory.create(validationQuery);
} QueryExecution qe = QueryExecutionFactory.create(q, d);
try {
qe.execAsk();
return true;
} catch (Exception e) {
e.printStackTrace();
return false;
} finally {
qe.close();
}
} finally {
d.close();
}
}
public void close() {
getStore().close();
}
public Model createModel(String arg0, boolean arg1) { public Model createModel(String modelName) {
// TODO Figure out if we can offer a "create if not found" option using SDB Model model = getModel(modelName);
return createModel(arg0); Model metadataModel = getMetadataModel();
} try {
metadataModel.add(
sdbResource,metadataModel.getProperty(
HAS_NAMED_MODEL_URI), modelName);
} finally {
metadataModel.close();
}
return model;
}
public GraphMaker getGraphMaker() { public Model createModel(String arg0, boolean arg1) {
throw new UnsupportedOperationException("GraphMaker not supported by "+this.getClass().getName()); // TODO Figure out if we can offer a "create if not found" option using SDB
} return createModel(arg0);
}
public boolean hasModel(String arg0) { public GraphMaker getGraphMaker() {
Model metadataModel = getMetadataModel(); throw new UnsupportedOperationException(
try { "GraphMaker not supported by " + this.getClass().getName());
StmtIterator stmtIt = metadataModel.listStatements(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),arg0); }
try {
return stmtIt.hasNext();
} finally {
if (stmtIt != null) {
stmtIt.close();
}
}
} finally {
metadataModel.close();
}
}
public ExtendedIterator listModels() { public boolean hasModel(String arg0) {
ArrayList<String> metaNameList = new ArrayList<String>(); Model metadataModel = getMetadataModel();
ArrayList<String> storeNameList = new ArrayList<String>(); try {
ArrayList<String> unionNameList = new ArrayList<String>(); StmtIterator stmtIt = metadataModel.listStatements(
sdbResource, metadataModel.getProperty(
Model metadataModel = getMetadataModel(); HAS_NAMED_MODEL_URI), arg0);
try {
Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(metadataModel.getProperty(HAS_NAMED_MODEL_URI)); return stmtIt.hasNext();
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore()); } finally {
Node node = null; if (stmtIt != null) {
RDFNode rdfNode = null; stmtIt.close();
}
// implement comparator to sort the lists }
} finally {
class sortList implements Comparator<String>{ metadataModel.close();
Collator collator = Collator.getInstance(); }
int compareResult; }
public int compare(String str1, String str2){
compareResult = collator.compare(str1, str2);
if(compareResult > 0)
return 1;
else if(compareResult < 0)
return -1;
else
return 0;
}
}
// put the names into the lists.
while (metadataNameIt.hasNext()) {
rdfNode = metadataNameIt.next();
if (rdfNode.isLiteral()) {
metaNameList.add( ((Literal)rdfNode).getLexicalForm());
}
}
while (storeNameIt.hasNext()){
node = storeNameIt.next();
storeNameList.add(node.getURI());
}
// sort the lists
if(metaNameList.size()!=0)
Collections.sort(metaNameList, new sortList());
if(storeNameList.size()!=0)
Collections.sort(storeNameList, new sortList());
// code to merge the lists.
Collator collator = Collator.getInstance();
int check = 0;
Iterator<String> metaItr = metaNameList.iterator();
Iterator<String> storeItr = storeNameList.iterator();
String metaString = null;
String storeString = null;
do{
if(metaString != null && storeString !=null){
check = collator.compare(metaString, storeString);
}
else if(metaString!=null && storeString == null){
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
continue;
}
else if(metaString==null && storeString!=null){
unionNameList.add(storeString);
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
continue;
}
else{
if(metaItr.hasNext()){
metaString = metaItr.next();
}
if(storeItr.hasNext()){
storeString = storeItr.next();
}
if(metaString!=null && storeString !=null)
check = collator.compare(metaString, storeString);
else
continue;
}
if(check > 0){
unionNameList.add(storeString);
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
}
else if(check < 0){
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
}
else{
unionNameList.add(metaString);
if(metaItr.hasNext())
metaString = metaItr.next();
else
metaString = null;
if(storeItr.hasNext())
storeString = storeItr.next();
else
storeString = null;
}
}while(metaString!=null || storeString!=null);
if (metadataModel != null) {
metadataModel.close();
}
return WrappedIterator.create(unionNameList.iterator());
}
public Model openModel(String arg0, boolean arg1) { public ExtendedIterator<String> listModels() {
return SDBFactory.connectNamedModel(getStore(),arg0); Model metadataModel = getMetadataModel();
} try {
return listModelNames(metadataModel);
} finally {
metadataModel.close();
}
}
private ExtendedIterator<String> listModelNames(Model metadataModel) {
Set<String> modelNameSet = new HashSet<String>();
Iterator<RDFNode> metadataNameIt = metadataModel.listObjectsOfProperty(
metadataModel.getProperty(HAS_NAMED_MODEL_URI));
while (metadataNameIt.hasNext()) {
RDFNode rdfNode = metadataNameIt.next();
if (rdfNode.isLiteral()) {
modelNameSet.add(((Literal) rdfNode).getLexicalForm());
}
}
Iterator<Node> storeNameIt = StoreUtils.storeGraphNames(getStore());
while (storeNameIt.hasNext()){
Node node = storeNameIt.next();
modelNameSet.add(node.getURI());
}
List<String> modelNameList = new ArrayList<String>();
modelNameList.addAll(modelNameSet);
Collections.sort(modelNameList, Collator.getInstance());
return WrappedIterator.create(modelNameList.iterator());
}
public void removeModel(String arg0) { public Model openModel(String arg0, boolean arg1) {
Model m = getModel(arg0); return SDBFactory.connectNamedModel(getStore(),arg0);
m.removeAll(null,null,null); }
Model metadataModel = getMetadataModel();
try {
metadataModel.remove(sdbResource,metadataModel.getProperty(HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
} finally {
metadataModel.close();
}
}
public Model addDescription(Model arg0, Resource arg1) { public void removeModel(String arg0) {
throw new UnsupportedOperationException("addDescription not supported by "+this.getClass().getName()); Model m = getModel(arg0);
} m.removeAll(null,null,null);
Model metadataModel = getMetadataModel();
try {
metadataModel.remove(sdbResource, metadataModel.getProperty(
HAS_NAMED_MODEL_URI),metadataModel.createLiteral(arg0));
} finally {
metadataModel.close();
}
}
public Model createModelOver(String arg0) { public Model addDescription(Model arg0, Resource arg1) {
throw new UnsupportedOperationException("createModelOver not supported by "+this.getClass().getName()); throw new UnsupportedOperationException(
} "addDescription not supported by " + this.getClass().getName());
}
public Model getDescription() { public Model createModelOver(String arg0) {
throw new UnsupportedOperationException("createModelOver not supported by "+this.getClass().getName()); throw new UnsupportedOperationException(
} "createModelOver not supported by " + this.getClass().getName());
}
public Model getDescription(Resource arg0) { public Model getDescription() {
throw new UnsupportedOperationException("getDescription not supported by "+this.getClass().getName()); throw new UnsupportedOperationException(
} "createModelOver not supported by " + this.getClass().getName());
}
public Model openModel() { public Model getDescription(Resource arg0) {
return SDBFactory.connectDefaultModel(getStore()); throw new UnsupportedOperationException(
} "getDescription not supported by "+this.getClass().getName());
}
public Model createDefaultModel() { public Model openModel() {
return SDBFactory.connectDefaultModel(getStore()); return SDBFactory.connectDefaultModel(getStore());
} }
public Model createFreshModel() { public Model createDefaultModel() {
throw new UnsupportedOperationException("createFreshModel not supported by "+this.getClass().getName()); return SDBFactory.connectDefaultModel(getStore());
} }
/** public Model createFreshModel() {
* @deprecated throw new UnsupportedOperationException(
*/ "createFreshModel not supported by " + this.getClass().getName());
public Model createModel() { }
return SDBFactory.connectDefaultModel(getStore());
}
/** /**
* @deprecated * @deprecated
*/ */
public Model getModel() { public Model createModel() {
return SDBFactory.connectDefaultModel(getStore()); return SDBFactory.connectDefaultModel(getStore());
} }
public Model openModel(String arg0) { /**
return SDBFactory.connectDefaultModel(getStore()); * @deprecated
} */
public Model getModel() {
return SDBFactory.connectDefaultModel(getStore());
}
public Model openModelIfPresent(String arg0) { public Model openModel(String arg0) {
return (this.hasModel(arg0)) ? SDBFactory.connectNamedModel(getStore(),arg0) : null; return SDBFactory.connectDefaultModel(getStore());
} }
public Model getModel(String modelName) { public Model openModelIfPresent(String arg0) {
SDBGraphGenerator graphGen = new SDBGraphGenerator( return (this.hasModel(arg0))
bds, storeDesc, modelName); ? SDBFactory.connectNamedModel(getStore(),arg0)
Graph g = new RegeneratingGraph( : null;
SDBFactory.connectNamedGraph(getStore(), modelName), graphGen); }
return ModelFactory.createModelForGraph(g);
}
public Model getModel(String arg0, ModelReader arg1) { public Model getModel(String modelName) {
throw new UnsupportedOperationException("getModel(String,ModelReader) not supported by "+this.getClass().getName()); SDBGraphGenerator graphGen = new SDBGraphGenerator(
} connGen, storeDesc, modelName);
Graph g = new RegeneratingGraph(
SDBFactory.connectNamedGraph(getStore(), modelName), graphGen);
return ModelFactory.createModelForGraph(g);
}
public Model getModel(String arg0, ModelReader arg1) {
throw new UnsupportedOperationException(
"getModel(String, ModelReader) not supported by " +
this.getClass().getName());
}
} }

7
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetup.java
View file

@ -107,8 +107,11 @@ public class JenaDataSourceSetup extends JenaDataSourceSetupBase
StoreDesc storeDesc = makeStoreDesc(ctx); StoreDesc storeDesc = makeStoreDesc(ctx);
setApplicationStoreDesc(storeDesc, ctx); setApplicationStoreDesc(storeDesc, ctx);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx); BasicDataSource bds = getApplicationDataSource(ctx);
this.setApplicationDataSource(bds, ctx); if (bds == null) {
bds = makeDataSourceFromConfigurationProperties(ctx);
setApplicationDataSource(bds, ctx);
}
Store store = connectStore(bds, storeDesc); Store store = connectStore(bds, storeDesc);
setApplicationStore(store, ctx); setApplicationStore(store, ctx);

263
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -34,17 +34,24 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaModelMaker;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaSDBModelMaker; import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaSDBModelMaker;
public class JenaDataSourceSetupBase extends JenaBaseDaoCon { public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
private static final String VITRO_DEFAULT_NAMESPACE = "Vitro.defaultNamespace"; private static final String VITRO_DEFAULT_NAMESPACE = "Vitro.defaultNamespace";
private static final Log log = LogFactory.getLog( private static final Log log = LogFactory.getLog(
JenaDataSourceSetupBase.class); JenaDataSourceSetupBase.class);
protected final static String MAX_ACTIVE_PROPERTY =
"VitroConnection.DataSource.pool.maxActive";
protected final static String MAX_IDLE_PROPERTY =
"VitroConnection.DataSource.pool.maxIdle";
protected final static int DEFAULT_MAXWAIT = 10000, // ms protected final static int DEFAULT_MAXWAIT = 10000, // ms
DEFAULT_MAXACTIVE = 40, DEFAULT_MAXACTIVE = 40,
MINIMUM_MAXACTIVE = 20,
DEFAULT_MAXIDLE = 10, DEFAULT_MAXIDLE = 10,
DEFAULT_TIMEBETWEENEVICTIONS = 3 * 1000, // ms DEFAULT_TIMEBETWEENEVICTIONS = 180 * 1000, // ms
DEFAULT_TESTSPEREVICTION = DEFAULT_MAXACTIVE, DEFAULT_TESTSPEREVICTION = DEFAULT_MAXACTIVE,
DEFAULT_MINEVICTIONIDLETIME = 3 * 1000; // ms DEFAULT_MINEVICTIONIDLETIME = 180 * 1000; // ms
protected final static boolean DEFAULT_TESTONBORROW = true, protected final static boolean DEFAULT_TESTONBORROW = true,
DEFAULT_TESTONRETURN = true, DEFAULT_TESTWHILEIDLE = true; DEFAULT_TESTONRETURN = true, DEFAULT_TESTWHILEIDLE = true;
@ -132,8 +139,8 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
static final OntModelSpec MEM_ONT_MODEL_SPEC = OntModelSpec.OWL_MEM; static final OntModelSpec MEM_ONT_MODEL_SPEC = OntModelSpec.OWL_MEM;
private String getJdbcUrl(ServletContext ctx) { private String getJdbcUrl(ServletContext ctx) {
String jdbcUrl = ConfigurationProperties.getBean(ctx).getProperty( String jdbcUrl = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.url"); "VitroConnection.DataSource.url");
// Ensure that MySQL handles unicode properly, else all kinds of // Ensure that MySQL handles unicode properly, else all kinds of
// horrible nastiness ensues. // horrible nastiness ensues.
@ -155,8 +162,8 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
String jdbcUrl = getJdbcUrl(ctx); String jdbcUrl = getJdbcUrl(ctx);
String username = ConfigurationProperties.getBean(ctx).getProperty( String username = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.username"); "VitroConnection.DataSource.username");
String password = ConfigurationProperties.getBean(ctx).getProperty( String password = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.password"); "VitroConnection.DataSource.password");
BasicDataSource ds = makeBasicDataSource( BasicDataSource ds = makeBasicDataSource(
@ -175,29 +182,29 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
* a properties file. * a properties file.
*/ */
public final BasicDataSource makeDataSourceFromConfigurationProperties(ServletContext ctx){ public final BasicDataSource makeDataSourceFromConfigurationProperties(ServletContext ctx){
String dbDriverClassname = ConfigurationProperties.getBean(ctx) String dbDriverClassname = ConfigurationProperties.getBean(ctx)
.getProperty("VitroConnection.DataSource.driver", .getProperty("VitroConnection.DataSource.driver",
getDbDriverClassName(ctx)); getDbDriverClassName(ctx));
String jdbcUrl = getJdbcUrl(ctx); String jdbcUrl = getJdbcUrl(ctx);
String username = ConfigurationProperties.getBean(ctx).getProperty( String username = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.username"); "VitroConnection.DataSource.username");
String password = ConfigurationProperties.getBean(ctx).getProperty( String password = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.password"); "VitroConnection.DataSource.password");
return makeBasicDataSource( return makeBasicDataSource(
dbDriverClassname, jdbcUrl, username, password, ctx); dbDriverClassname, jdbcUrl, username, password, ctx);
} }
public void setApplicationDataSource(BasicDataSource bds, public void setApplicationDataSource(BasicDataSource bds,
ServletContext ctx) { ServletContext ctx) {
ctx.setAttribute(getDataSourceAttributeName(), bds); ctx.setAttribute(getDataSourceAttributeName(), bds);
} }
public static BasicDataSource getApplicationDataSource(ServletContext ctx) { public static BasicDataSource getApplicationDataSource(ServletContext ctx) {
return (BasicDataSource) ctx.getAttribute(getDataSourceAttributeName()); return (BasicDataSource) ctx.getAttribute(getDataSourceAttributeName());
} }
private static String getDataSourceAttributeName() { private static String getDataSourceAttributeName() {
return JenaDataSourceSetupBase.class.getName() + ".dataSource"; return JenaDataSourceSetupBase.class.getName() + ".dataSource";
} }
public static BasicDataSource makeBasicDataSource(String dbDriverClassname, public static BasicDataSource makeBasicDataSource(String dbDriverClassname,
@ -213,21 +220,29 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
ds.setUsername(username); ds.setUsername(username);
ds.setPassword(password); ds.setPassword(password);
int maxActiveInt = DEFAULT_MAXACTIVE; int maxActiveInt = DEFAULT_MAXACTIVE;
String maxActiveStr = ConfigurationProperties.getBean(ctx).getProperty( String maxActiveStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.pool.maxActive"); MAX_ACTIVE_PROPERTY);
if (!StringUtils.isEmpty(maxActiveStr)) { if (!StringUtils.isEmpty(maxActiveStr)) {
try { try {
maxActiveInt = Integer.parseInt(maxActiveStr); int maxActiveIntFromConfigProperties = Integer.parseInt(maxActiveStr);
if (maxActiveIntFromConfigProperties < MINIMUM_MAXACTIVE) {
log.warn("Specified value for " + MAX_ACTIVE_PROPERTY +
" is too low. Using minimum value of " +
MINIMUM_MAXACTIVE);
maxActiveInt = MINIMUM_MAXACTIVE;
} else {
maxActiveInt = maxActiveIntFromConfigProperties;
}
} catch (NumberFormatException nfe) { } catch (NumberFormatException nfe) {
log.error("Unable to parse connection pool maxActive setting " log.error("Unable to parse connection pool maxActive setting "
+ maxActiveStr + " as an integer"); + maxActiveStr + " as an integer");
} }
} }
int maxIdleInt = (maxActiveInt > DEFAULT_MAXACTIVE) int maxIdleInt = (maxActiveInt > DEFAULT_MAXACTIVE)
? maxActiveInt / 4 ? maxActiveInt / 4
: DEFAULT_MAXIDLE; : DEFAULT_MAXIDLE;
String maxIdleStr = ConfigurationProperties.getBean(ctx).getProperty( String maxIdleStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.pool.maxIdle"); MAX_IDLE_PROPERTY);
if (!StringUtils.isEmpty(maxIdleStr)) { if (!StringUtils.isEmpty(maxIdleStr)) {
try { try {
maxIdleInt = Integer.parseInt(maxIdleStr); maxIdleInt = Integer.parseInt(maxIdleStr);
@ -245,6 +260,7 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
ds.setMinEvictableIdleTimeMillis(DEFAULT_MINEVICTIONIDLETIME); ds.setMinEvictableIdleTimeMillis(DEFAULT_MINEVICTIONIDLETIME);
ds.setNumTestsPerEvictionRun(maxActiveInt); ds.setNumTestsPerEvictionRun(maxActiveInt);
ds.setTimeBetweenEvictionRunsMillis(DEFAULT_TIMEBETWEENEVICTIONS); ds.setTimeBetweenEvictionRunsMillis(DEFAULT_TIMEBETWEENEVICTIONS);
ds.setInitialSize(ds.getMaxActive() / 10);
try { try {
ds.getConnection().close(); ds.getConnection().close();
@ -256,19 +272,19 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
} }
public enum TripleStoreType { public enum TripleStoreType {
RDB, SDB RDB, SDB
} }
public static boolean isFirstStartup() { public static boolean isFirstStartup() {
return firstStartup; return firstStartup;
} }
protected Model makeDBModel(BasicDataSource ds, protected Model makeDBModel(BasicDataSource ds,
String jenaDbModelname, String jenaDbModelname,
OntModelSpec jenaDbOntModelSpec, OntModelSpec jenaDbOntModelSpec,
ServletContext ctx) { ServletContext ctx) {
return makeDBModel( return makeDBModel(
ds, jenaDbModelname, jenaDbOntModelSpec, TripleStoreType.RDB, ctx); ds, jenaDbModelname, jenaDbOntModelSpec, TripleStoreType.RDB, ctx);
} }
protected Model makeDBModel(BasicDataSource ds, protected Model makeDBModel(BasicDataSource ds,
@ -290,28 +306,28 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
try { try {
Graph g = null; Graph g = null;
switch (storeType) { switch (storeType) {
case RDB: case RDB:
g = new RegeneratingGraph( g = new RegeneratingGraph(
new RDBGraphGenerator( new RDBGraphGenerator(
ds, dbType, jenaDbModelName)); ds, dbType, jenaDbModelName));
break; break;
case SDB: case SDB:
String layoutStr = ConfigurationProperties.getBean(ctx) String layoutStr = ConfigurationProperties.getBean(ctx)
.getProperty( .getProperty(
"VitroConnection.DataSource.sdb.layout", "VitroConnection.DataSource.sdb.layout",
"layout2/hash"); "layout2/hash");
String dbtypeStr = ConfigurationProperties.getBean(ctx) String dbtypeStr = ConfigurationProperties.getBean(ctx)
.getProperty("VitroConnection.DataSource.dbtype", .getProperty("VitroConnection.DataSource.dbtype",
"MySQL"); "MySQL");
StoreDesc desc = new StoreDesc( StoreDesc desc = new StoreDesc(
LayoutType.fetch(layoutStr), LayoutType.fetch(layoutStr),
DatabaseType.fetch(dbtypeStr) ); DatabaseType.fetch(dbtypeStr) );
g = new RegeneratingGraph( g = new RegeneratingGraph(
new SDBGraphGenerator( new SDBGraphGenerator(
ds, desc, jenaDbModelName)); ds, desc, jenaDbModelName));
break; break;
default: throw new RuntimeException ( default: throw new RuntimeException (
"Unsupported store type " + storeType); "Unsupported store type " + storeType);
} }
dbModel = ModelFactory.createModelForGraph(g); dbModel = ModelFactory.createModelForGraph(g);
log.debug("Using database at "+ds.getUrl()); log.debug("Using database at "+ds.getUrl());
@ -336,28 +352,28 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
} }
public static void readOntologyFileFromPath(String p, Model model, ServletContext ctx) { public static void readOntologyFileFromPath(String p, Model model, ServletContext ctx) {
//Check that this is a file and not a directory //Check that this is a file and not a directory
File f = new File(ctx.getRealPath(p)); File f = new File(ctx.getRealPath(p));
if(f.exists() && f.isFile()){ if(f.exists() && f.isFile()){
String format = getRdfFormat(p); String format = getRdfFormat(p);
log.info("Loading ontology file at " + p + log.info("Loading ontology file at " + p +
" as format " + format); " as format " + format);
InputStream ontologyInputStream = ctx.getResourceAsStream(p); InputStream ontologyInputStream = ctx.getResourceAsStream(p);
try { try {
model.read(ontologyInputStream, null, format); model.read(ontologyInputStream, null, format);
log.debug("...successful"); log.debug("...successful");
} catch (Throwable t) { } catch (Throwable t) {
log.error("Failed to load ontology file at '" + p + log.error("Failed to load ontology file at '" + p +
"' as format " + format, t); "' as format " + format, t);
} }
} else { } else {
if(!f.exists()) { if(!f.exists()) {
log.debug("File for path " + p + " does not exist"); log.debug("File for path " + p + " does not exist");
} }
else if(f.isDirectory()) { else if(f.isDirectory()) {
log.debug("Path " + p + " corresponds to directory and not file so was not read in"); log.debug("Path " + p + " corresponds to directory and not file so was not read in");
} }
} }
} }
private static String getRdfFormat(String filename){ private static String getRdfFormat(String filename){
@ -378,32 +394,33 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
private static final String rdbModelMaker = "vitroJenaModelMaker"; private static final String rdbModelMaker = "vitroJenaModelMaker";
protected void makeModelMakerFromConnectionProperties(TripleStoreType type, ServletContext ctx){ protected void makeModelMakerFromConnectionProperties(TripleStoreType type, ServletContext ctx){
String jdbcUrl = getJdbcUrl(ctx); String jdbcUrl = getJdbcUrl(ctx);
String dbtypeStr = ConfigurationProperties.getBean(ctx).getProperty("VitroConnection.DataSource.dbtype","MySQL"); String dbtypeStr = ConfigurationProperties.getBean(ctx).getProperty("VitroConnection.DataSource.dbtype","MySQL");
String username = ConfigurationProperties.getBean(ctx).getProperty( String username = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.username"); "VitroConnection.DataSource.username");
String password = ConfigurationProperties.getBean(ctx).getProperty( String password = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.password"); "VitroConnection.DataSource.password");
if (TripleStoreType.RDB.equals(type)){ if (TripleStoreType.RDB.equals(type)){
vjmm = new VitroJenaModelMaker( vjmm = new VitroJenaModelMaker(
jdbcUrl, username, password, dbtypeStr, ctx); jdbcUrl, username, password, dbtypeStr, ctx);
} } else if (TripleStoreType.SDB.equals(type)) {
StoreDesc storeDesc = new StoreDesc(
else if(TripleStoreType.SDB.equals(type)){ LayoutType.LayoutTripleNodesHash, DatabaseType.fetch(dbtypeStr));
StoreDesc storeDesc = new StoreDesc( BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource(
LayoutType.LayoutTripleNodesHash, DatabaseType.fetch(dbtypeStr)); getDbDriverClassName(ctx), jdbcUrl, username, password, ctx);
BasicDataSource bds = JenaDataSourceSetup.makeBasicDataSource( bds.setMaxActive(4); // for now, the SDB model makers should not use more
getDbDriverClassName(ctx), jdbcUrl, username, password, ctx); // than a small handful of connections
try { bds.setMaxIdle(2);
vsmm = new VitroJenaSDBModelMaker(storeDesc, bds); try {
} catch (SQLException sqle) { vsmm = new VitroJenaSDBModelMaker(storeDesc, bds);
log.error("Unable to set up SDB ModelMaker", sqle); } catch (SQLException sqle) {
} log.error("Unable to set up SDB ModelMaker", sqle);
} }
}
return;
return;
} }
/** /**
@ -441,47 +458,47 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
public static void setVitroJenaModelMaker(VitroJenaModelMaker vjmm, public static void setVitroJenaModelMaker(VitroJenaModelMaker vjmm,
ServletContext ctx){ ServletContext ctx){
ctx.setAttribute(rdbModelMaker, vjmm); ctx.setAttribute(rdbModelMaker, vjmm);
} }
public static void setVitroJenaSDBModelMaker(VitroJenaSDBModelMaker vsmm, public static void setVitroJenaSDBModelMaker(VitroJenaSDBModelMaker vsmm,
ServletContext ctx){ ServletContext ctx){
ctx.setAttribute(sdbModelMaker, vsmm); ctx.setAttribute(sdbModelMaker, vsmm);
} }
protected String getDefaultNamespace(ServletContext ctx) { protected String getDefaultNamespace(ServletContext ctx) {
String dns = ConfigurationProperties.getBean(ctx).getProperty( String dns = ConfigurationProperties.getBean(ctx).getProperty(
VITRO_DEFAULT_NAMESPACE); VITRO_DEFAULT_NAMESPACE);
if ((dns != null) && (!dns.isEmpty())) { if ((dns != null) && (!dns.isEmpty())) {
return dns; return dns;
} else { } else {
throw new IllegalStateException("deploy.properties does not " throw new IllegalStateException("deploy.properties does not "
+ "contain a value for '" + VITRO_DEFAULT_NAMESPACE + "'"); + "contain a value for '" + VITRO_DEFAULT_NAMESPACE + "'");
} }
} }
protected VitroJenaModelMaker getVitroJenaModelMaker(){ protected VitroJenaModelMaker getVitroJenaModelMaker(){
return vjmm; return vjmm;
} }
protected VitroJenaSDBModelMaker getVitroJenaSDBModelMaker(){ protected VitroJenaSDBModelMaker getVitroJenaSDBModelMaker(){
return vsmm; return vsmm;
} }
private static String getDbType(ServletContext ctx) { private static String getDbType(ServletContext ctx) {
return ConfigurationProperties.getBean(ctx).getProperty( // database type return ConfigurationProperties.getBean(ctx).getProperty( // database type
"VitroConnection.DataSource.dbtype", "MySQL"); "VitroConnection.DataSource.dbtype", "MySQL");
} }
private static String getDbDriverClassName(ServletContext ctx) { private static String getDbDriverClassName(ServletContext ctx) {
return ConfigurationProperties.getBean(ctx).getProperty( return ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.driver", "com.mysql.jdbc.Driver"); "VitroConnection.DataSource.driver", "com.mysql.jdbc.Driver");
} }
private static String getValidationQuery(ServletContext ctx) { private static String getValidationQuery(ServletContext ctx) {
return ConfigurationProperties.getBean(ctx).getProperty( return ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.validationQuery", "SELECT 1"); "VitroConnection.DataSource.validationQuery", "SELECT 1");
} }
} }

12
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaPersistentDataSourceSetup.java
View file

@ -9,6 +9,7 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener; import javax.servlet.ServletContextListener;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.joda.time.DateTime; import org.joda.time.DateTime;
@ -39,10 +40,13 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
// we do not want to fetch imports when we wrap Models in OntModels // we do not want to fetch imports when we wrap Models in OntModels
OntDocumentManager.getInstance().setProcessImports(false); OntDocumentManager.getInstance().setProcessImports(false);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx);
setApplicationDataSource(bds, ctx);
// user accounts Model // user accounts Model
try { try {
Model userAccountsDbModel = makeDBModelFromConfigurationProperties( Model userAccountsDbModel = makeDBModel(bds,
JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (userAccountsDbModel.size() == 0) { if (userAccountsDbModel.size() == 0) {
firstStartup = true; firstStartup = true;
@ -66,7 +70,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
// display, editing and navigation Model // display, editing and navigation Model
try { try {
Model displayDbModel = makeDBModelFromConfigurationProperties( Model displayDbModel = makeDBModel(bds,
JENA_DISPLAY_METADATA_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_METADATA_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (displayDbModel.size() == 0) { if (displayDbModel.size() == 0) {
readOntologyFilesInPathSet(APPPATH, ctx,displayDbModel); readOntologyFilesInPathSet(APPPATH, ctx,displayDbModel);
@ -85,7 +89,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
//display tbox - currently reading in every time //display tbox - currently reading in every time
try { try {
Model displayTboxModel = makeDBModelFromConfigurationProperties( Model displayTboxModel = makeDBModel(bds,
JENA_DISPLAY_TBOX_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_TBOX_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time //Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time? //TODO: Check if original needs to be cleared/removed every time?
@ -103,7 +107,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
//Display Display model, currently empty, create if doesn't exist but no files to load //Display Display model, currently empty, create if doesn't exist but no files to load
try { try {
Model displayDisplayModel = makeDBModelFromConfigurationProperties( Model displayDisplayModel = makeDBModel(bds,
JENA_DISPLAY_DISPLAY_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_DISPLAY_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time //Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time? //TODO: Check if original needs to be cleared/removed every time?