litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-2440 Remove RDB support

Browse source
This commit is contained in:
brianjlowe 2011-05-27 20:36:48 +00:00
parent 1777aec82c
commit 04edace976
13 changed files with 710 additions and 1371 deletions
Download patch file Download diff file Expand all files Collapse all files

2
webapp/config/web.xml
View file

@ -114,11 +114,9 @@
<!-- See documentation for details. -->
<!-- If used, must be run after JenaDataSourceSetup -->
<!--
<listener>
<listener-class>edu.cornell.mannlib.vitro.webapp.servlet.setup.PelletReasonerSetup</listener-class>
</listener>
-->
<!-- The followng listener records all edit changes, in reified form, to another database model -->
<!-- still at an experimental stage -->

130
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SDBSetupController.java
View file

@ -1,130 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.controller.freemarker;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.usepages.UseMiscellaneousAdminPages;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupSDB;
public class SDBSetupController extends FreemarkerHttpServlet {
private static final Log log = LogFactory.getLog(
SDBSetupController.class);
private static final String SDB_SETUP_FTL = "sdbSetup.ftl";
@Override
protected Actions requiredActions(VitroRequest vreq) {
return new Actions(new UseMiscellaneousAdminPages());
}
protected ResponseValues processRequest(VitroRequest vreq) {
Map<String, Object> body = new HashMap<String, Object>();
String messageStr = "";
try {
JenaDataSourceSetupSDB jenaDataSourceSetupSDB = new JenaDataSourceSetupSDB();
Boolean done = (Boolean)getServletContext().getAttribute("done");
String setupsignal = (String) vreq.getParameter("setupsignal");
if (done!=null && done) {
messageStr = "SDB is currently being set up.";
} else{
String sdbsetup = (String)getServletContext().getAttribute("sdbsetup");
if(sdbsetup == null || sdbsetup.equals("showButton") || setupsignal == null){
body.put("link", "show");
messageStr = null;
getServletContext().setAttribute("sdbsetup", "yes");
if(getServletContext().getAttribute("sdbstatus")!=null)
body.put("sdbstatus",getServletContext().getAttribute("sdbstatus"));
else
body.put("sdbstatus"," ");
}
else if(setupsignal!=null && setupsignal.equals("setup")){
new Thread(new SDBSetupRunner(jenaDataSourceSetupSDB, vreq)).start();
messageStr = "SDB setup started.";
getServletContext().setAttribute("sdbsetup", "showButton");
}
}
} catch (Exception e) {
log.error("Error setting up SDB store", e);
body.put("errorMessage",
"Error setting up SDB store: " +
e.getMessage());
return new ExceptionResponseValues(
SDB_SETUP_FTL, body, e);
}
body.put("message", messageStr);
return new TemplateResponseValues(SDB_SETUP_FTL, body);
}
private class SDBSetupRunner implements Runnable {
private JenaDataSourceSetupSDB jenaDataSourceSetupSDB;
final OntModelSpec MEM_ONT_MODEL_SPEC = OntModelSpec.OWL_MEM;
private final ServletContext ctx;
public SDBSetupRunner(JenaDataSourceSetupSDB jenaDataSourceSetupSDB, HttpServletRequest req) {
this.jenaDataSourceSetupSDB = jenaDataSourceSetupSDB;
this.ctx = req.getSession().getServletContext();
}
@Override
public void run() {
Boolean done = true;
getServletContext().setAttribute("done",done);
StoreDesc storeDesc = jenaDataSourceSetupSDB.makeStoreDesc(ctx);
BasicDataSource bds = jenaDataSourceSetupSDB.makeDataSourceFromConfigurationProperties(getServletContext());
Store store = null;
try {
store = JenaDataSourceSetupSDB.connectStore(bds, storeDesc);
} catch (SQLException e) {
log.error("Error while getting the sdb store with given store description and basic data source", e);
}
OntModel memModel = (OntModel)getServletContext().getAttribute("jenaOntModel");
if (memModel == null) {
memModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
log.warn("WARNING: no database connected. Changes will disappear after context restart.");
}
OntModel inferenceModel = (OntModel)getServletContext().getAttribute("inferenceOntModel");
if(inferenceModel == null){
inferenceModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
}
if (store!=null) {
log.info("Setting up SDB store.");
try{
jenaDataSourceSetupSDB.setupSDB(getServletContext(), store, memModel, inferenceModel);
getServletContext().setAttribute("sdbstatus","SDB setup done successfully");
}
catch(Exception e){
getServletContext().setAttribute("sdbstatus",e.getMessage());
}
log.info("SDB setup complete.");
}
done = false;
getServletContext().setAttribute("done",done);
}
}
}

94
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/JenaExportController.java
View file

@ -93,7 +93,9 @@ public class JenaExportController extends BaseEditController {
Model model = null;
OntModel ontModel = ModelFactory.createOntologyModel();
if(!subgraphParam.equalsIgnoreCase("tbox") && !subgraphParam.equalsIgnoreCase("abox") && !subgraphParam.equalsIgnoreCase("full")){
if(!subgraphParam.equalsIgnoreCase("tbox")
&& !subgraphParam.equalsIgnoreCase("abox")
&& !subgraphParam.equalsIgnoreCase("full")){
ontologyURI = subgraphParam;
subgraphParam = "tbox";
char[] uri = ontologyURI.toCharArray();
@ -101,38 +103,20 @@ public class JenaExportController extends BaseEditController {
for(int i =0; i < uri.length-1;i++)
ontologyURI = ontologyURI + uri[i];
}
String mode = (JenaDataSourceSetupBase.isSDBActive(vreq)) ? "SDB" : "RDB";
if( "abox".equals(subgraphParam)){
model = ModelFactory.createDefaultModel();
if("inferred".equals(assertedOrInferredParam)){
if(mode.equals("RDB")){
Dataset jenaDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("jenaOntModel"));
Dataset inferenceDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("inferenceOntModel"));
model = xutil.extractABox(jenaDataset,inferenceDataset,null);
}
else{
model = ModelContext.getInferenceOntModelSelector(getServletContext()).getABoxModel();
}
model = ModelContext.getInferenceOntModelSelector(
getServletContext()).getABoxModel();
}
else if("full".equals(assertedOrInferredParam)){
if(mode.equals("RDB")){
model = xutil.extractABox((OntModel)getServletContext().getAttribute("jenaOntModel"));
}
else{
model = ModelContext.getUnionOntModelSelector(getServletContext()).getABoxModel();
}
model = ModelContext.getUnionOntModelSelector(
getServletContext()).getABoxModel();
}
else if("asserted".equals(assertedOrInferredParam)){
if(mode.equals("RDB")){
Dataset jenaDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("jenaOntModel"));
Dataset baseDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("baseOntModel"));
model = xutil.extractABox(jenaDataset,baseDataset,null);
}
else{
model = ModelContext.getBaseOntModelSelector(getServletContext()).getABoxModel();
}
model = ModelContext.getBaseOntModelSelector(
getServletContext()).getABoxModel();
}
}
else if("tbox".equals(subgraphParam)){
@ -160,53 +144,35 @@ public class JenaExportController extends BaseEditController {
ModelContext.getBaseOntModelSelector(
getServletContext()).getTBoxModel(), ontologyURI);
}
// if("inferred".equals(assertedOrInferredParam)){
// model = xutil.extractTBox(dataset, ontologyURI,INFERENCE_GRAPH);
// }
// else if("full".equals(assertedOrInferredParam)){
// model = xutil.extractTBox(dataset, ontologyURI, FULL_GRAPH);
// }
// else{
// model = xutil.extractTBox(dataset, ontologyURI, ASSERTIONS_GRAPH);
// }
}
else if("full".equals(subgraphParam)){
if("inferred".equals(assertedOrInferredParam)){
ontModel = xutil.extractTBox(dataset, ontologyURI,INFERENCE_GRAPH);
if(mode.equals("RDB")){
Dataset jenaDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("jenaOntModel"));
Dataset inferenceDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("inferenceOntModel"));
ontModel.addSubModel(xutil.extractABox(jenaDataset, inferenceDataset, null));
}
else{
ontModel.addSubModel(ModelContext.getInferenceOntModelSelector(getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getInferenceOntModelSelector(getServletContext()).getTBoxModel());
}
ontModel = xutil.extractTBox(
dataset, ontologyURI, INFERENCE_GRAPH);
ontModel.addSubModel(ModelContext.getInferenceOntModelSelector(
getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getInferenceOntModelSelector(
getServletContext()).getTBoxModel());
}
else if("full".equals(assertedOrInferredParam)){
ontModel = xutil.extractTBox(dataset, ontologyURI, FULL_GRAPH);
if(mode.equals("RDB")){
ontModel.addSubModel(xutil.extractABox((OntModel)getServletContext().getAttribute("jenaOntModel")));
}
else{
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(getServletContext()).getTBoxModel());
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(getServletContext()).getApplicationMetadataModel());
}
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(
getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(
getServletContext()).getTBoxModel());
ontModel.addSubModel(ModelContext.getUnionOntModelSelector(
getServletContext()).getApplicationMetadataModel());
}
else{
ontModel = xutil.extractTBox(dataset, ontologyURI, ASSERTIONS_GRAPH);
if(mode.equals("RDB")){
Dataset jenaDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("jenaOntModel"));
Dataset baseDataset = DatasetFactory.create((OntModel)getServletContext().getAttribute("baseOntModel"));
ontModel.addSubModel(xutil.extractABox(jenaDataset,baseDataset,null));
}
else{
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(getServletContext()).getTBoxModel());
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(getServletContext()).getApplicationMetadataModel());
}
ontModel = xutil.extractTBox(
dataset, ontologyURI, ASSERTIONS_GRAPH);
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(
getServletContext()).getABoxModel());
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(
getServletContext()).getTBoxModel());
ontModel.addSubModel(ModelContext.getBaseOntModelSelector(
getServletContext()).getApplicationMetadataModel());
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/jena/JenaIngestController.java
View file

@ -126,7 +126,7 @@ public class JenaIngestController extends BaseEditController {
String modelT = (String)getServletContext().getAttribute("modelT");
String info = (String)getServletContext().getAttribute("info");
if(modelT == null){
boolean initialSwitch = JenaDataSourceSetupBase.isSDBActive(vreq);
boolean initialSwitch = true; // SDB mode initially
if(initialSwitch){
VitroJenaSDBModelMaker vsmm = (VitroJenaSDBModelMaker) getServletContext().getAttribute("vitroJenaSDBModelMaker");
vreq.getSession().setAttribute("vitroJenaModelMaker", vsmm);

6
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/WebappDaoFactorySDBPrep.java
View file

@ -61,10 +61,8 @@ public class WebappDaoFactorySDBPrep implements Filter {
public void doFilter(ServletRequest request, ServletResponse response,
FilterChain filterChain) throws IOException, ServletException {
if ( (!(JenaDataSourceSetupBase.isSDBActive(request))) ||
(request.getAttribute(
"WebappDaoFactorySDBPrep.setup") != null) ) {
// don't run multiple times or if SDB is not active
if ( request.getAttribute("WebappDaoFactorySDBPrep.setup") != null ) {
// don't run multiple times
filterChain.doFilter(request, response);
return;
}

21
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/AttachSubmodels.java
View file

@ -34,19 +34,18 @@ public class AttachSubmodels implements ServletContextListener {
if (AbortStartup.isStartupAborted(ctx)) {
return;
}
if (true) {
(new FileGraphSetup()).contextInitialized(sce);
return;
// use filegraphs instead of submodels if we're running SDB
}
// The future of AttachSubmodels is uncertain.
// Presently unreachable code follows.
try {
//FIXME refactor this
String tripleStoreTypeStr = ConfigurationProperties.getBean(sce)
.getProperty("VitroConnection.DataSource.tripleStoreType",
"RDB");
if ("SDB".equals(tripleStoreTypeStr)) {
(new FileGraphSetup()).contextInitialized(sce);
return;
// use filegraphs instead of submodels if we're running SDB
}
int attachmentCount = 0;
OntModel baseModel = (OntModel) ctx.getAttribute( JenaBaseDao.ASSERTIONS_ONT_MODEL_ATTRIBUTE_NAME );
Set<String> pathSet = ctx.getResourcePaths( PATH );

753
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetup.java
View file

@ -2,6 +2,8 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@ -9,17 +11,35 @@ import java.util.List;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.Individual;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.sdb.SDB;
import com.hp.hpl.jena.sdb.SDBFactory;
import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc;
import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.sdb.store.DatabaseType;
import com.hp.hpl.jena.sdb.store.LayoutType;
import com.hp.hpl.jena.sdb.util.StoreUtils;
import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.util.iterator.ClosableIterator;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
@ -28,116 +48,45 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDaoCon;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaModelUtils;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.dao.jena.SimpleOntModelSelector;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelSynchronizer;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelector;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelectorImpl;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaModelMaker;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaSDBModelMaker;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactoryJena;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactorySDB;
import edu.cornell.mannlib.vitro.webapp.ontology.update.KnowledgeBaseUpdater;
import edu.cornell.mannlib.vitro.webapp.utils.NamespaceMapper;
import edu.cornell.mannlib.vitro.webapp.utils.jena.InitialJenaModelUtils;
import edu.cornell.mannlib.vitro.webapp.utils.jena.NamespaceMapperJena;
public class JenaDataSourceSetup extends JenaDataSourceSetupBase implements javax.servlet.ServletContextListener {
private static final Log log = LogFactory.getLog(JenaDataSourceSetup.class.getName());
private static final Log log = LogFactory.getLog(JenaDataSourceSetup.class);
@Override
public void contextInitialized(ServletContextEvent sce) {
ServletContext ctx = sce.getServletContext();
String tripleStoreTypeStr =
ConfigurationProperties.getBean(sce).getProperty(
"VitroConnection.DataSource.tripleStoreType", "RDB");
if ("SDB".equals(tripleStoreTypeStr)) {
(new JenaDataSourceSetupSDB()).contextInitialized(sce);
return;
}
if (AbortStartup.isStartupAborted(ctx)) {
return;
}
try {
OntModel memModel = (OntModel) ctx.getAttribute("jenaOntModel");
if (memModel == null) {
memModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
log.warn("WARNING: no database connected. Changes will disappear after context restart.");
ctx.setAttribute("jenaOntModel",memModel);
}
memModel.addSubModel((new JenaBaseDaoCon()).getConstModel()); // add the vitro tbox to the model
OntModel inferenceModel = ontModelFromContextAttribute(ctx, "inferenceOntModel");
OntModel userAccountsModel = ontModelFromContextAttribute(ctx, "userAccountsOntModel");
if (userAccountsModel.size() == 0) {
checkMainModelForUserAccounts(memModel, userAccountsModel);
}
OntModel unionModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,ModelFactory.createUnion(memModel, inferenceModel));
SimpleOntModelSelector baseOms = new SimpleOntModelSelector(memModel);
SimpleOntModelSelector inferenceOms = new SimpleOntModelSelector(inferenceModel);
SimpleOntModelSelector unionOms = new SimpleOntModelSelector(unionModel);
baseOms.setUserAccountsModel(userAccountsModel);
inferenceOms.setUserAccountsModel(userAccountsModel);
unionOms.setUserAccountsModel(userAccountsModel);
OntModel displayModel = ontModelFromContextAttribute(ctx,"displayOntModel");
baseOms.setDisplayModel(displayModel);
inferenceOms.setDisplayModel(displayModel);
unionOms.setDisplayModel(displayModel);
checkForNamespaceMismatch( memModel, defaultNamespace, sce );
ctx.setAttribute("baseOntModel", memModel);
WebappDaoFactory baseWadf = new WebappDaoFactoryJena(
baseOms, defaultNamespace, null, null);
ctx.setAttribute("assertionsWebappDaoFactory",baseWadf);
ModelContext.setBaseOntModelSelector(baseOms, ctx);
ctx.setAttribute("inferenceOntModel", inferenceModel);
WebappDaoFactory infWadf = new WebappDaoFactoryJena(
inferenceOms, defaultNamespace, null, null);
ctx.setAttribute("deductionsWebappDaoFactory", infWadf);
ModelContext.setInferenceOntModelSelector(inferenceOms, ctx);
ctx.setAttribute("jenaOntModel", unionModel);
WebappDaoFactory wadf = new WebappDaoFactoryJena(
unionOms, baseOms, inferenceOms, defaultNamespace, null, null);
ctx.setAttribute("webappDaoFactory",wadf);
ModelContext.setUnionOntModelSelector(unionOms, ctx);
ApplicationBean appBean = wadf.getApplicationDao().getApplicationBean();
if (appBean != null) {
ctx.setAttribute("applicationBean", appBean);
}
if (isEmpty(memModel)) {
loadDataFromFilesystem(memModel, ctx);
}
if (userAccountsModel.size() == 0) {
readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsModel);
if (userAccountsModel.size() == 0) {
createInitialAdminUser(userAccountsModel, ctx);
}
}
ensureEssentialInterfaceData(memModel, sce, wadf);
NamespaceMapper namespaceMapper = new NamespaceMapperJena(unionModel, unionModel, defaultNamespace);
ctx.setAttribute("NamespaceMapper", namespaceMapper);
memModel.getBaseModel().register(namespaceMapper);
makeModelMakerFromConnectionProperties(TripleStoreType.RDB, ctx);
VitroJenaModelMaker vjmm = getVitroJenaModelMaker();
setVitroJenaModelMaker(vjmm,sce);
makeModelMakerFromConnectionProperties(TripleStoreType.SDB, ctx);
VitroJenaSDBModelMaker vsmm = getVitroJenaSDBModelMaker();
setVitroJenaSDBModelMaker(vsmm,sce);
long startTime = System.currentTimeMillis();
setUpJenaDataSource(ctx);
log.info((System.currentTimeMillis() - startTime) / 1000 +
" seconds to set up SDB store");
} catch (MigrationRequiredError mre) {
throw new MigrationRequiredError(mre.getMessage());
} catch (SQLException sqle) {
// SQL exceptions are fatal and should halt startup
AbortStartup.abortStartup(ctx);
log.error("Error using SQL database; startup aborted.", sqle);
// print to catalina.out for good measure
System.out.println("Error using SQL database; startup aborted.");
sqle.printStackTrace();
throw new Error(this.getClass().getName() + "failed");
} catch (Throwable t) {
log.error("Throwable in " + this.getClass().getName(), t);
// printing the error because Tomcat doesn't print context listener
@ -145,36 +94,254 @@ public class JenaDataSourceSetup extends JenaDataSourceSetupBase implements java
t.printStackTrace();
throw new Error(this.getClass().getName() + "failed");
}
}
private void setUpJenaDataSource(ServletContext ctx) throws SQLException {
/* commenting out during development of 1.3 - this will be redone.
if ( updateRequired(ctx, memModel)) {
log.error(getMigrationErrString());
System.out.println(getMigrationErrString());
// The rest of the application should not
// start if this condition is encountered
AbortStartup.abortStartup(ctx);
throw new MigrationRequiredError(getMigrationErrString());
}
*/
OntModelSelectorImpl baseOms = new OntModelSelectorImpl();
OntModelSelectorImpl inferenceOms = new OntModelSelectorImpl();
OntModelSelectorImpl unionOms = new OntModelSelectorImpl();
OntModel userAccountsModel = ontModelFromContextAttribute(
ctx, "userAccountsOntModel");
baseOms.setUserAccountsModel(userAccountsModel);
inferenceOms.setUserAccountsModel(userAccountsModel);
unionOms.setUserAccountsModel(userAccountsModel);
OntModel displayModel = ontModelFromContextAttribute(
ctx,"displayOntModel");
baseOms.setDisplayModel(displayModel);
inferenceOms.setDisplayModel(displayModel);
unionOms.setDisplayModel(displayModel);
// SDB setup
// union default graph
SDB.getContext().set(SDB.unionDefaultGraph, true) ;
StoreDesc storeDesc = makeStoreDesc(ctx);
setApplicationStoreDesc(storeDesc, ctx);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx);
this.setApplicationDataSource(bds, ctx);
Store store = connectStore(bds, storeDesc);
setApplicationStore(store, ctx);
if (!isSetUp(store)) {
log.info("Initializing SDB store");
if (isFirstStartup()) {
setupSDB(ctx, store);
} else {
migrateToSDBFromExistingRDBStore(ctx, store);
}
}
// The code below, which sets up the OntModelSelectors, controls whether each
// model is maintained in memory, in the DB, or both while the application
// is running.
// Populate the three OntModelSelectors (BaseOntModel=assertions, InferenceOntModel=inferences
// and JenaOntModel=union of assertions and inferences) with the post-SDB-conversion models.
// ABox assertions
Model aboxAssertions = makeDBModel(bds, JenaDataSourceSetupBase.JENA_DB_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
Model listenableAboxAssertions = ModelFactory.createUnion(aboxAssertions, ModelFactory.createDefaultModel());
baseOms.setABoxModel(ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, listenableAboxAssertions));
// ABox inferences
Model aboxInferences = makeDBModel(bds, JenaDataSourceSetupBase.JENA_INF_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
Model listenableAboxInferences = ModelFactory.createUnion(aboxInferences, ModelFactory.createDefaultModel());
inferenceOms.setABoxModel(ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, listenableAboxInferences));
// Since the TBox models are in memory, they do not have time out issues like the
// ABox models do (and so don't need the extra step to make them listenable).
// TBox assertions
try {
Model tboxAssertionsDB = makeDBModel(bds, JENA_TBOX_ASSERTIONS_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel tboxAssertions = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
if (tboxAssertionsDB != null) {
long startTime = System.currentTimeMillis();
System.out.println("Copying cached tbox assertions into memory");
tboxAssertions.add(tboxAssertionsDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load tbox assertions");
}
tboxAssertions.getBaseModel().register(new ModelSynchronizer(tboxAssertionsDB));
baseOms.setTBoxModel(tboxAssertions);
} catch (Throwable e) {
log.error("Unable to load tbox assertion cache from DB", e);
}
// TBox inferences
try {
Model tboxInferencesDB = makeDBModel(bds, JENA_TBOX_INF_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel tboxInferences = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
if (tboxInferencesDB != null) {
long startTime = System.currentTimeMillis();
System.out.println("Copying cached tbox inferences into memory");
tboxInferences.add(tboxInferencesDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load tbox inferences");
}
tboxInferences.getBaseModel().register(new ModelSynchronizer(tboxInferencesDB));
inferenceOms.setTBoxModel(tboxInferences);
} catch (Throwable e) {
log.error("Unable to load tbox inference cache from DB", e);
}
// union ABox
OntModel unionABoxModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,ModelFactory.createUnion(baseOms.getABoxModel(), inferenceOms.getABoxModel()));
unionOms.setABoxModel(unionABoxModel);
// union TBox
OntModel unionTBoxModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,ModelFactory.createUnion(baseOms.getTBoxModel(), inferenceOms.getTBoxModel()));
unionOms.setTBoxModel(unionTBoxModel);
// Application metadata model is cached in memory.
try {
Model applicationMetadataModelDB = makeDBModel(bds, JENA_APPLICATION_METADATA_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel applicationMetadataModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
long startTime = System.currentTimeMillis();
System.out.println("Copying cached application metadata model into memory");
applicationMetadataModel.add(applicationMetadataModelDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load application metadata model assertions of size " + applicationMetadataModel.size());
applicationMetadataModel.getBaseModel().register(new ModelSynchronizer(applicationMetadataModelDB));
if (isFirstStartup()) {
applicationMetadataModel.add(InitialJenaModelUtils.loadInitialModel(ctx, defaultNamespace));
} else if (applicationMetadataModelDB.size() == 0) {
repairAppMetadataModel(applicationMetadataModel, aboxAssertions, aboxInferences);
}
baseOms.setApplicationMetadataModel(applicationMetadataModel);
inferenceOms.setApplicationMetadataModel(baseOms.getApplicationMetadataModel());
unionOms.setApplicationMetadataModel(baseOms.getApplicationMetadataModel());
} catch (Throwable e) {
log.error("Unable to load application metadata model cache from DB", e);
}
checkForNamespaceMismatch( baseOms.getApplicationMetadataModel(), defaultNamespace, ctx );
if (isFirstStartup()) {
loadDataFromFilesystem(baseOms, ctx);
}
log.info("Adding vitro application ontology");
// add the vitroontologies to the tbox models
OntModel vitroTBoxModel = (new JenaBaseDaoCon()).getConstModel();
baseOms.getTBoxModel().addSubModel(vitroTBoxModel);
inferenceOms.getTBoxModel().addSubModel(vitroTBoxModel);
unionOms.getTBoxModel().addSubModel(vitroTBoxModel);
log.info("Setting up union models and DAO factories");
// create TBox + ABox union models and set up webapp DAO factories
OntModel baseUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(baseOms.getABoxModel(), baseOms.getTBoxModel()));
baseOms.setFullModel(baseUnion);
ModelContext.setBaseOntModel(baseOms.getFullModel(), ctx);
WebappDaoFactory baseWadf = new WebappDaoFactorySDB(
baseOms,
bds,
storeDesc,
defaultNamespace,
null,
null,
WebappDaoFactorySDB.SDBDatasetMode.ASSERTIONS_ONLY);
ctx.setAttribute("assertionsWebappDaoFactory",baseWadf);
OntModel inferenceUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(inferenceOms.getABoxModel(), inferenceOms.getTBoxModel()));
inferenceOms.setFullModel(inferenceUnion);
ModelContext.setInferenceOntModel(inferenceOms.getFullModel(), ctx);
WebappDaoFactory infWadf = new WebappDaoFactorySDB(
inferenceOms,
bds,
storeDesc,
defaultNamespace,
null,
null,
WebappDaoFactorySDB.SDBDatasetMode.INFERENCES_ONLY);
ctx.setAttribute("deductionsWebappDaoFactory", infWadf);
OntModel masterUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(unionABoxModel, unionTBoxModel));
unionOms.setFullModel(masterUnion);
ctx.setAttribute("jenaOntModel", masterUnion);
WebappDaoFactory wadf = new WebappDaoFactorySDB(unionOms, bds, storeDesc, defaultNamespace, null, null);
ctx.setAttribute("webappDaoFactory",wadf);
ModelContext.setUnionOntModelSelector(unionOms, ctx); // assertions and inferences
ModelContext.setBaseOntModelSelector(baseOms, ctx); // assertions
ModelContext.setInferenceOntModelSelector(inferenceOms, ctx); // inferences
//log.info("Setting up namespace mapper");
//NamespaceMapper namespaceMapper = new NamespaceMapperJena(masterUnion, masterUnion, defaultNamespace);
//ctx.setAttribute("NamespaceMapper", namespaceMapper);
//unionOms.getFullModel().getBaseModel().register(namespaceMapper);
ctx.setAttribute("defaultNamespace", defaultNamespace);
log.info("SDB store ready for use");
makeModelMakerFromConnectionProperties(TripleStoreType.RDB, ctx);
VitroJenaModelMaker vjmm = getVitroJenaModelMaker();
setVitroJenaModelMaker(vjmm, ctx);
makeModelMakerFromConnectionProperties(TripleStoreType.SDB, ctx);
VitroJenaSDBModelMaker vsmm = getVitroJenaSDBModelMaker();
setVitroJenaSDBModelMaker(vsmm, ctx);
log.info("Model makers set up");
}
private void checkForNamespaceMismatch(OntModel model, String defaultNamespace, ServletContextEvent sce) {
String defaultNamespaceFromDeployProperties = ConfigurationProperties
.getBean(sce).getProperty("Vitro.defaultNamespace");
if( defaultNamespaceFromDeployProperties == null ){
private void checkForNamespaceMismatch(OntModel model, String defaultNamespace, ServletContext ctx) {
String defaultNamespaceFromDeployProperites = ConfigurationProperties
.getBean(ctx).getProperty("Vitro.defaultNamespace");
if( defaultNamespaceFromDeployProperites == null ){
log.error("Could not get namespace from deploy.properties.");
}
List<String> portalURIs = new ArrayList<String>();
try {
model.enterCriticalSection(Lock.READ);
Iterator portalIt = model.listIndividuals(PORTAL);
Iterator<Individual> portalIt = model.listIndividuals(PORTAL);
while (portalIt.hasNext()) {
portalURIs.add( ((Individual)portalIt.next()).getURI() );
portalURIs.add( portalIt.next().getURI() );
}
} finally {
model.leaveCriticalSection();
}
if( portalURIs.size() > 0 ){
for( String portalUri : portalURIs){
if( portalUri != null && ! portalUri.startsWith(defaultNamespaceFromDeployProperties)){
if( portalUri != null && ! portalUri.startsWith(defaultNamespaceFromDeployProperites)){
log.error("Namespace mismatch between db and deploy.properties.");
String portalNamespace = portalUri.substring(0, portalUri.lastIndexOf("/")+1);
log.error("Vivo will not start up correctly because the default namespace specified in deploy.properties does not match the namespace of " +
"a portal in the database. Namespace from deploy.properties: \"" + defaultNamespaceFromDeployProperties +
"\". Namespace from an existing portal: \"" + portalNamespace + "\". To get the application to start with this " +
"database, change the default namespace in deploy.properties to \"" + portalNamespace +
"\". Another possibility is that deploy.properties does not specify the intended database.");
"a portal in the database. Namespace from deploy.properties: \"" + defaultNamespaceFromDeployProperites +
"\" Namespace from an existing portal: \"" + portalUri + "\" To get the application to start with this " +
"database change the default namespace in deploy.properties " + portalUri.substring(0, portalUri.lastIndexOf("/")+1) +
" Another possibility is that deploy.properties does not specify the intended database.");
}
}
}
@ -186,88 +353,318 @@ public class JenaDataSourceSetup extends JenaDataSourceSetupBase implements java
@Override
public void contextDestroyed(ServletContextEvent sce) {
// Nothing to do
}
private void ensureEssentialInterfaceData(OntModel memModel, ServletContextEvent sce, WebappDaoFactory wadf) {
Model essentialInterfaceData = null;
ClosableIterator portalIt = memModel.listIndividuals(memModel.getResource(VitroVocabulary.PORTAL));
try {
if (!portalIt.hasNext()) {
log.debug("Loading initial site configuration");
essentialInterfaceData = InitialJenaModelUtils.loadInitialModel(sce.getServletContext(), defaultNamespace);
if (essentialInterfaceData.size() == 0) {
essentialInterfaceData = InitialJenaModelUtils.basicInterfaceData(defaultNamespace);
essentialInterfaceData.add(InitialJenaModelUtils.basicClassgroup(wadf.getDefaultNamespace()));
}
//JenaModelUtils.makeClassGroupsFromRootClasses(wadf,memModel,essentialInterfaceData);
memModel.add(essentialInterfaceData);
} else {
//Set the default namespace to the namespace of the first portal object we find.
//This will keep existing applications from dying when the default namespace
//config option is missing.
Individual portal = (Individual) portalIt.next();
if (portal.getNameSpace() != null) {
defaultNamespace = portal.getNameSpace();
}
}
} finally {
portalIt.close();
}
// Nothing to do.
}
private void checkMainModelForUserAccounts(OntModel mainModel, OntModel userAccountsModel) {
Model extractedUserData = ((new JenaModelUtils()).extractUserAccountsData(mainModel));
if (extractedUserData.size() > 0) {
userAccountsModel.enterCriticalSection(Lock.WRITE);
try {
userAccountsModel.add(extractedUserData);
} finally {
userAccountsModel.leaveCriticalSection();
}
mainModel.enterCriticalSection(Lock.WRITE);
try {
mainModel.remove(extractedUserData);
} finally {
mainModel.leaveCriticalSection();
}
}
Model extractedUserData = ((new JenaModelUtils()).extractUserAccountsData(mainModel));
if (extractedUserData.size() > 0) {
userAccountsModel.enterCriticalSection(Lock.WRITE);
try {
userAccountsModel.add(extractedUserData);
} finally {
userAccountsModel.leaveCriticalSection();
}
mainModel.enterCriticalSection(Lock.WRITE);
try {
mainModel.remove(extractedUserData);
} finally {
mainModel.leaveCriticalSection();
}
}
}
private OntModel ontModelFromContextAttribute(ServletContext ctx, String attribute) {
OntModel ontModel;
Object attributeValue = ctx.getAttribute(attribute);
if (attributeValue != null && attributeValue instanceof OntModel) {
ontModel = (OntModel) attributeValue;
} else {
ontModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
ctx.setAttribute(attribute, ontModel);
}
return ontModel;
OntModel ontModel;
Object attributeValue = ctx.getAttribute(attribute);
if (attributeValue != null && attributeValue instanceof OntModel) {
ontModel = (OntModel) attributeValue;
} else {
ontModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
ctx.setAttribute(attribute, ontModel);
}
return ontModel;
}
private boolean isEmpty(Model model) {
ClosableIterator closeIt = model.listStatements();
try {
if (closeIt.hasNext()) {
return false;
} else {
return true;
}
} finally {
closeIt.close();
}
ClosableIterator<Statement> closeIt = model.listStatements(
null, RDF.type, ResourceFactory.createResource(
VitroVocabulary.PORTAL));
try {
if (closeIt.hasNext()) {
return false;
} else {
return true;
}
} finally {
closeIt.close();
}
}
private void loadDataFromFilesystem(OntModel ontModel, ServletContext ctx) {
OntModel initialDataModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
Long startTime = System.currentTimeMillis();
log.debug("Reading ontology files");
readOntologyFilesInPathSet(USERPATH, ctx, initialDataModel);
readOntologyFilesInPathSet(SYSTEMPATH, ctx, initialDataModel);
log.debug(((System.currentTimeMillis()-startTime)/1000)+" seconds to read ontology files ");
ontModel.add(initialDataModel);
private void loadDataFromFilesystem(OntModelSelector baseOms, ServletContext ctx) {
Long startTime = System.currentTimeMillis();
log.debug("Initializing models from RDF files");
readOntologyFilesInPathSet(USER_ABOX_PATH, ctx, baseOms.getABoxModel());
readOntologyFilesInPathSet(USER_TBOX_PATH, ctx, baseOms.getTBoxModel());
readOntologyFilesInPathSet(
USER_APPMETA_PATH, ctx, baseOms.getApplicationMetadataModel());
//readOntologyFilesInPathSet(SYSTEMPATH, ctx, initialDataModel);
log.debug(((System.currentTimeMillis()-startTime)/1000)
+ " seconds to read RDF files ");
}
}
private static void getTBoxModel(Model fullModel, Model submodels, Model tboxModel) {
JenaModelUtils modelUtils = new JenaModelUtils();
Model tempModel = ModelFactory.createUnion(fullModel, submodels);
Model tempTBoxModel = modelUtils.extractTBox(tempModel);
// copy intersection of tempTBoxModel and fullModel to tboxModel.
StmtIterator iter = tempTBoxModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.next();
if (fullModel.contains(stmt)) {
tboxModel.add(stmt);
}
}
return;
}
/*
* Copy all statements from model 1 that are not in model 2 to model 3.
*/
private static void copyDifference(Model model1, Model model2, Model model3) {
StmtIterator iter = model1.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.next();
if (!model2.contains(stmt)) {
model3.add(stmt);
}
}
return;
}
private static void getAppMetadata(Model source, Model target) {
String amdQuery = "DESCRIBE ?x WHERE { " +
"{?x a <" + VitroVocabulary.PORTAL +"> } UNION " +
"{?x a <" + VitroVocabulary.PROPERTYGROUP +"> } UNION " +
"{?x a <" + VitroVocabulary.CLASSGROUP +"> } } ";
try {
Query q = QueryFactory.create(amdQuery, Syntax.syntaxARQ);
QueryExecution qe = QueryExecutionFactory.create(q, source);
qe.execDescribe(target);
} catch (Exception e) {
log.error("unable to create the application metadata model",e);
}
return;
}
private static void repairAppMetadataModel(Model applicationMetadataModel,
Model aboxAssertions,
Model aboxInferences) {
log.info("Moving application metadata from ABox to dedicated model");
getAppMetadata(aboxAssertions, applicationMetadataModel);
getAppMetadata(aboxInferences, applicationMetadataModel);
aboxAssertions.remove(applicationMetadataModel);
aboxInferences.remove(applicationMetadataModel);
return;
}
public static StoreDesc makeStoreDesc(ServletContext ctx) {
String layoutStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.sdb.layout", "layout2/hash");
String dbtypeStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.dbtype", "MySQL");
return new StoreDesc(
LayoutType.fetch(layoutStr),
DatabaseType.fetch(dbtypeStr) );
}
public static Store connectStore(BasicDataSource bds, StoreDesc storeDesc)
throws SQLException {
SDBConnection conn = new SDBConnection(bds.getConnection()) ;
return SDBFactory.connectStore(conn, storeDesc);
}
public static void setupSDB(ServletContext ctx, Store store) {
setupSDB(ctx,
store,
ModelFactory.createDefaultModel(),
ModelFactory.createDefaultModel());
}
public static void setupSDB(ServletContext ctx,
Store store,
Model memModel,
Model inferenceModel) {
store.getTableFormatter().create();
store.getTableFormatter().truncate();
store.getTableFormatter().dropIndexes(); // improve load performance
try {
// This is a one-time copy of stored KB data - from a Jena RDB store
// to a Jena SDB store. In the process, we will also separate out the
// TBox from the Abox; these are in the same graph in pre 1.2 VIVO
// versions and will now be stored and maintained in separate models
// Access to the Jena RDB data is through the OntModelSelectors that have
// been set up earlier in the current session by
// JenaPersistentDataSourceSetup.java
// In the code below, note that the current getABoxModel() methods on
// the OntModelSelectors return a graph with both ABox and TBox data.
OntModel submodels = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
readOntologyFilesInPathSet(SUBMODELS, ctx, submodels);
Model tboxAssertions = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
// initially putting the results in memory so we have a
// cheaper way of computing the difference when we copy the ABox
Model memTboxAssertions = ModelFactory.createDefaultModel();
getTBoxModel(memModel, submodels, memTboxAssertions);
tboxAssertions.add(memTboxAssertions);
Model tboxInferences = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_TBOX_INF_MODEL);
// initially putting the results in memory so we have a
// cheaper way of computing the difference when we copy the ABox
Model memTboxInferences = ModelFactory.createDefaultModel();
getTBoxModel(inferenceModel, submodels, memTboxInferences);
tboxInferences.add(memTboxInferences);
Model aboxAssertions = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_DB_MODEL);
copyDifference(memModel, memTboxAssertions, aboxAssertions);
Model aboxInferences = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_INF_MODEL);
copyDifference(inferenceModel, memTboxInferences, aboxInferences);
// Set up the application metadata model
Model applicationMetadataModel = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_APPLICATION_METADATA_MODEL);
getAppMetadata(memModel, applicationMetadataModel);
log.info("During initial SDB setup, created an application metadata model of size "
+ applicationMetadataModel.size());
// remove application metadata from ABox model
aboxAssertions.remove(applicationMetadataModel);
aboxInferences.remove(applicationMetadataModel);
// Make sure the reasoner takes into account the newly-set-up data.
SimpleReasonerSetup.setRecomputeRequired(ctx);
} finally {
log.info("Adding indexes to SDB database tables.");
store.getTableFormatter().addIndexes();
log.info("Indexes created.");
}
}
private void migrateToSDBFromExistingRDBStore(ServletContext ctx,
Store store) {
Model rdbAssertionsModel = makeDBModelFromConfigurationProperties(
JENA_DB_MODEL, DB_ONT_MODEL_SPEC, ctx);
Model rdbInferencesModel = makeDBModelFromConfigurationProperties(
JENA_INF_MODEL, DB_ONT_MODEL_SPEC, ctx);
setupSDB(ctx, store, rdbAssertionsModel, rdbInferencesModel);
}
/**
* Tests whether an SDB store has been formatted and populated for use.
* @param store
* @return
*/
private boolean isSetUp(Store store) throws SQLException {
if (!(StoreUtils.isFormatted(store))) {
return false;
}
// even if the store exists, it may be empty
try {
return (SDBFactory.connectNamedModel(store, JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL)).size() > 0;
} catch (Exception e) {
return false;
}
}
private static final String STOREDESC_ATTR = "storeDesc";
private static final String STORE_ATTR = "kbStore";
public static void setApplicationStoreDesc(StoreDesc storeDesc,
ServletContext ctx) {
ctx.setAttribute(STOREDESC_ATTR, storeDesc);
}
public static StoreDesc getApplicationStoreDesc(ServletContext ctx) {
return (StoreDesc) ctx.getAttribute(STOREDESC_ATTR);
}
public static void setApplicationStore(Store store,
ServletContext ctx) {
ctx.setAttribute(STORE_ATTR, store);
}
public static Store getApplicationStore(ServletContext ctx) {
return (Store) ctx.getAttribute(STORE_ATTR);
}
/**
* Executes a SPARQL ASK query to determine whether the knowledge base
* needs to be updated to conform to a new ontology version
*/
public boolean updateRequired(ServletContext ctx, OntModel m) throws IOException {
boolean required = false;
String sparqlQueryStr = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskQueryPath(ctx));
if (sparqlQueryStr == null) {
return required;
}
Query query = QueryFactory.create(sparqlQueryStr);
QueryExecution isUpdated = QueryExecutionFactory.create(query, m);
// if the ASK query DOES have a solution (i.e. the assertions exist
// showing that the update has already been performed), then the update
// is NOT required.
if (isUpdated.execAsk()) {
required = false;
} else {
required = true;
String sparqlQueryStr2 = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskEmptyQueryPath(ctx));
if (sparqlQueryStr2 != null) {
Query query2 = QueryFactory.create(sparqlQueryStr2);
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, m);
required = isNotEmpty.execAsk();
}
}
return required;
}
private class MigrationRequiredError extends Error {
public MigrationRequiredError(String string) {
super(string);
}
}
}

32
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -7,7 +7,6 @@ import java.sql.SQLException;
import java.util.Set;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletRequest;
import javax.servlet.http.HttpServletRequest;
@ -51,10 +50,14 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
protected static String BASE = "/WEB-INF/ontologies/";
protected static String USERPATH = BASE+"user/";
protected static String USER_ABOX_PATH = BASE+"user/abox";
protected static String USER_TBOX_PATH = BASE+"user/tbox";
protected static String USER_APPMETA_PATH = BASE+"user/applicationMetadata";
protected static String SYSTEMPATH = BASE+"system/";
protected static String AUTHPATH = BASE+"auth/";
public static String APPPATH = BASE+"app/";
protected static String SUBMODELS = "/WEB-INF/submodels/";
protected static boolean firstStartup = false;
String DB_USER = "jenatest"; // database user id
String DB_PASSWD = "jenatest"; // database password
@ -250,6 +253,10 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
RDB, SDB
}
protected boolean isFirstStartup() {
return firstStartup;
}
protected Model makeDBModel(BasicDataSource ds,
String jenaDbModelname,
OntModelSpec jenaDbOntModelSpec,
@ -410,28 +417,13 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
}
public static void setVitroJenaModelMaker(VitroJenaModelMaker vjmm,
ServletContextEvent sce){
sce.getServletContext().setAttribute(rdbModelMaker, vjmm);
ServletContext ctx){
ctx.setAttribute(rdbModelMaker, vjmm);
}
public static void setVitroJenaSDBModelMaker(VitroJenaSDBModelMaker vsmm,
ServletContextEvent sce){
sce.getServletContext().setAttribute(sdbModelMaker, vsmm);
}
public static boolean isSDBActive(ServletRequest req) {
if (!(req instanceof HttpServletRequest)) {
return false;
}
HttpServletRequest hreq = (HttpServletRequest) req;
return isSDBActive(hreq.getSession().getServletContext());
}
public static boolean isSDBActive(ServletContext ctx) {
String tripleStoreTypeStr =
ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.tripleStoreType", "RDB");
return ("SDB".equals(tripleStoreTypeStr));
ServletContext ctx){
ctx.setAttribute(sdbModelMaker, vsmm);
}
protected VitroJenaModelMaker getVitroJenaModelMaker(){

721
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupSDB.java
View file

@ -1,721 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import org.apache.commons.dbcp.BasicDataSource;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.Individual;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.sdb.SDB;
import com.hp.hpl.jena.sdb.SDBFactory;
import com.hp.hpl.jena.sdb.Store;
import com.hp.hpl.jena.sdb.StoreDesc;
import com.hp.hpl.jena.sdb.sql.SDBConnection;
import com.hp.hpl.jena.sdb.store.DatabaseType;
import com.hp.hpl.jena.sdb.store.LayoutType;
import com.hp.hpl.jena.sdb.util.StoreUtils;
import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.util.iterator.ClosableIterator;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaBaseDaoCon;
import edu.cornell.mannlib.vitro.webapp.dao.jena.JenaModelUtils;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelSynchronizer;
import edu.cornell.mannlib.vitro.webapp.dao.jena.OntModelSelectorImpl;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaModelMaker;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VitroJenaSDBModelMaker;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactorySDB;
import edu.cornell.mannlib.vitro.webapp.ontology.update.KnowledgeBaseUpdater;
import edu.cornell.mannlib.vitro.webapp.utils.NamespaceMapper;
import edu.cornell.mannlib.vitro.webapp.utils.jena.InitialJenaModelUtils;
import edu.cornell.mannlib.vitro.webapp.utils.jena.NamespaceMapperJena;
public class JenaDataSourceSetupSDB extends JenaDataSourceSetupBase implements javax.servlet.ServletContextListener {
private static final Log log = LogFactory.getLog(JenaDataSourceSetupSDB.class);
@Override
public void contextInitialized(ServletContextEvent sce) {
ServletContext ctx = sce.getServletContext();
if (AbortStartup.isStartupAborted(ctx)) {
return;
}
try {
// JenaPersistentDataSourceSetup should have already set this up - it just sets
// up things related to the DB.
// TODO: I would like to make this code (before the sdb try/catch conditional so
// that it is not executed in a post-sdb-conversion environment.
OntModel memModel = (OntModel) ctx.getAttribute("jenaOntModel");
//memModel.writeAll(System.out,"N3",null);
/* commenting out during development of 1.3 - this will be redone.
if ( updateRequired(ctx, memModel)) {
log.error(getMigrationErrString());
System.out.println(getMigrationErrString());
// The rest of the application should not
// start if this condition is encountered
AbortStartup.abortStartup(ctx);
throw new MigrationRequiredError(getMigrationErrString());
}
*/
if (memModel == null) {
memModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
log.warn("WARNING: no database connected. Changes will disappear after context restart.");
ctx.setAttribute("jenaOntModel",memModel);
}
OntModel inferenceModel = ontModelFromContextAttribute(ctx, "inferenceOntModel");
OntModel userAccountsModel = ontModelFromContextAttribute(ctx, "userAccountsOntModel");
if (userAccountsModel.size() == 0) {
checkMainModelForUserAccounts(memModel, userAccountsModel);
}
OntModelSelectorImpl baseOms = new OntModelSelectorImpl();
OntModelSelectorImpl inferenceOms = new OntModelSelectorImpl();
OntModelSelectorImpl unionOms = new OntModelSelectorImpl();
baseOms.setUserAccountsModel(userAccountsModel);
inferenceOms.setUserAccountsModel(userAccountsModel);
unionOms.setUserAccountsModel(userAccountsModel);
OntModel displayModel = ontModelFromContextAttribute(ctx,"displayOntModel");
baseOms.setDisplayModel(displayModel);
inferenceOms.setDisplayModel(displayModel);
unionOms.setDisplayModel(displayModel);
checkForNamespaceMismatch( memModel, defaultNamespace, sce );
// SDB setup
// union default graph
SDB.getContext().set(SDB.unionDefaultGraph, true) ;
StoreDesc storeDesc = makeStoreDesc(ctx);
setApplicationStoreDesc(storeDesc, ctx);
BasicDataSource bds = makeDataSourceFromConfigurationProperties(ctx);
this.setApplicationDataSource(bds, ctx);
Store store = connectStore(bds, storeDesc);
setApplicationStore(store, ctx);
if (!isSetUp(store)) {
log.info("Non-SDB system detected. Setting up SDB store");
setupSDB(ctx, store, memModel, inferenceModel);
}
// The code below, which sets up the OntModelSelectors, controls whether each
// model is maintained in memory, in the DB, or both while the application
// is running.
// Populate the three OntModelSelectors (BaseOntModel=assertions, InferenceOntModel=inferences
// and JenaOntModel=union of assertions and inferences) with the post-SDB-conversion models.
// ABox assertions
Model aboxAssertions = makeDBModel(bds, JenaDataSourceSetupBase.JENA_DB_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
Model listenableAboxAssertions = ModelFactory.createUnion(aboxAssertions, ModelFactory.createDefaultModel());
baseOms.setABoxModel(ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, listenableAboxAssertions));
// ABox inferences
Model aboxInferences = makeDBModel(bds, JenaDataSourceSetupBase.JENA_INF_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
Model listenableAboxInferences = ModelFactory.createUnion(aboxInferences, ModelFactory.createDefaultModel());
inferenceOms.setABoxModel(ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM, listenableAboxInferences));
// Since the TBox models are in memory, they do not have time out issues like the
// ABox models do (and so don't need the extra step to make them listenable).
// TBox assertions
try {
Model tboxAssertionsDB = makeDBModel(bds, JENA_TBOX_ASSERTIONS_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel tboxAssertions = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
if (tboxAssertionsDB != null) {
long startTime = System.currentTimeMillis();
System.out.println("Copying cached tbox assertions into memory");
tboxAssertions.add(tboxAssertionsDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load tbox assertions");
}
tboxAssertions.getBaseModel().register(new ModelSynchronizer(tboxAssertionsDB));
baseOms.setTBoxModel(tboxAssertions);
} catch (Throwable e) {
log.error("Unable to load tbox assertion cache from DB", e);
}
// TBox inferences
try {
Model tboxInferencesDB = makeDBModel(bds, JENA_TBOX_INF_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel tboxInferences = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
if (tboxInferencesDB != null) {
long startTime = System.currentTimeMillis();
System.out.println("Copying cached tbox inferences into memory");
tboxInferences.add(tboxInferencesDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load tbox inferences");
}
tboxInferences.getBaseModel().register(new ModelSynchronizer(tboxInferencesDB));
inferenceOms.setTBoxModel(tboxInferences);
} catch (Throwable e) {
log.error("Unable to load tbox inference cache from DB", e);
}
// union ABox
OntModel unionABoxModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,ModelFactory.createUnion(baseOms.getABoxModel(), inferenceOms.getABoxModel()));
unionOms.setABoxModel(unionABoxModel);
// union TBox
OntModel unionTBoxModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,ModelFactory.createUnion(baseOms.getTBoxModel(), inferenceOms.getTBoxModel()));
unionOms.setTBoxModel(unionTBoxModel);
// Application metadata model is cached in memory.
try {
Model applicationMetadataModelDB = makeDBModel(bds, JENA_APPLICATION_METADATA_MODEL, DB_ONT_MODEL_SPEC, TripleStoreType.SDB, ctx);
OntModel applicationMetadataModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
long startTime = System.currentTimeMillis();
System.out.println("Copying cached application metadata model into memory");
applicationMetadataModel.add(applicationMetadataModelDB);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to load application metadata model assertions of size " + applicationMetadataModel.size());
applicationMetadataModel.getBaseModel().register(new ModelSynchronizer(applicationMetadataModelDB));
if (applicationMetadataModelDB.size() == 0) {
repairAppMetadataModel(applicationMetadataModel, aboxAssertions, aboxInferences);
}
baseOms.setApplicationMetadataModel(applicationMetadataModel);
inferenceOms.setApplicationMetadataModel(baseOms.getApplicationMetadataModel());
unionOms.setApplicationMetadataModel(baseOms.getApplicationMetadataModel());
} catch (Throwable e) {
log.error("Unable to load application metadata model cache from DB", e);
}
log.info("Adding vitro application ontology");
// add the vitroontologies to the tbox models
OntModel vitroTBoxModel = (new JenaBaseDaoCon()).getConstModel();
baseOms.getTBoxModel().addSubModel(vitroTBoxModel);
inferenceOms.getTBoxModel().addSubModel(vitroTBoxModel);
unionOms.getTBoxModel().addSubModel(vitroTBoxModel);
log.info("Setting up union models and DAO factories");
// create TBox + ABox union models and set up webapp DAO factories
OntModel baseUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(baseOms.getABoxModel(), baseOms.getTBoxModel()));
baseOms.setFullModel(baseUnion);
ModelContext.setBaseOntModel(baseOms.getFullModel(), ctx);
WebappDaoFactory baseWadf = new WebappDaoFactorySDB(
baseOms,
bds,
storeDesc,
defaultNamespace,
null,
null,
WebappDaoFactorySDB.SDBDatasetMode.ASSERTIONS_ONLY);
ctx.setAttribute("assertionsWebappDaoFactory",baseWadf);
OntModel inferenceUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(inferenceOms.getABoxModel(), inferenceOms.getTBoxModel()));
inferenceOms.setFullModel(inferenceUnion);
ModelContext.setInferenceOntModel(inferenceOms.getFullModel(), ctx);
WebappDaoFactory infWadf = new WebappDaoFactorySDB(
inferenceOms,
bds,
storeDesc,
defaultNamespace,
null,
null,
WebappDaoFactorySDB.SDBDatasetMode.INFERENCES_ONLY);
ctx.setAttribute("deductionsWebappDaoFactory", infWadf);
OntModel masterUnion = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM,
ModelFactory.createUnion(unionABoxModel, unionTBoxModel));
unionOms.setFullModel(masterUnion);
ctx.setAttribute("jenaOntModel", masterUnion);
WebappDaoFactory wadf = new WebappDaoFactorySDB(unionOms, bds, storeDesc, defaultNamespace, null, null);
ctx.setAttribute("webappDaoFactory",wadf);
ModelContext.setUnionOntModelSelector(unionOms, ctx); // assertions and inferences
ModelContext.setBaseOntModelSelector(baseOms, ctx); // assertions
ModelContext.setInferenceOntModelSelector(inferenceOms, ctx); // inferences
ApplicationBean appBean = wadf.getApplicationDao().getApplicationBean();
if (appBean != null) {
ctx.setAttribute("applicationBean", appBean);
}
//if (isEmpty(unionOms.getFullModel())) {
// loadDataFromFilesystem(unionOms.getFullModel(), ctx);
//}
log.info("Checking for user account data");
if (userAccountsModel.size() == 0) {
readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsModel);
if (userAccountsModel.size() == 0) {
createInitialAdminUser(userAccountsModel, ctx);
}
}
log.info("Checking for minimal interface metadata");
ensureEssentialInterfaceData(unionOms.getApplicationMetadataModel(), sce, wadf);
//log.info("Setting up namespace mapper");
//NamespaceMapper namespaceMapper = new NamespaceMapperJena(masterUnion, masterUnion, defaultNamespace);
//ctx.setAttribute("NamespaceMapper", namespaceMapper);
//unionOms.getFullModel().getBaseModel().register(namespaceMapper);
ctx.setAttribute("defaultNamespace", defaultNamespace);
log.info("SDB store ready for use");
makeModelMakerFromConnectionProperties(TripleStoreType.RDB, ctx);
VitroJenaModelMaker vjmm = getVitroJenaModelMaker();
setVitroJenaModelMaker(vjmm,sce);
makeModelMakerFromConnectionProperties(TripleStoreType.SDB, ctx);
VitroJenaSDBModelMaker vsmm = getVitroJenaSDBModelMaker();
setVitroJenaSDBModelMaker(vsmm,sce);
log.info("Model makers set up");
} catch (MigrationRequiredError mre) {
throw new MigrationRequiredError(mre.getMessage());
} catch (SQLException sqle) {
// SQL exceptions are fatal and should halt startup
AbortStartup.abortStartup(ctx);
log.error("Error using SQL database; startup aborted.", sqle);
// print to catalina.out for good measure
System.out.println("Error using SQL database; startup aborted.");
sqle.printStackTrace();
throw new Error(this.getClass().getName() + "failed");
} catch (Throwable t) {
log.error("Throwable in " + this.getClass().getName(), t);
// printing the error because Tomcat doesn't print context listener
// errors the same way it prints other errors at startup
t.printStackTrace();
throw new Error(this.getClass().getName() + "failed");
}
}
private void checkForNamespaceMismatch(OntModel model, String defaultNamespace, ServletContextEvent sce) {
String defaultNamespaceFromDeployProperites = ConfigurationProperties
.getBean(sce).getProperty("Vitro.defaultNamespace");
if( defaultNamespaceFromDeployProperites == null ){
log.error("Could not get namespace from deploy.properties.");
}
List<String> portalURIs = new ArrayList<String>();
try {
model.enterCriticalSection(Lock.READ);
Iterator<Individual> portalIt = model.listIndividuals(PORTAL);
while (portalIt.hasNext()) {
portalURIs.add( portalIt.next().getURI() );
}
} finally {
model.leaveCriticalSection();
}
if( portalURIs.size() > 0 ){
for( String portalUri : portalURIs){
if( portalUri != null && ! portalUri.startsWith(defaultNamespaceFromDeployProperites)){
log.error("Namespace mismatch between db and deploy.properties.");
log.error("Vivo will not start up correctly because the default namespace specified in deploy.properties does not match the namespace of " +
"a portal in the database. Namespace from deploy.properties: \"" + defaultNamespaceFromDeployProperites +
"\" Namespace from an existing portal: \"" + portalUri + "\" To get the application to start with this " +
"database change the default namespace in deploy.properties " + portalUri.substring(0, portalUri.lastIndexOf("/")+1) +
" Another possibility is that deploy.properties does not specify the intended database.");
}
}
}
}
/* ====================================================================== */
@Override
public void contextDestroyed(ServletContextEvent sce) {
// Nothing to do.
}
private void ensureEssentialInterfaceData(OntModel memModel, ServletContextEvent sce, WebappDaoFactory wadf) {
Model essentialInterfaceData = null;
ClosableIterator<Individual> portalIt = memModel.listIndividuals(
memModel.getResource(VitroVocabulary.PORTAL));
try {
if (!portalIt.hasNext()) {
log.info("Loading initial site configuration");
essentialInterfaceData = InitialJenaModelUtils.loadInitialModel(sce.getServletContext(), defaultNamespace);
if (essentialInterfaceData.size() == 0) {
log.info("Using basic initial site configuration.");
essentialInterfaceData = InitialJenaModelUtils.basicInterfaceData(defaultNamespace);
essentialInterfaceData.add(InitialJenaModelUtils.basicClassgroup(wadf.getDefaultNamespace()));
}
//JenaModelUtils.makeClassGroupsFromRootClasses(wadf,memModel,essentialInterfaceData);
memModel.add(essentialInterfaceData);
} else {
//Set the default namespace to the namespace of the first portal object we find.
//This will keep existing applications from dying when the default namespace
//config option is missing.
Individual portal = portalIt.next();
if (portal.getNameSpace() != null) {
defaultNamespace = portal.getNameSpace();
}
}
} finally {
portalIt.close();
}
}
private void checkMainModelForUserAccounts(OntModel mainModel, OntModel userAccountsModel) {
Model extractedUserData = ((new JenaModelUtils()).extractUserAccountsData(mainModel));
if (extractedUserData.size() > 0) {
userAccountsModel.enterCriticalSection(Lock.WRITE);
try {
userAccountsModel.add(extractedUserData);
} finally {
userAccountsModel.leaveCriticalSection();
}
mainModel.enterCriticalSection(Lock.WRITE);
try {
mainModel.remove(extractedUserData);
} finally {
mainModel.leaveCriticalSection();
}
}
}
private OntModel ontModelFromContextAttribute(ServletContext ctx, String attribute) {
OntModel ontModel;
Object attributeValue = ctx.getAttribute(attribute);
if (attributeValue != null && attributeValue instanceof OntModel) {
ontModel = (OntModel) attributeValue;
} else {
ontModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
ctx.setAttribute(attribute, ontModel);
}
return ontModel;
}
private boolean isEmpty(Model model) {
ClosableIterator<Statement> closeIt = model.listStatements(
null, RDF.type, ResourceFactory.createResource(
VitroVocabulary.PORTAL));
try {
if (closeIt.hasNext()) {
return false;
} else {
return true;
}
} finally {
closeIt.close();
}
}
private void loadDataFromFilesystem(OntModel ontModel, ServletContext ctx) {
OntModel initialDataModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
Long startTime = System.currentTimeMillis();
log.debug("Reading ontology files");
readOntologyFilesInPathSet(USERPATH, ctx, initialDataModel);
readOntologyFilesInPathSet(SYSTEMPATH, ctx, initialDataModel);
log.debug(((System.currentTimeMillis()-startTime)/1000)+" seconds to read ontology files ");
ontModel.add(initialDataModel);
}
private static void getTBoxModel(Model fullModel, Model submodels, Model tboxModel) {
JenaModelUtils modelUtils = new JenaModelUtils();
Model tempModel = ModelFactory.createUnion(fullModel, submodels);
Model tempTBoxModel = modelUtils.extractTBox(tempModel);
// copy intersection of tempTBoxModel and fullModel to tboxModel.
StmtIterator iter = tempTBoxModel.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.next();
if (fullModel.contains(stmt)) {
tboxModel.add(stmt);
}
}
return;
}
/*
* Copy all statements from model 1 that are not in model 2 to model 3.
*/
private static void copyDifference(Model model1, Model model2, Model model3) {
StmtIterator iter = model1.listStatements();
while (iter.hasNext()) {
Statement stmt = iter.next();
if (!model2.contains(stmt)) {
model3.add(stmt);
}
}
return;
}
private static void getAppMetadata(Model source, Model target) {
String amdQuery = "DESCRIBE ?x WHERE { " +
"{?x a <" + VitroVocabulary.PORTAL +"> } UNION " +
"{?x a <" + VitroVocabulary.PROPERTYGROUP +"> } UNION " +
"{?x a <" + VitroVocabulary.CLASSGROUP +"> } } ";
try {
Query q = QueryFactory.create(amdQuery, Syntax.syntaxARQ);
QueryExecution qe = QueryExecutionFactory.create(q, source);
qe.execDescribe(target);
} catch (Exception e) {
log.error("unable to create the application metadata model",e);
}
return;
}
private static void repairAppMetadataModel(Model applicationMetadataModel,
Model aboxAssertions,
Model aboxInferences) {
log.info("Moving application metadata from ABox to dedicated model");
getAppMetadata(aboxAssertions, applicationMetadataModel);
getAppMetadata(aboxInferences, applicationMetadataModel);
aboxAssertions.remove(applicationMetadataModel);
aboxInferences.remove(applicationMetadataModel);
return;
}
public static StoreDesc makeStoreDesc(ServletContext ctx) {
String layoutStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.sdb.layout", "layout2/hash");
String dbtypeStr = ConfigurationProperties.getBean(ctx).getProperty(
"VitroConnection.DataSource.dbtype", "MySQL");
return new StoreDesc(
LayoutType.fetch(layoutStr),
DatabaseType.fetch(dbtypeStr) );
}
public static Store connectStore(BasicDataSource bds, StoreDesc storeDesc)
throws SQLException {
SDBConnection conn = new SDBConnection(bds.getConnection()) ;
return SDBFactory.connectStore(conn, storeDesc);
}
public static void setupSDB(ServletContext ctx,
Store store,
Model memModel,
Model inferenceModel) {
store.getTableFormatter().create();
store.getTableFormatter().truncate();
store.getTableFormatter().dropIndexes(); // improve load performance
try {
// This is a one-time copy of stored KB data - from a Jena RDB store
// to a Jena SDB store. In the process, we will also separate out the
// TBox from the Abox; these are in the same graph in pre 1.2 VIVO
// versions and will now be stored and maintained in separate models
// Access to the Jena RDB data is through the OntModelSelectors that have
// been set up earlier in the current session by
// JenaPersistentDataSourceSetup.java
// In the code below, note that the current getABoxModel() methods on
// the OntModelSelectors return a graph with both ABox and TBox data.
OntModel submodels = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
readOntologyFilesInPathSet(SUBMODELS, ctx, submodels);
Model tboxAssertions = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL);
// initially putting the results in memory so we have a
// cheaper way of computing the difference when we copy the ABox
Model memTboxAssertions = ModelFactory.createDefaultModel();
getTBoxModel(memModel, submodels, memTboxAssertions);
tboxAssertions.add(memTboxAssertions);
Model tboxInferences = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_TBOX_INF_MODEL);
// initially putting the results in memory so we have a
// cheaper way of computing the difference when we copy the ABox
Model memTboxInferences = ModelFactory.createDefaultModel();
getTBoxModel(inferenceModel, submodels, memTboxInferences);
tboxInferences.add(memTboxInferences);
Model aboxAssertions = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_DB_MODEL);
copyDifference(memModel, memTboxAssertions, aboxAssertions);
Model aboxInferences = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_INF_MODEL);
copyDifference(inferenceModel, memTboxInferences, aboxInferences);
// Set up the application metadata model
Model applicationMetadataModel = SDBFactory.connectNamedModel(
store, JenaDataSourceSetupBase.JENA_APPLICATION_METADATA_MODEL);
getAppMetadata(memModel, applicationMetadataModel);
log.info("During initial SDB setup, created an application metadata model of size "
+ applicationMetadataModel.size());
// remove application metadata from ABox model
aboxAssertions.remove(applicationMetadataModel);
aboxInferences.remove(applicationMetadataModel);
// Make sure the reasoner takes into account the newly-set-up data.
SimpleReasonerSetup.setRecomputeRequired(ctx);
} finally {
log.info("Adding indexes to SDB database tables.");
store.getTableFormatter().addIndexes();
log.info("Indexes created.");
}
}
/**
* Tests whether an SDB store has been formatted and populated for use.
* @param store
* @return
*/
private boolean isSetUp(Store store) throws SQLException {
if (!(StoreUtils.isFormatted(store))) {
return false;
}
// even if the store exists, it may be empty
try {
return (SDBFactory.connectNamedModel(store, JenaDataSourceSetupBase.JENA_TBOX_ASSERTIONS_MODEL)).size() > 0;
} catch (Exception e) {
return false;
}
}
private static final String STOREDESC_ATTR = "storeDesc";
private static final String STORE_ATTR = "kbStore";
public static void setApplicationStoreDesc(StoreDesc storeDesc,
ServletContext ctx) {
ctx.setAttribute(STOREDESC_ATTR, storeDesc);
}
public static StoreDesc getApplicationStoreDesc(ServletContext ctx) {
return (StoreDesc) ctx.getAttribute(STOREDESC_ATTR);
}
public static void setApplicationStore(Store store,
ServletContext ctx) {
ctx.setAttribute(STORE_ATTR, store);
}
public static Store getApplicationStore(ServletContext ctx) {
return (Store) ctx.getAttribute(STORE_ATTR);
}
/**
* Executes a SPARQL ASK query to determine whether the knowledge base
* needs to be updated to conform to a new ontology version
*/
public boolean updateRequired(ServletContext ctx, OntModel m) throws IOException {
boolean required = false;
String sparqlQueryStr = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskQueryPath(ctx));
if (sparqlQueryStr == null) {
return required;
}
Query query = QueryFactory.create(sparqlQueryStr);
QueryExecution isUpdated = QueryExecutionFactory.create(query, m);
// if the ASK query DOES have a solution (i.e. the assertions exist
// showing that the update has already been performed), then the update
// is NOT required.
if (isUpdated.execAsk()) {
required = false;
} else {
required = true;
String sparqlQueryStr2 = KnowledgeBaseUpdater.loadSparqlQuery(UpdateKnowledgeBase.getAskEmptyQueryPath(ctx));
if (sparqlQueryStr2 != null) {
Query query2 = QueryFactory.create(sparqlQueryStr2);
QueryExecution isNotEmpty = QueryExecutionFactory.create(query2, m);
required = isNotEmpty.execAsk();
}
}
return required;
}
private String getMigrationErrString() {
String errMessage = "\n*******************************************************************";
errMessage += "\nA knowledge base migration is " +
"required and this must be done in" +
" RDB mode before converting to SDB. " +
"Please change deploy.properties to" +
" use RDB mode, redeploy, and restart. After " +
"the knowledge base migration has completed " +
"successfully, change deploy.properties to use " +
"SDB mode, redeploy, and restart.\n";
errMessage += "*******************************************************************";
return errMessage;
}
private class MigrationRequiredError extends Error {
public MigrationRequiredError(String string) {
super(string);
}
}
}

105
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaPersistentDBOnlyDataSourceSetup.java
View file

@ -1,105 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.util.iterator.ClosableIterator;
/**
* WARNING: This configuration may not reconnect to MySQL after connection timeouts.
* @author bjl23
*
*/
public class JenaPersistentDBOnlyDataSourceSetup extends JenaDataSourceSetupBase implements ServletContextListener {
private static final Log log = LogFactory.getLog(JenaPersistentDataSourceSetup.class.getName());
@Override
public void contextInitialized(ServletContextEvent sce) {
ServletContext ctx = sce.getServletContext();
OntModel memModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
OntModel dbModel = null;
try {
Model dbPlainModel = makeDBModelFromConfigurationProperties(JENA_DB_MODEL, DB_ONT_MODEL_SPEC, ctx);
dbModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,dbPlainModel);
boolean isEmpty = true;
ClosableIterator stmtIt = dbModel.listStatements();
try {
if (stmtIt.hasNext()) {
isEmpty = false;
}
} finally {
stmtIt.close();
}
if (isEmpty) {
long startTime = System.currentTimeMillis();
System.out.println("Reading ontology files into database");
readOntologyFilesInPathSet(USERPATH, ctx, dbModel);
readOntologyFilesInPathSet(AUTHPATH, ctx, dbModel);
readOntologyFilesInPathSet(SYSTEMPATH, ctx, dbModel);
System.out.println((System.currentTimeMillis()-startTime)/1000+" seconds to populate DB");
}
//readOntologyFilesInPathSet(ctx.getResourcePaths(AUTHPATH), sce, dbModel);
//readOntologyFilesInPathSet(ctx.getResourcePaths(SYSTEMPATH), sce, dbModel);
memModel = dbModel;
} catch (Throwable t) {
System.out.println("**** ERROR *****");
System.out.println("Vitro unable to open Jena database model.");
System.out.println("Check that the configuration properties file has been created in WEB-INF/classes, ");
System.out.println("and that the database connection parameters are accurate. ");
System.out.println("****************");
}
try {
if (dbModel==null) {
System.out.println("Reading ontology files");
readOntologyFilesInPathSet(USERPATH, ctx, dbModel);
readOntologyFilesInPathSet(AUTHPATH, ctx, dbModel);
readOntologyFilesInPathSet(SYSTEMPATH, ctx, dbModel);
}
} catch (Exception f) {
log.error(f);
}
// default inference graph
try {
Model infDbPlainModel = makeDBModelFromConfigurationProperties(JENA_INF_MODEL, DB_ONT_MODEL_SPEC, ctx);
OntModel infDbModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC,infDbPlainModel);
ctx.setAttribute("inferenceOntModel",infDbModel);
} catch (Throwable e) {
log.error(e, e);
}
ctx.setAttribute("jenaOntModel", memModel);
ctx.setAttribute("persistentOntModel", dbModel);
// BJL23: This is a funky hack until I completely rework how the models get set up in a more sane fashion
ctx.setAttribute("useModelSynchronizers", "false");
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
//Close the database connection
//try {
// ((IDBConnection)sce.getServletContext().getAttribute("jenaConnection")).close();
//} catch (Exception e) {
// //log.debug("could not close the JDBC connection.");
//}
}
}

110
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaPersistentDataSourceSetup.java
View file

@ -32,96 +32,13 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
if (AbortStartup.isStartupAborted(ctx)) {
return;
}
Model dbModel;
OntModel memModel = ModelFactory.createOntologyModel(
this.DB_ONT_MODEL_SPEC);
try {
dbModel = makeDBModelFromConfigurationProperties(
JENA_DB_MODEL, DB_ONT_MODEL_SPEC, ctx);
boolean firstStartup = isFirstStartup(dbModel);
if (firstStartup) {
long startTime = System.currentTimeMillis();
System.out.println("Reading ontology files into database");
readOntologyFilesInPathSet(USERPATH, ctx, dbModel);
readOntologyFilesInPathSet(SYSTEMPATH, ctx, dbModel);
System.out.println(
(System.currentTimeMillis() - startTime) / 1000 +
" seconds to populate DB");
}
if (isSDBActive(ctx)) {
memModel = ModelFactory.createOntologyModel(
this.DB_ONT_MODEL_SPEC, dbModel);
// no in-memory copying
} else {
System.out.println(
"Populating in-memory Jena model from " +
"persistent DB model");
long startTime = System.currentTimeMillis();
memModel.add(dbModel);
System.out.println(
(System.currentTimeMillis() - startTime) / 1000 +
" seconds to synchronize models");
memModel.getBaseModel().register(
new ModelSynchronizer(dbModel));
}
} catch (OutOfMemoryError ome) {
System.out.println("**** ERROR *****");
System.out.println("Insufficient memory to load ");
System.out.println("database contents for vitro.");
System.out.println("Refer to servlet container documentation ");
System.out.println("about increasing heap space.");
System.out.println("****************");
} catch (Throwable t) {
System.out.println("Logging error details");
log.error("Unable to open db model", t);
System.out.println("**** ERROR *****");
System.out.println("Vitro unable to open Jena database model.");
System.out.println("Check that the configuration properties file ");
System.out.println("has been created in WEB-INF/classes, ");
System.out.println("and that the database connection parameters ");
System.out.println("are accurate. ");
System.out.println("****************");
}
// default inference graph
try {
Model infDbModel = makeDBModelFromConfigurationProperties(
JENA_INF_MODEL, DB_ONT_MODEL_SPEC, ctx);
OntModel infModel = null;
if (infDbModel != null) {
if (isSDBActive(ctx)) {
infModel = ModelFactory.createOntologyModel(
MEM_ONT_MODEL_SPEC, infDbModel);
} else {
infModel = ModelFactory.createOntologyModel(
MEM_ONT_MODEL_SPEC);
long startTime = System.currentTimeMillis();
System.out.println("Copying cached inferences into memory");
infModel.add(infDbModel);
System.out.println(
(System.currentTimeMillis() - startTime) / 1000 +
" seconds to load inferences");
}
}
infModel.getBaseModel().register(new ModelSynchronizer(infDbModel));
ctx.setAttribute("inferenceOntModel",infModel);
} catch (Throwable e) {
log.error("Unable to load inference cache from DB", e);
}
// user accounts Model
try {
Model userAccountsDbModel = makeDBModelFromConfigurationProperties(
JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (userAccountsDbModel.size() == 0) {
firstStartup = true;
readOntologyFilesInPathSet(AUTHPATH, sce.getServletContext(),
userAccountsDbModel);
if (userAccountsDbModel.size() == 0) {
@ -135,6 +52,9 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
new ModelSynchronizer(userAccountsDbModel));
sce.getServletContext().setAttribute(
"userAccountsOntModel", userAccountsModel);
if (userAccountsModel.isEmpty()) {
initializeUserAccounts(ctx, userAccountsModel);
}
} catch (Throwable t) {
log.error("Unable to load user accounts model from DB", t);
}
@ -154,26 +74,20 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
} catch (Throwable t) {
log.error("Unable to load user application configuration model from DB", t);
}
ctx.setAttribute("jenaOntModel", memModel);
}
private boolean isFirstStartup(Model dbModel) {
ClosableIterator stmtIt = dbModel.listStatements(
null,
RDF.type,
ResourceFactory.createResource(VitroVocabulary.PORTAL));
try {
return (!stmtIt.hasNext());
} finally {
stmtIt.close();
}
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
// Nothing to do.
}
private void initializeUserAccounts(ServletContext ctx,
Model userAccountsModel) {
readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsModel);
if (userAccountsModel.size() == 0) {
createInitialAdminUser(userAccountsModel, ctx);
}
}
}

22
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/PelletReasonerSetup.java
View file

@ -26,20 +26,18 @@ public class PelletReasonerSetup implements ServletContextListener {
if (AbortStartup.isStartupAborted(sce.getServletContext())) {
return;
}
if (true) {
(new SimpleReasonerSetup()).contextInitialized(sce);
return;
// use the simple reasoner instead of Pellet for ABox inferences
// if we're running SDB
}
// presently unreachable code follows
try {
//FIXME refactor this
String tripleStoreTypeStr =
ConfigurationProperties.getBean(sce).getProperty(
"VitroConnection.DataSource.tripleStoreType", "RDB");
if ("SDB".equals(tripleStoreTypeStr)) {
(new SimpleReasonerSetup()).contextInitialized(sce);
return;
// use the simple reasoner instead of Pellet for ABox inferences
// if we're running SDB
}
OntModel memoryModel = (OntModel) sce.getServletContext().getAttribute("jenaOntModel");
OntModel baseModel = (OntModel) sce.getServletContext().getAttribute("baseOntModel");
OntModel inferenceModel = (OntModel) sce.getServletContext().getAttribute("inferenceOntModel");

83
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/Csv2Rdf.java
View file

@ -56,6 +56,10 @@ public class Csv2Rdf {
this.propertyNameBase = individualNameBase+"_";
}
public Model[] convertToRdf(InputStream fis) throws IOException {
return convertToRdf(fis, null, null);
}
public Model[] convertToRdf(InputStream fis,VitroRequest vreq, Model destination) throws IOException {
OntModel ontModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM);
OntModel tboxOntModel = ModelFactory.createOntologyModel(OntModelSpec.OWL_MEM);
@ -65,13 +69,11 @@ public class Csv2Rdf {
CSVReader cReader = new SimpleReader();
cReader.setSeperator(separatorChar);
cReader.setQuoteCharacters(quoteChars);
WebappDaoFactory wdf = vreq.getFullWebappDaoFactory();
Random random = new Random();
boolean uriIsGood = false;
boolean inDestination = false;
int attempts = 0;
String uri = null;
String errMsg = null;
URIGenerator uriGen = (vreq != null && destination != null)
? new RandomURIGenerator(vreq, destination)
: new SequentialURIGenerator();
List<String[]> fileRows = cReader.parse(fis);
String[] columnHeaders = fileRows.get(0);
@ -82,24 +84,8 @@ public class Csv2Rdf {
dpArray[i] = tboxOntModel.createDatatypeProperty(tboxNamespace+propertyNameBase+columnHeaders[i].replaceAll("\\W",""));
}
Individual ind = null;
for (int row=1; row<fileRows.size(); row++) {
if(namespace!=null && !namespace.isEmpty()){
while( uriIsGood == false && attempts < 30 ){
uri = namespace+individualNameBase+random.nextInt( Math.min(Integer.MAX_VALUE,(int)Math.pow(2,attempts + 13)) );
errMsg = wdf.checkURI(uri);
Resource res = ResourceFactory.createResource(uri);
inDestination = destination.contains(res, null);
if( errMsg != null && !inDestination)
uri = null;
else
uriIsGood = true;
attempts++;
}
}
uriIsGood = false;
attempts =0;
inDestination = false;
for (int row=1; row<fileRows.size(); row++) {
String uri = uriGen.getNextURI();
if(uri!=null)
ind = ontModel.createIndividual(uri,theClass);
else
@ -123,4 +109,51 @@ public class Csv2Rdf {
}
private interface URIGenerator {
public String getNextURI();
}
private class RandomURIGenerator implements URIGenerator {
private VitroRequest vreq;
private Model destination;
private Random random = new Random(System.currentTimeMillis());
public RandomURIGenerator(VitroRequest vreq, Model destination) {
this.vreq = vreq;
this.destination = destination;
}
public String getNextURI() {
boolean uriIsGood = false;
boolean inDestination = false;
String uri = null;
int attempts = 0;
if(namespace!=null && !namespace.isEmpty()){
while( uriIsGood == false && attempts < 30 ){
uri = namespace+individualNameBase+random.nextInt( Math.min(Integer.MAX_VALUE,(int)Math.pow(2,attempts + 13)) );
String errMsg = vreq.getWebappDaoFactory().checkURI(uri);
Resource res = ResourceFactory.createResource(uri);
inDestination = destination.contains(res, null);
if( errMsg != null && !inDestination)
uri = null;
else
uriIsGood = true;
attempts++;
}
}
return uri;
}
}
private class SequentialURIGenerator implements URIGenerator {
private int index = 0;
public String getNextURI() {
index++;
return namespace + individualNameBase + Integer.toString(index);
}
}
}