litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-823 Improve the TDB implementation of RDFService

Browse source 
Use a single dataset for long-term and short-term use. Get read and write locks for all access to the dataset.
Create a "VitroConnection.DataSource.tdbDirectory" property that will force and configure TDB for content models.
This commit is contained in:
Jim Blake 2014-07-28 11:54:12 -04:00
parent 0e5ee7226b
commit 2baa2c33e0
4 changed files with 141 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
webapp/src/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/tdb/RDFServiceTDB.java
View file

@ -3,14 +3,18 @@
package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.tdb;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.tdb.TDB;
import com.hp.hpl.jena.tdb.TDBFactory;
@ -65,10 +69,16 @@ public class RDFServiceTDB extends RDFServiceJena {
log.debug("Change Set: " + changeSet);
}
notifyListenersOfPreChangeEvents(changeSet);
applyChangeSetToModel(changeSet, dataset);
TDB.sync(dataset);
dataset.getLock().enterCriticalSection(Lock.WRITE);
try {
TDB.sync(dataset);
} finally {
dataset.getLock().leaveCriticalSection();
}
notifyListenersOfChanges(changeSet);
notifyListenersOfPostChangeEvents(changeSet);
@ -82,7 +92,77 @@ public class RDFServiceTDB extends RDFServiceJena {
@Override
public void close() {
if (this.dataset != null) {
dataset.close();
dataset.getLock().enterCriticalSection(Lock.WRITE);
try {
dataset.close();
} finally {
dataset.getLock().leaveCriticalSection();
}
}
}
@Override
public InputStream sparqlConstructQuery(String query,
ModelSerializationFormat resultFormat) throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
return super.sparqlConstructQuery(query, resultFormat);
} finally {
dataset.getLock().leaveCriticalSection();
}
}
@Override
public InputStream sparqlDescribeQuery(String query,
ModelSerializationFormat resultFormat) throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
return super.sparqlDescribeQuery(query, resultFormat);
} finally {
dataset.getLock().leaveCriticalSection();
}
}
@Override
public void sparqlSelectQuery(String query, ResultFormat resultFormat,
OutputStream outputStream) throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
super.sparqlSelectQuery(query, resultFormat, outputStream);
} finally {
dataset.getLock().leaveCriticalSection();
}
}
@Override
public InputStream sparqlSelectQuery(String query, ResultFormat resultFormat)
throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
return super.sparqlSelectQuery(query, resultFormat);
} finally {
dataset.getLock().leaveCriticalSection();
}
}
@Override
public boolean sparqlAskQuery(String query) throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
return super.sparqlAskQuery(query);
} finally {
dataset.getLock().leaveCriticalSection();
}
}
@Override
public List<String> getGraphURIs() throws RDFServiceException {
dataset.getLock().enterCriticalSection(Lock.READ);
try {
return super.getGraphURIs();
} finally {
dataset.getLock().leaveCriticalSection();
}
}
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/rdfsetup/RDFSetup.java
View file

@ -2,7 +2,10 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup.rdfsetup;
import static edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService.CONFIGURATION;
import static edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService.CONTENT;
import static edu.cornell.mannlib.vitro.webapp.servlet.setup.rdfsetup.impl.sparql.RDFSourceSPARQL.PROPERTY_SPARQL_ENDPOINT_URI;
import static edu.cornell.mannlib.vitro.webapp.servlet.setup.rdfsetup.impl.tdb.RDFSourceTDB.PROPERTY_CONTENT_TDB_PATH;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
@ -70,10 +73,12 @@ public class RDFSetup implements ServletContextListener {
private void createRdfSources() {
if (isSparqlEndpointContentConfigured()) {
contentRdfSource = new RDFSourceSPARQL(ctx, this);
} else if (isTdbConfigured()) {
contentRdfSource = new RDFSourceTDB(ctx, this, CONTENT);
} else {
contentRdfSource = new RDFSourceSDB(ctx, this);
}
configurationRdfSource = new RDFSourceTDB(ctx, this);
configurationRdfSource = new RDFSourceTDB(ctx, this, CONFIGURATION);
}
private boolean isSparqlEndpointContentConfigured() {
@ -81,6 +86,11 @@ public class RDFSetup implements ServletContextListener {
.getProperty(PROPERTY_SPARQL_ENDPOINT_URI));
}
private boolean isTdbConfigured() {
return StringUtils.isNotBlank(configProps
.getProperty(PROPERTY_CONTENT_TDB_PATH));
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
if (configurationRdfSource != null) {

30
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/rdfsetup/impl/tdb/ContentModelMakerFactoryTDB.java
View file

@ -9,16 +9,14 @@ import edu.cornell.mannlib.vitro.webapp.modelaccess.ContentModelMakerFactory;
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelMakerFactory;
import edu.cornell.mannlib.vitro.webapp.modelaccess.adapters.ListCachingModelMaker;
import edu.cornell.mannlib.vitro.webapp.modelaccess.adapters.MemoryMappingModelMaker;
import edu.cornell.mannlib.vitro.webapp.modelaccess.adapters.ShadowingModelMaker;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
/**
* In TDB, is there any difference between short-term and long-term connections?
* For now, use long-term connections for all models, memory-mapping the small
* ones.
*
* Anyway, memory-map the small models, and use a short-term connection for the
* others (when available).
*
* RDFService doesn't support empty models, so support them with ListCaching
* RDFService doesn't support empty models, so support them with ListCaching.
*/
public class ContentModelMakerFactoryTDB extends ContentModelMakerFactory
implements ModelMakerFactory {
@ -26,14 +24,13 @@ public class ContentModelMakerFactoryTDB extends ContentModelMakerFactory
private final ModelMaker longTermModelMaker;
public ContentModelMakerFactoryTDB(RDFService longTermRdfService) {
this.longTermModelMaker = new ListCachingModelMaker(new MemoryMappingModelMaker(
new RDFServiceModelMaker(longTermRdfService),
SMALL_CONTENT_MODELS));
this.longTermModelMaker = new ListCachingModelMaker(
new MemoryMappingModelMaker(new RDFServiceModelMaker(
longTermRdfService), SMALL_CONTENT_MODELS));
}
/**
* The small content models (tbox, app_metadata) are memory mapped, for
* speed.
* The small content models are memory mapped, for speed.
*/
@Override
public ModelMaker getModelMaker(RDFService longTermRdfService) {
@ -41,19 +38,12 @@ public class ContentModelMakerFactoryTDB extends ContentModelMakerFactory
}
/**
* For short-term use, the large models (abox) will come from a short-term
* service. The small models can be the memory-mapped ones that we created
* for long-term use.
* There are no connections or connection pool, so short-term use is the
* same as long-term use.
*/
@Override
public ModelMaker getShortTermModelMaker(RDFService shortTermRdfService) {
ModelMaker shortTermModelMaker = new RDFServiceModelMaker(
shortTermRdfService);
// No need to create a fresh memory map of the small models: use the
// long-term ones.
return addContentDecorators(new ShadowingModelMaker(
shortTermModelMaker, longTermModelMaker, SMALL_CONTENT_MODELS));
return addContentDecorators(longTermModelMaker);
}
}

45
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/rdfsetup/impl/tdb/RDFSourceTDB.java
View file

@ -2,6 +2,9 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup.rdfsetup.impl.tdb;
import static edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService.CONFIGURATION;
import static edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService.CONTENT;
import java.io.File;
import java.io.IOException;
@ -11,17 +14,21 @@ import javax.servlet.ServletContextListener;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.tdb.TDB;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.modelaccess.ModelMakerFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils.WhichService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.tdb.RDFServiceTDB;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.rdfsetup.RDFSource;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
/**
* Create the connection to the TDB triple-store.
* Create the connection to the TDB triple-store. This connection is either for
* CONTENT or for CONFIGURATION, but not both.
*
* Create the RDFService on the directory. Create the RDFServiceFactory.
*/
@ -29,19 +36,33 @@ public class RDFSourceTDB implements RDFSource {
private static final Log log = LogFactory.getLog(RDFSourceTDB.class);
private static final String DIRECTORY_TDB = "tdbModels";
public static final String PROPERTY_CONTENT_TDB_PATH = "VitroConnection.DataSource.tdbDirectory";
private final ConfigurationProperties props;
private final StartupStatus ss;
private final WhichService which;
private final RDFService rdfService;
private final RDFServiceFactory rdfServiceFactory;
public RDFSourceTDB(ServletContext ctx, ServletContextListener parent) {
public RDFSourceTDB(ServletContext ctx, ServletContextListener parent,
WhichService which) {
this.props = ConfigurationProperties.getBean(ctx);
this.ss = StartupStatus.getBean(ctx);
this.which = which;
configureTDB();
String tdbPath;
if (CONTENT == which) {
tdbPath = props.getProperty(PROPERTY_CONTENT_TDB_PATH);
} else {
String vitroHome = props.getProperty("vitro.home");
tdbPath = vitroHome + File.separatorChar + DIRECTORY_TDB;
}
try {
this.rdfService = createRdfService();
this.rdfService = new RDFServiceTDB(tdbPath);
this.rdfServiceFactory = createRDFServiceFactory();
ss.info(parent, "Initialized the RDF source for TDB");
} catch (IOException e) {
@ -50,10 +71,8 @@ public class RDFSourceTDB implements RDFSource {
}
}
private RDFService createRdfService() throws IOException {
String vitroHome = props.getProperty("vitro.home");
String directoryPath = vitroHome + File.separatorChar + DIRECTORY_TDB;
return new RDFServiceTDB(directoryPath);
private void configureTDB() {
TDB.getContext().setTrue(TDB.symUnionDefaultGraph);
}
private RDFServiceFactory createRDFServiceFactory() {
@ -67,12 +86,20 @@ public class RDFSourceTDB implements RDFSource {
@Override
public ModelMakerFactory getContentModelMakerFactory() {
return new ContentModelMakerFactoryTDB(this.rdfService);
if (CONTENT == which) {
return new ContentModelMakerFactoryTDB(this.rdfService);
} else {
throw new IllegalStateException("This RDFSource is for " + which);
}
}
@Override
public ModelMakerFactory getConfigurationModelMakerFactory() {
return new ConfigurationModelMakerFactoryTDB(this.rdfService);
if (CONFIGURATION == which) {
return new ConfigurationModelMakerFactoryTDB(this.rdfService);
} else {
throw new IllegalStateException("This RDFSource is for " + which);
}
}
@Override