litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Faster implementation of TPF for SDB (#54)

Browse source 
* Initial performance improvements to TPF

* Add faster counts for TPF

* Correct end count

* Bug fixes, use hashes for filtering

* Initial LDF SPARQL client

* Handle blank nodes correctly
This commit is contained in:
grahamtriggs 2017-01-31 21:09:49 +00:00 • committed by GitHub
parent 0191ae6dbb
commit b5da11c2be
17 changed files with 1446 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/RDFService.java
View file

@ -8,6 +8,7 @@ import java.util.List;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelChangedListener;
import org.apache.jena.rdf.model.RDFNode;
/**
* Interface for API to write, read, and update Vitro's RDF store, with support
@ -247,7 +248,11 @@ public interface RDFService {
*
* @return ChangeSet an empty ChangeSet object
*/
public ChangeSet manufactureChangeSet();
public ChangeSet manufactureChangeSet();
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException;
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException;
/**
* Frees any resources held by this RDFService object

10
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/filter/LanguageFilteringRDFService.java
View file

@ -468,6 +468,16 @@ public class LanguageFilteringRDFService implements RDFService {
return s.manufactureChangeSet();
}
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
return s.countTriples(subject, predicate, object);
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
return s.getTriples(subject, predicate, object, limit, offset);
}
@Override
public void close() {
s.close();

11
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceFactorySingle.java
View file

@ -16,6 +16,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString;
import org.apache.jena.rdf.model.RDFNode;
/**
* An RDFServiceFactory that always returns the same RDFService object
@ -192,6 +193,16 @@ public class RDFServiceFactorySingle implements RDFServiceFactory {
return s.manufactureChangeSet();
}
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
return s.countTriples(subject, predicate, object);
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
return s.getTriples(subject, predicate, object, limit, offset);
}
@Override
public void close() {
// Don't close s. It's being used by everybody.

125
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/RDFServiceImpl.java
View file

@ -12,18 +12,24 @@ import java.util.concurrent.CopyOnWriteArrayList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.jena.atlas.io.StringWriterI;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.graph.Triple;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryFactory;
import org.apache.jena.query.QueryParseException;
import org.apache.jena.query.QuerySolution;
import org.apache.jena.query.Syntax;
import org.apache.jena.rdf.model.Literal;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelChangedListener;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.jena.riot.out.NodeFormatter;
import org.apache.jena.riot.out.NodeFormatterTTL;
import org.apache.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeListener;
@ -34,6 +40,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString;
import org.vivoweb.linkeddatafragments.datasource.rdfservice.RDFServiceBasedRequestProcessorForTPFs;
public abstract class RDFServiceImpl implements RDFService {
@ -340,5 +347,121 @@ public abstract class RDFServiceImpl implements RDFService {
public String toString() {
return ToString.simpleName(this) + "[" + ToString.hashHex(this) + "]";
}
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
StringBuilder whereClause = new StringBuilder();
StringBuilder orderBy = new StringBuilder();
if ( subject != null ) {
appendNode(whereClause.append(' '), subject);
} else {
whereClause.append(" ?s");
orderBy.append(" ?s");
}
if ( predicate != null ) {
appendNode(whereClause.append(' '), predicate);
} else {
whereClause.append(" ?p");
orderBy.append(" ?p");
}
if ( object != null ) {
appendNode(whereClause.append(' '), object);
} else {
whereClause.append(" ?o");
orderBy.append(" ?o");
}
long estimate = -1;
StringBuilder count = new StringBuilder();
count.append("SELECT (COUNT(*) AS ?count) WHERE { ");
count.append(whereClause.toString());
count.append(" . ");
count.append(" }");
CountConsumer countConsumer = new CountConsumer();
this.sparqlSelectQuery(count.toString(), countConsumer);
return countConsumer.count;
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
StringBuilder whereClause = new StringBuilder();
StringBuilder orderBy = new StringBuilder();
if ( subject != null ) {
appendNode(whereClause.append(' '), subject);
} else {
whereClause.append(" ?s");
orderBy.append(" ?s");
}
if ( predicate != null ) {
appendNode(whereClause.append(' '), predicate);
} else {
whereClause.append(" ?p");
orderBy.append(" ?p");
}
if ( object != null ) {
appendNode(whereClause.append(' '), object);
} else {
whereClause.append(" ?o");
orderBy.append(" ?o");
}
StringBuilder constructQuery = new StringBuilder();
constructQuery.append("CONSTRUCT { ");
constructQuery.append(whereClause.toString());
constructQuery.append(" } WHERE { ");
constructQuery.append(whereClause.toString()).append(" . ");
constructQuery.append(" }");
if (orderBy.length() > 0) {
constructQuery.append(" ORDER BY").append(orderBy.toString());
}
if (limit > 0) {
constructQuery.append(" LIMIT ").append(limit);
}
if (offset > 0) {
constructQuery.append(" OFFSET ").append(offset);
}
Model triples = ModelFactory.createDefaultModel();
this.sparqlConstructQuery(constructQuery.toString(), triples);
return triples;
}
private void appendNode(StringBuilder builder, RDFNode node) {
if (node.isLiteral()) {
builder.append(literalToString(node.asLiteral()));
} else if (node.isURIResource()) {
builder.append('<' + node.asResource().getURI() + '>');
}
}
private String literalToString(Literal l) {
StringWriterI sw = new StringWriterI();
NodeFormatter fmt = new NodeFormatterTTL(null, null);
fmt.formatLiteral(sw, l.asNode());
return sw.toString();
}
class CountConsumer extends ResultSetConsumer {
public long count = -1;
@Override
protected void processQuerySolution(QuerySolution qs) {
if (count == -1) {
Literal literal = qs.getLiteral("count");
count = literal.getLong();
}
}
}
}

69
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/RDFServiceJena.java
View file

@ -16,6 +16,9 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import org.apache.commons.io.IOUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.jena.query.QuerySolutionMap;
import org.apache.jena.query.Syntax;
import org.apache.jena.rdf.model.Literal;
import org.apache.jena.riot.RDFDataMgr;
import org.apache.log4j.lf5.util.StreamUtils;
@ -612,6 +615,71 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
return graph.isIsomorphicWith(fromTripleStoreModel);
}
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
Query countQuery = QueryFactory.create("SELECT (COUNT(?s) AS ?count) WHERE { ?s ?p ?o } ORDER BY ?s ?p ?o", Syntax.syntaxSPARQL_11);
QuerySolutionMap map = new QuerySolutionMap();
if ( subject != null ) {
map.add("s", subject);
}
if ( predicate != null ) {
map.add("p", predicate);
}
if ( object != null ) {
map.add("o", object);
}
DatasetWrapper dw = getDatasetWrapper();
try {
Dataset d = dw.getDataset();
try (QueryExecution qexec = QueryExecutionFactory.create(countQuery, d, map)) {
ResultSet results = qexec.execSelect();
if (results.hasNext()) {
QuerySolution soln = results.nextSolution() ;
Literal literal = soln.getLiteral("count");
return literal.getLong();
}
}
} finally {
dw.close();
}
return 0;
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
Query query = QueryFactory.create("CONSTRUCT WHERE { ?s ?p ?o }", Syntax.syntaxSPARQL_11);
QuerySolutionMap map = new QuerySolutionMap();
if ( subject != null ) {
map.add("s", subject);
}
if ( predicate != null ) {
map.add("p", predicate);
}
if ( object != null ) {
map.add("o", object);
}
query.setOffset(offset);
query.setLimit(limit);
Model triples = ModelFactory.createDefaultModel();
DatasetWrapper dw = getDatasetWrapper();
try {
Dataset d = dw.getDataset();
try (QueryExecution qexec = QueryExecutionFactory.create(query, d, map)) {
qexec.execConstruct(triples);
}
return triples;
} finally {
dw.close();
}
}
@Override
public void close() {
// nothing
@ -620,5 +688,4 @@ public abstract class RDFServiceJena extends RDFServiceImpl implements RDFServic
protected QueryExecution createQueryExecution(String queryString, Query q, Dataset d) {
return QueryExecutionFactory.create(q, d);
}
}

181
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/jena/sdb/RDFServiceSDB.java
View file

@ -4,20 +4,34 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.sdb;
import java.io.IOException;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import javax.sql.DataSource;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.jena.datatypes.RDFDatatype;
import org.apache.jena.datatypes.TypeMapper;
import org.apache.jena.datatypes.xsd.XSDDatatype;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.graph.Triple;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryExecution;
import org.apache.jena.query.QueryExecutionFactory;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.sdb.SDBFactory;
import org.apache.jena.sdb.Store;
import org.apache.jena.sdb.StoreDesc;
import org.apache.jena.sdb.layout2.NodeLayout2;
import org.apache.jena.sdb.layout2.ValueType;
import org.apache.jena.sdb.sql.SDBConnection;
import edu.cornell.mannlib.vitro.webapp.dao.jena.DatasetWrapper;
@ -28,6 +42,8 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.RDFServiceJena;
import org.apache.jena.sdb.store.DatabaseType;
import org.apache.jena.sdb.store.LayoutType;
public class RDFServiceSDB extends RDFServiceJena implements RDFService {
@ -147,8 +163,118 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
// This used to execute against the default model if the query included an OPTIONAL
// However, in recent Jena this turns out to be much slower than executing against the dataset directly
}
@Override
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
if (LayoutType.LayoutTripleNodesHash.equals(storeDesc.getLayout())) {
if (DatabaseType.MySQL.equals(storeDesc.getDbType()) ||
DatabaseType.PostgreSQL.equals(storeDesc.getDbType())) {
SDBConnection sdbConn = getSDBConnection();
try {
String whereClause = makeWhereClause(subject, predicate, object);
Statement stmt = sdbConn.getSqlConnection().createStatement();
ResultSet rs = stmt.executeQuery("SELECT count(DISTINCT s,p,o) AS tcount FROM Quads" + (StringUtils.isEmpty(whereClause) ? "" : " WHERE " + whereClause));
try {
while (rs.next()) {
return rs.getLong("tcount");
}
} finally {
rs.close();
}
} catch (SQLException sqle) {
throw new RDFServiceException("Unable to retrieve triples", sqle);
} finally {
close(sdbConn);
}
}
} else {
return super.countTriples(subject, predicate, object);
}
return super.countTriples(subject, predicate, object);
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
if (LayoutType.LayoutTripleNodesHash.equals(storeDesc.getLayout())) {
if (DatabaseType.MySQL.equals(storeDesc.getDbType()) ||
DatabaseType.PostgreSQL.equals(storeDesc.getDbType())) {
Model triples = ModelFactory.createDefaultModel();
SDBConnection sdbConn = getSDBConnection();
try {
String whereClause = makeWhereClause(subject, predicate, object);
Statement stmt = sdbConn.getSqlConnection().createStatement();
ResultSet rs = stmt.executeQuery("SELECT \n" +
"N1.lex AS s_lex,\n" +
"N1.lang AS s_lang,\n" +
"N1.datatype AS s_datatype,\n" +
"N1.type AS s_type,\n" +
"N2.lex AS p_lex,\n" +
"N2.lang AS p_lang,\n" +
"N2.datatype AS p_datatype,\n" +
"N2.type AS p_type,\n" +
"N3.lex AS o_lex,\n" +
"N3.lang AS o_lang,\n" +
"N3.datatype AS o_datatype,\n" +
"N3.type AS o_type\n" +
"FROM\n" +
"(SELECT DISTINCT s,p,o FROM Quads" +
(StringUtils.isEmpty(whereClause) ? "" : " WHERE " + whereClause) +
" ORDER BY s,p,o " +
(limit > 0 ? "LIMIT " + limit : "") +
(offset > 0 ? " OFFSET " + offset : "") + ") Q\n" +
"LEFT OUTER JOIN\n" +
"\tNodes AS N1\n" +
"ON ( Q.s = N1.hash )\n" +
"LEFT OUTER JOIN\n" +
"\tNodes AS N2\n" +
"ON ( Q.p = N2.hash )\n" +
"LEFT OUTER JOIN\n" +
"\tNodes AS N3\n" +
"ON ( Q.o = N3.hash )");
try {
while (rs.next()) {
Node subjectNode = makeNode(
rs.getString("s_lex"),
rs.getString("s_datatype"),
rs.getString("s_lang"),
ValueType.lookup(rs.getInt("s_type")));
Node predicateNode = makeNode(
rs.getString("p_lex"),
rs.getString("p_datatype"),
rs.getString("p_lang"),
ValueType.lookup(rs.getInt("p_type")));
Node objectNode = makeNode(
rs.getString("o_lex"),
rs.getString("o_datatype"),
rs.getString("o_lang"),
ValueType.lookup(rs.getInt("o_type")));
triples.add(
triples.asStatement(Triple.create(subjectNode, predicateNode, objectNode))
);
}
} finally {
rs.close();
}
} catch (SQLException sqle) {
throw new RDFServiceException("Unable to retrieve triples", sqle);
} finally {
close(sdbConn);
}
return triples;
}
}
return super.getTriples(subject, predicate, object, limit, offset);
}
@Override
public void close() {
if (conn != null) {
try {
@ -159,4 +285,55 @@ public class RDFServiceSDB extends RDFServiceJena implements RDFService {
}
}
// Copied from Jena SQLBridge2
private static Node makeNode(String lex, String datatype, String lang, ValueType vType) {
switch(vType) {
case BNODE:
return NodeFactory.createBlankNode(lex);
case URI:
return NodeFactory.createURI(lex);
case STRING:
return NodeFactory.createLiteral(lex, lang);
case XSDSTRING:
return NodeFactory.createLiteral(lex, XSDDatatype.XSDstring);
case INTEGER:
return NodeFactory.createLiteral(lex, XSDDatatype.XSDinteger);
case DOUBLE:
return NodeFactory.createLiteral(lex, XSDDatatype.XSDdouble);
case DATETIME:
return NodeFactory.createLiteral(lex, XSDDatatype.XSDdateTime);
case OTHER:
RDFDatatype dt = TypeMapper.getInstance().getSafeTypeByName(datatype);
return NodeFactory.createLiteral(lex, dt);
default:
log.warn("Unrecognized: (" + lex + ", " + lang + ", " + vType + ")");
return NodeFactory.createLiteral("UNRECOGNIZED");
}
}
private String makeWhereClause(RDFNode subject, RDFNode predicate, RDFNode object) {
StringBuilder whereClause = new StringBuilder();
if (subject != null) {
if (whereClause.length() > 0) {
whereClause.append(" AND ");
}
whereClause.append("s=").append(NodeLayout2.hash(subject.asNode()));
}
if (predicate != null) {
if (whereClause.length() > 0) {
whereClause.append(" AND ");
}
whereClause.append("p=").append(NodeLayout2.hash(predicate.asNode()));
}
if (object != null) {
if (whereClause.length() > 0) {
whereClause.append(" AND ");
}
whereClause.append("o=").append(NodeLayout2.hash(object.asNode()));
}
return whereClause.length() > 0 ? whereClause.toString() : null;
}
}

11
api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/impl/logging/LoggingRDFService.java
View file

@ -14,6 +14,7 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import org.apache.jena.rdf.model.RDFNode;
/**
* This RDFService wrapper adds instrumentation to the time-consuming methods of
@ -182,6 +183,16 @@ public class LoggingRDFService implements RDFService {
return innerService.manufactureChangeSet();
}
@Override
public long countTriples(RDFNode subject, RDFNode predicate, RDFNode object) throws RDFServiceException {
return innerService.countTriples(subject, predicate, object);
}
@Override
public Model getTriples(RDFNode subject, RDFNode predicate, RDFNode object, long limit, long offset) throws RDFServiceException {
return innerService.getTriples(subject, predicate, object, limit, offset);
}
@Override
public void close() {
innerService.close();

2
api/src/main/java/org/linkeddatafragments/views/HtmlTriplePatternFragmentWriterImpl.java
View file

@ -106,7 +106,7 @@ public class HtmlTriplePatternFragmentWriterImpl extends TriplePatternFragmentWr
// Calculate start and end triple number
Long start = ((tpfRequest.getPageNumber() - 1) * fragment.getMaxPageSize()) + 1;
data.put("start", start);
data.put("end", start + (triples.size() < fragment.getMaxPageSize() ? triples.size() : fragment.getMaxPageSize()));
data.put("end", (start - 1) + (triples.size() < fragment.getMaxPageSize() ? triples.size() : fragment.getMaxPageSize()));
// Compose query object
Map query = new HashMap();

174
api/src/main/java/org/vivoweb/linkeddatafragments/datasource/rdfservice/RDFServiceBasedRequestProcessorForTPFs.java
View file

@ -6,6 +6,9 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
import edu.cornell.mannlib.vitro.webapp.rdfservice.ResultSetConsumer;
import org.apache.jena.atlas.io.StringWriterI;
import org.apache.jena.graph.Node;
import org.apache.jena.graph.NodeFactory;
import org.apache.jena.graph.Triple;
import org.apache.jena.query.Dataset;
import org.apache.jena.query.Query;
import org.apache.jena.query.QueryExecution;
@ -19,6 +22,8 @@ import org.apache.jena.rdf.model.Literal;
import org.apache.jena.rdf.model.Model;
import org.apache.jena.rdf.model.ModelFactory;
import org.apache.jena.rdf.model.RDFNode;
import org.apache.jena.rdf.model.Statement;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.jena.riot.out.NodeFormatter;
import org.apache.jena.riot.out.NodeFormatterTTL;
import org.apache.jena.tdb.TDBFactory;
@ -74,6 +79,32 @@ public class RDFServiceBasedRequestProcessorForTPFs
return sw.toString();
}
private Node skolemize(Node node) {
if (node != null && node.isBlank()) {
return NodeFactory.createURI("bnode://" + node.getBlankNodeLabel());
}
return node;
}
private RDFNode deskolemize(RDFNode node) {
if (node == null) {
return null;
}
if (node.isResource()) {
String uri = node.asResource().getURI();
if (uri != null && uri.startsWith("bnode://")) {
String bnodeId = uri.substring(8);
return ModelFactory.createDefaultModel().asRDFNode(
NodeFactory.createBlankNode(bnodeId)
);
}
}
return node;
}
@Override
protected ILinkedDataFragment createFragment(
final ITriplePatternElement<RDFNode,String,String> subject,
@ -82,100 +113,51 @@ public class RDFServiceBasedRequestProcessorForTPFs
final long offset,
final long limit )
{
StringBuilder whereClause = new StringBuilder();
StringBuilder filter = new StringBuilder();
StringBuilder orderBy = new StringBuilder();
if ( ! subject.isVariable() ) {
appendNode(whereClause.append(' '), subject.asConstantTerm());
} else {
whereClause.append(" ?s");
if (filter.length() > 0) { filter.append(" && "); }
filter.append("!isBlank(?s)");
orderBy.append(" ?s");
}
if ( ! predicate.isVariable() ) {
appendNode(whereClause.append(' '), predicate.asConstantTerm());
} else {
whereClause.append(" ?p");
if (filter.length() > 0) { filter.append(" && "); }
filter.append("!isBlank(?p)");
orderBy.append(" ?p");
}
if ( ! object.isVariable() ) {
appendNode(whereClause.append(' '), object.asConstantTerm());
} else {
whereClause.append(" ?o");
if (filter.length() > 0) { filter.append(" && "); }
filter.append("!isBlank(?o)");
orderBy.append(" ?o");
}
StringBuilder constructQuery = new StringBuilder();
constructQuery.append("CONSTRUCT { ");
constructQuery.append(whereClause.toString());
constructQuery.append(" } WHERE { ");
constructQuery.append(whereClause.toString()).append(" . ");
if (filter.length() > 0) {
constructQuery.append(" FILTER(").append(filter.toString()).append(")");
}
constructQuery.append(" }");
if (orderBy.length() > 0) {
constructQuery.append(" ORDER BY").append(orderBy.toString());
}
if (limit > 0) {
constructQuery.append(" LIMIT ").append(limit);
}
if (offset > 0) {
constructQuery.append(" OFFSET ").append(offset);
}
Model triples = ModelFactory.createDefaultModel();
try {
rdfService.sparqlConstructQuery(constructQuery.toString(), triples);
RDFNode nSubject = subject.isVariable() ? null : deskolemize(subject.asConstantTerm());
RDFNode nPredicate = predicate.isVariable() ? null : deskolemize(predicate.asConstantTerm());
RDFNode nObject = object.isVariable() ? null : deskolemize(object.asConstantTerm());
Model triples = rdfService.getTriples(nSubject, nPredicate, nObject, limit, offset);
if (triples == null || triples.isEmpty()) {
return createEmptyTriplePatternFragment();
}
if (triples.size() > 0) {
Model replacedBlankNodes = ModelFactory.createDefaultModel();
StmtIterator iter = triples.listStatements();
while (iter.hasNext()) {
Statement oldStmt = iter.next();
Triple t = oldStmt.asTriple();
replacedBlankNodes.add(
replacedBlankNodes.asStatement(
new Triple(
skolemize(t.getSubject()),
skolemize(t.getPredicate()),
skolemize(t.getObject())
)
)
);
}
triples = replacedBlankNodes;
}
long size = triples.size();
long estimate = -1;
estimate = rdfService.countTriples(nSubject, nPredicate, nObject);
// No estimate or incorrect
if (estimate < offset + size) {
estimate = (size == limit) ? offset + size + 1 : offset + size;
}
// create the fragment
final boolean isLastPage = ( estimate < offset + limit );
return createTriplePatternFragment( triples, estimate, isLastPage );
} catch (RDFServiceException e) {
return createEmptyTriplePatternFragment();
}
if (triples.isEmpty()) {
return createEmptyTriplePatternFragment();
}
// Try to get an estimate
long size = triples.size();
long estimate = -1;
StringBuilder count = new StringBuilder();
count.append("SELECT (COUNT(*) AS ?count) WHERE { ");
count.append(whereClause.toString());
count.append(" . ");
if (filter.length() > 0) {
count.append(" FILTER(").append(filter.toString()).append(") ");
}
count.append(" }");
try {
CountConsumer countConsumer = new CountConsumer();
rdfService.sparqlSelectQuery(count.toString(), countConsumer);
estimate = countConsumer.estimate;
} catch (RDFServiceException e) {
return createEmptyTriplePatternFragment();
}
// No estimate or incorrect
if (estimate < offset + size) {
estimate = (size == limit) ? offset + size + 1 : offset + size;
}
// create the fragment
final boolean isLastPage = ( estimate < offset + limit );
return createTriplePatternFragment( triples, estimate, isLastPage );
}
} // end of class Worker
@ -186,16 +168,4 @@ public class RDFServiceBasedRequestProcessorForTPFs
*/
public RDFServiceBasedRequestProcessorForTPFs() {
}
class CountConsumer extends ResultSetConsumer {
public long estimate = -1;
@Override
protected void processQuerySolution(QuerySolution qs) {
if (estimate == -1) {
Literal literal = qs.getLiteral("count");
estimate = literal.getLong();
}
}
}
}

2
api/src/main/java/org/vivoweb/linkeddatafragments/views/HtmlTriplePatternFragmentWriterImpl.java
View file

@ -124,7 +124,7 @@ public class HtmlTriplePatternFragmentWriterImpl extends TriplePatternFragmentWr
// Calculate start and end triple number
Long start = ((tpfRequest.getPageNumber() - 1) * fragment.getMaxPageSize()) + 1;
data.put("start", start);
data.put("end", start + (triples.size() < fragment.getMaxPageSize() ? triples.size() : fragment.getMaxPageSize()));
data.put("end", (start - 1) + (triples.size() < fragment.getMaxPageSize() ? triples.size() : fragment.getMaxPageSize()));
// Compose query object
Map query = new HashMap();

64
api/src/main/resources/org/vivoweb/linkeddatafragments/servlet/ldf-client-ui-packaged.js Normal file
View file

File diff suppressed because one or more lines are too long

183
api/src/main/resources/org/vivoweb/linkeddatafragments/servlet/ldf-client-worker.js Normal file
View file

File diff suppressed because one or more lines are too long

549
api/src/main/resources/org/vivoweb/linkeddatafragments/servlet/ldf-client.css Normal file
View file

@ -0,0 +1,549 @@
/*! @license MIT ©2014–2016 Ruben Verborgh, Ghent University – imec */
html {
font-size: 11pt;
background: #f6f6f6;
}
html, input, textarea, button, pre {
font: 1em/1.4 'Open Sans', Verdana, Arial, sans-serif;
-webkit-text-size-adjust: none;
}
* {
outline: none !important;
}
body {
max-width: 650px;
margin: 0 auto;
padding: 1em 3em;
background: white;
box-shadow: 2px 2px 15px 0px rgba(50, 50, 50, 0.75);
}
a, button, label {
font-weight: bold;
}
a, button, input:hover, label {
color: #be1622;
cursor: pointer;
text-decoration: none;
}
a:hover {
color: #be1622;
text-decoration: underline;
}
button {
font-weight: bold;
color: black;
background-color: #f6f6f6;
border: 1px solid #999999;
border-radius: 3px;
margin: 0;
padding: 5px 8px;
cursor: pointer;
}
button:hover {
background-color: #ececec;
}
button:active {
margin: 1px -1px -1px 1px;
}
input {
padding: 1px 5px;
font-size: 10pt;
}
label:after {
content: ":";
}
header h1 {
margin: .3em 0 .1em;
font-size: 1.95em;
}
header p {
font-weight: bold;
margin: 0 0 1.5em;
}
header a {
color: inherit;
}
header .logo {
float: right;
}
header img {
margin: -9px -11px 0 0;
height: 81px;
}
main {
clear: both;
}
fieldset {
border: none;
padding: 0;
margin: 0;
}
.ldf-client, .ldf-client * {
max-width: 100%;
}
ul {
list-style: none;
padding: 0;
margin: 0;
}
fieldset > ul > li {
margin: .3em 0;
clear: both;
}
fieldset > ul > li:after {
content: "";
display: table;
clear: both;
}
select {
width: 430px;
}
.timing {
float: right;
font-size: 10pt;
padding: .7em 0;
}
.chosen {
float: right;
}
input.datetime {
float: right;
font-size: 13px;
border: 1px solid #999999;
border-radius: 3px;
}
.details {
display: none;
}
.details-toggle {
display: block;
width: 24px;
height: 24px;
margin: 0 -35px;
float: right;
cursor: pointer;
background: url(../images/memento.svg) no-repeat center center / contain;
opacity: .8;
}
.details-toggle:hover, .details-toggle.enabled {
opacity: 1;
}
.details-toggle.enabled {
background-image: url(../images/memento.svg#active);
width: 25px; /* for Chrome */
}
textarea {
box-sizing: border-box;
min-height: 11em;
width: 100%;
padding-left: 1em;
border: none;
cursor: pointer;
font-size: .95em;
background-color: transparent;
}
.results {
height: 23.5em;
font-size: 10pt;
margin: .5em 0 1em;
}
.results a {
font-weight: normal;
color: inherit;
}
.result {
width: calc(100% - 2px);
}
.result dl {
border: 1px solid #be1622;
margin: 0 0 1em;
border-radius: 5px;
}
.result dl:hover {
color: white;
background-color: #be1622;
}
.result dl:after {
content: "";
display: table;
clear: both;
}
.result dt {
float: left;
clear: left;
background-color: #be1622;
color: white;
font-style: normal;
font-weight: bold;
padding: 0 .5em;
margin: 0 .5em 0 0;
}
.result dd {
margin: 0 .5em 0 1em;
}
.result dt:last-of-type {
border-radius: 0 0 0 5px;
}
.results .text {
white-space: pre-wrap;
margin-right: -999999px; /* avoid that long URLs alter body width */
}
.log {
color: #666;
height: 6.3em;
font-size: 9pt;
line-height: 1.6;
margin: .5em -999999px 1em 0;
}
.log p {
margin: 0;
}
.log a {
font-weight: normal;
}
.results, .log {
width: 100%;
overflow: scroll;
-webkit-overflow-scrolling: touch;
position: relative;
contain: layout;
}
footer {
font-size: .8em;
}
@media (max-width: 725px) {
body {
padding: .1em 1em;
font-size: 10pt;
}
h1 {
font-size: 1.4em;
}
header img {
height: 70px;
}
header p {
margin: 0 0 .5em;
}
select {
width: 350px;
}
.chosen, input.datetime {
float: none;
margin-bottom: 0;
}
.details-toggle {
display: inline-block;
margin: 0 0 -8px 10px;
}
.log {
max-width: 90%;
}
}
@media (max-width: 550px) {
h1 {
ine-height: 1.2;
}
header p, .logo {
display: none;
}
label {
display: block;
}
select {
width: 250px;
}
}
/*
* Styling below derived from
* Chosen, a Select Box Enhancer for jQuery and Prototype
* by Patrick Filler for Harvest, http://getharvest.com
*
* Version 1.4.2
* Full source at https://github.com/harvesthq/chosen
* Copyright (c) 2011-2015 Harvest http://getharvest.com
*
* MIT License, https://github.com/harvesthq/chosen/blob/master/LICENSE.md
*/
.chosen {
position: relative;
display: inline-block;
vertical-align: middle;
font-size: 13px;
zoom: 1;
*display: inline;
user-select: none;
}
.chosen * {
box-sizing: border-box;
}
.chosen .chosen-drop {
position: absolute;
top: 100%;
left: -9999px;
z-index: 1010;
width: 100%;
border: 1px solid #999999;
border-top: 0;
background: #fff;
}
.chosen.chosen-with-drop .chosen-drop {
left: 0;
}
.chosen a {
cursor: pointer;
}
.chosen-single .chosen-single {
position: relative;
display: block;
overflow: hidden;
padding: 0 0 0 8px;
height: 25px;
border: 1px solid #999999;
border-radius: 3px;
background-color: #fff;
text-decoration: none;
white-space: nowrap;
line-height: 24px;
font-weight: normal;
color: inherit;
}
.chosen-single .chosen-single span {
display: block;
overflow: hidden;
margin-right: 26px;
text-overflow: ellipsis;
white-space: nowrap;
}
.chosen-single .chosen-single-with-deselect span {
margin-right: 38px;
}
.chosen-single .chosen-single abbr {
position: absolute;
top: 6px;
right: 26px;
display: block;
width: 12px;
height: 12px;
background: url('../images/chosen-sprite.png') -42px 1px no-repeat;
font-size: 1px;
}
.chosen-single .chosen-single abbr:hover {
background-position: -42px -10px;
}
.chosen-single .chosen-single div {
position: absolute;
top: 0;
right: 0;
display: block;
width: 18px;
height: 100%;
}
.chosen-single .chosen-single div b {
display: block;
width: 100%;
height: 100%;
background: url('../images/chosen-sprite.png') no-repeat 0px 2px;
}
.chosen-single .chosen-search {
position: relative;
z-index: 1010;
margin: 0;
padding: 3px 4px;
white-space: nowrap;
}
.chosen-single .chosen-search input[type="text"] {
margin: 1px 0;
padding: 4px 20px 4px 5px;
width: 100%;
height: auto;
outline: 0;
border: 1px solid #999999;
background: white url('../images/chosen-sprite.png') no-repeat 100% -20px;
background: url('../images/chosen-sprite.png') no-repeat 100% -20px;
font-size: 1em;
line-height: normal;
border-radius: 0;
}
.chosen-single .chosen-drop {
margin-top: -1px;
border-radius: 0 0 4px 4px;
background-clip: padding-box;
}
.chosen-single.chosen-single-nosearch .chosen-search {
position: absolute;
left: -9999px;
}
.chosen-results {
position: relative;
overflow-x: hidden;
overflow-y: auto;
margin: 0 4px 4px 0;
padding: 0 0 0 4px;
max-height: 240px;
-webkit-overflow-scrolling: touch;
}
.chosen-results li {
display: none;
margin: 0;
padding: 5px 6px;
list-style: none;
line-height: 15px;
word-wrap: break-word;
-webkit-touch-callout: none;
}
li.active-result {
display: list-item;
cursor: pointer;
}
li.highlighted {
background-color: #be1622;
color: #fff;
}
li.no-results {
display: none;
}
.chosen-results li em {
font-style: normal;
text-decoration: underline;
}
.chosen-choices {
position: relative;
overflow: hidden;
margin: 0;
padding: 0 5px;
width: 100%;
height: auto !important;
height: 1%;
border: 1px solid #999999;
border-radius: 3px;
background-color: #fff;
cursor: text;
}
.chosen-choices li {
float: left;
list-style: none;
}
li.search-field {
margin: 0;
padding: 0;
white-space: nowrap;
}
li.search-field input[type="text"] {
margin: 1px 0;
padding: 0;
height: 20px;
outline: 0;
border: 0 !important;
font-size: 100%;
line-height: normal;
border-radius: 0;
}
li.search-choice {
position: relative;
margin: 1px 5px 1px 0;
padding: 3px 20px 3px 5px;
max-width: 100%;
border-radius: 3px;
background-color: #f6f6f6;
border: 1px solid #cacaca;
line-height: 13px;
cursor: default;
}
li.search-choice:hover {
background-color: #ececec;
}
li.search-choice span {
word-wrap: break-word;
}
li.search-choice .search-choice-close {
position: absolute;
top: 4px;
right: 3px;
display: block;
width: 12px;
height: 12px;
background: url('../images/chosen-sprite.png') -42px 1px no-repeat;
font-size: 1px;
}
li.search-choice .search-choice-close:hover {
background-position: -42px -10px;
}
li.search-choice-focus {
background: #d4d4d4;
}
li.search-choice-focus .search-choice-close {
background-position: -42px -10px;
}
.chosen-results {
margin: 0;
padding: 0;
}
.chosen-drop .result-selected {
display: list-item;
cursor: default;
}
.chosen-active.chosen-with-drop .chosen-single {
border: 1px solid #999999;
border-bottom-right-radius: 0;
border-bottom-left-radius: 0;
}
.chosen-active.chosen-with-drop .chosen-single div {
border-left: none;
}
.chosen-active.chosen-with-drop .chosen-single div b {
background-position: -18px 2px;
}
.chosen-active .chosen-choices {
z-index: 100;
max-height: 500px;
}
@media only screen and (-webkit-min-device-pixel-ratio: 1.5), only screen and (min-resolution: 144dpi), only screen and (min-resolution: 1.5dppx) {
.chosen-rtl .chosen-search input[type="text"],
.chosen-single .chosen-single abbr,
.chosen-single .chosen-single div b,
.chosen-single .chosen-search input[type="text"],
.chosen-multi .chosen-choices .search-choice .search-choice-close,
.chosen .chosen-results-scroll-down span,
.chosen .chosen-results-scroll-up span {
background-image: url('../images/chosen-sprite@2x.png') !important;
background-size: 52px 37px !important;
background-repeat: no-repeat !important;
}
}
.results {
line-height: 1.5em;
}

2
api/src/main/resources/org/vivoweb/linkeddatafragments/servlet/style.css
View file

@ -85,7 +85,7 @@ legend {
}
label {
width: 100px;
width: 200px;
display: block;
float: left;
clear: both;

43
api/src/main/resources/tpf/client.ftl.html Normal file
View file

@ -0,0 +1,43 @@
<h2>Query from your browser</h2>
<p>
Your browser executes these queries locally
using <a href="http://linkeddatafragments.org/in-depth/#tpf" target="_blank">Triple Pattern Fragments</a>.
</p>
<fieldset class="ldf-client">
<ul>
<li>
<label>Query</label>
<textarea class="queryText"></textarea>
</li>
<li>
<button class="start" style="display: inline-block;">Execute query</button>
<button class="stop" style="display: none;">Stop execution</button>
<span class="timing"></span>
</li>
<!-- example queries -->
<li>
<label>Query results</label>
<div class="results"><div class="text"></div><div class="scrollRunway" style="position: absolute; height: 1px; width: 1px; transition: transform 0.2s; -webkit-transition: transform 0.2s; transform: translate(0px, 0px);"> </div></div>
</li>
</ul>
</fieldset>
<link rel="stylesheet" href="${ assetsPath }ldf-client.css">
<script>
var ldfAssetsPath = '${ assetsPath }';
</script>
<script src="${ assetsPath }ldf-client-ui-packaged.js"></script>
<script>jQuery(function($){$(".ldf-client").queryui(
{
"datasources": [
<#if datasources??>
<#list datasources?keys as datasourceName>
{
"name": "${datasources[datasourceName].getTitle() }",
"url": "${homePath}/${datasourceName}"
}<#if datasourceName?has_next>, </#if>
</#list>
</#if>
]
}
)})</script>

122
api/src/main/resources/tpf/examples.ftl.html Normal file
View file

@ -0,0 +1,122 @@
<li>
<label>Example queries</label>
<select class="query" style="display: none;"><option></option><option value="SELECT ?movie ?title ?name
WHERE {
?movie dbpedia-owl:starring [ rdfs:label &quot;Brad Pitt&quot;@en ];
rdfs:label ?title;
dbpedia-owl:director [ rdfs:label ?name ].
FILTER LANGMATCHES(LANG(?title), &quot;EN&quot;)
FILTER LANGMATCHES(LANG(?name), &quot;EN&quot;)
}">Directors of movies starring Brad Pitt</option><option value="SELECT DISTINCT ?entity WHERE {
?entity a dbpedia-owl:Airport;
dbpprop:cityServed dbpedia:Italy.
}">Airports in Italy</option><option value="SELECT ?name ?deathDate WHERE {
?person a dbpedia-owl:Artist;
rdfs:label ?name;
dbpedia-owl:birthPlace [ rdfs:label &quot;York&quot;@en ].
FILTER LANGMATCHES(LANG(?name), &quot;EN&quot;)
OPTIONAL { ?person dbpprop:dateOfDeath ?deathDate. }
}">Artists born in York</option><option value="CONSTRUCT {
?artist a dbpedia-owl:Artist.
?artist dbpedia-owl:birthDate ?date.
}
WHERE {
?artist dbpedia-owl:influencedBy dbpedia:Pablo_Picasso.
?artist a dbpedia-owl:Artist.
?artist dbpedia-owl:birthDate ?date.
}">Artists influenced by Picasso</option><option value="SELECT DISTINCT ?book ?author
WHERE {
?book rdf:type dbpedia-owl:Book;
dbpedia-owl:author ?author.
}
LIMIT 100">Authors of books</option><option value="SELECT ?award WHERE {
?award a dbpedia-owl:Award;
dbpprop:country [ dbpedia-owl:language dbpedia:Dutch_language ].
}">Award ceremonies in Dutch speaking countries</option><option value="SELECT DISTINCT ?performer ?name WHERE {
?work dbpedia-owl:writer dbpedia:Michael_Jackson;
dbpedia-owl:musicalArtist ?performer.
OPTIONAL {
?performer rdfs:label ?name.
FILTER LANGMATCHES(LANG(?name), &quot;EN&quot;)
}
}">Bands Michael Jackson wrote a song for</option><option value="SELECT ?software ?company WHERE {
?software dbpedia-owl:developer ?company.
?company dbpedia-owl:locationCountry [ rdfs:label &quot;Belgium&quot;@en ].
}">Belgian software</option><option value="PREFIX yago: &lt;http://dbpedia.org/class/yago/&gt;
SELECT ?person
WHERE {
?person a yago:Carpenters, yago:PeopleExecutedByCrucifixion.
}">Carpenters killed by crucifixion</option><option value="PREFIX dbpedia-owl: &lt;http://dbpedia.org/ontology/&gt;
SELECT ?actor ?cause
WHERE {
?actor dbpedia-owl:deathCause ?cause.
?actor dc:subject &lt;http://dbpedia.org/resource/Category:American_male_film_actors&gt;
}">Death causes of male American actors</option><option value="SELECT ?dessert ?fruit
WHERE {
?dessert dbpedia-owl:type &lt;http://dbpedia.org/resource/Dessert&gt;;
dbpedia-owl:ingredient ?fruit.
?fruit dbpedia-owl:kingdom &lt;http://dbpedia.org/resource/Plant&gt;.
}">Desserts made with plants</option><option value="SELECT DISTINCT ?device WHERE {
dbpedia:Raspberry_Pi dbpprop:os ?operatingSystem.
?device a dbpedia-owl:Device;
dbpprop:os ?operatingSystem.
FILTER (!(?device = dbpedia:Raspberry_Pi))
}">Devices with the same OS as the Raspberry Pi</option><option value="SELECT DISTINCT ?entity ?event
WHERE {
?entity a dbpedia-owl:Event;
rdfs:label ?event;
?predicate &lt;http://dbpedia.org/resource/Trentino&gt; .
FILTER(langMatches(lang(?event), &quot;EN&quot;))
}">Events that took place in the Trentino region</option><option value="SELECT ?titleEng ?title
WHERE {
?movie dbpprop:starring [ rdfs:label &quot;Natalie Portman&quot;@en ];
rdfs:label ?titleEng, ?title.
FILTER LANGMATCHES(LANG(?titleEng), &quot;EN&quot;)
FILTER (!LANGMATCHES(LANG(?title), &quot;EN&quot;))
}">Foreign titles of Natalie Portman movies</option><option value="SELECT ?indDish ?belDish ?ingredient
WHERE {
?indDish a dbpedia-owl:Food;
dbpedia-owl:origin dbpedia:India;
dbpedia-owl:ingredient ?ingredient.
?belDish a dbpedia-owl:Food;
dbpedia-owl:origin dbpedia:Belgium;
dbpedia-owl:ingredient ?ingredient.
}">Indian dishes that have ingredients in common with Belgian dishes</option><option value="SELECT DISTINCT ?artist ?band ?bandName WHERE {
{ &lt;http://dbpedia.org/resource/Queen_(band)&gt; dbpedia-owl:bandMember ?artist. }
UNION
{ &lt;http://dbpedia.org/resource/Queen_(band)&gt; dbpedia-owl:formerBandMember ?artist. }
?band dbpedia-owl:formerBandMember ?artist;
rdfs:label ?bandName.
FILTER (?band != &lt;http://dbpedia.org/resource/Queen_(band)&gt;)
}">Other bands of Queen members</option><option value="SELECT DISTINCT ?person
WHERE {
dbpedia:Jesus dc:subject ?common.
?person a foaf:Person;
dc:subject ?common.
}
LIMIT 1000">People who have something in common with Jesus</option><option value="SELECT ?place ?relation
WHERE {
?place rdf:type dbpedia-owl:Settlement;
?relation dbpedia:Barack_Obama;
}">Places that have something to do with Barack Obama</option><option value="SELECT ?clubName ?playerName WHERE {
?club a dbpedia-owl:SoccerClub;
dbpedia-owl:ground ?city;
rdfs:label ?clubName.
?player dbpedia-owl:team ?club;
dbpedia-owl:birthPlace ?city;
rdfs:label ?playerName.
?city dbpedia-owl:country dbpedia:Spain.
FILTER LANGMATCHES(LANG(?clubName), &quot;EN&quot;)
FILTER LANGMATCHES(LANG(?playerName), &quot;EN&quot;)
}">Soccer players born in their club's city</option><option value="SELECT ?entity ?label ?comment
WHERE {
?entity a dbpedia-owl:MythologicalFigure;
rdfs:label ?label;
dc:subject &lt;http://dbpedia.org/resource/Category:Women_in_Greek_mythology&gt;;
rdfs:comment ?comment
FILTER(langMatches(lang(?label), &quot;EN&quot;))
FILTER(langMatches(lang(?comment), &quot;EN&quot;))
}">Women in Greek mythology</option></select><div class="chosen chosen-single" style="width: 430px;" title=""><a class="chosen-single chosen-default" tabindex="-1"><span> </span><div><b></b></div></a><div class="chosen-drop"><div class="chosen-search"><input type="text" autocomplete="off"></div><ul class="chosen-results"></ul></div></div>
</li>

3
api/src/main/resources/tpf/index.ftl.html
View file

@ -12,10 +12,9 @@
</#list>
</#if>
</dl>
<p>The current dataset <em class="dataset">index</em> contains metadata about these datasets.</p>
</div>
<#include "fragment.ftl.html">
<#include "client.ftl.html">
</#macro>
<@display_page/>