From 2f78e0976505a5eb1a4b625f40881d791f81e317 Mon Sep 17 00:00:00 2001 From: j2blake Date: Mon, 4 Nov 2013 15:02:15 -0500 Subject: [PATCH] VIVO-452 Revise SparqlQueryDataGetter to use RDFService if possible. --- .../search/indexing/IndexWorkerThread.java | 16 +- .../FakeApplicationOntologyService.java | 6 +- .../dataGetter/SparqlQueryDataGetter.java | 220 ++++++++++-------- .../dataGetter/SparqlQueryDataGetterTest.java | 4 +- 4 files changed, 142 insertions(+), 104 deletions(-) diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java index 6008d330f..57caebb5d 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexWorkerThread.java @@ -12,8 +12,10 @@ import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.search.IndexingException; import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument; +import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; -class IndexWorkerThread extends Thread{ +class IndexWorkerThread extends VitroBackgroundThread{ + private static final Log log = LogFactory.getLog(IndexWorkerThread.class); protected final int threadNum; protected IndividualToSolrDocument individualToSolrDoc; @@ -21,7 +23,6 @@ class IndexWorkerThread extends Thread{ protected final Iterator individualsToIndex; protected boolean stopRequested = false; - private Log log = LogFactory.getLog(IndexWorkerThread.class); private static AtomicLong countCompleted= new AtomicLong(); private static AtomicLong countToIndex= new AtomicLong(); private static long starttime = 0; @@ -38,7 +39,8 @@ class IndexWorkerThread extends Thread{ } public void run(){ - + setWorkLevel(WorkLevel.WORKING, "indexing " + individualsToIndex + " individuals"); + while( ! stopRequested ){ //do the actual indexing work @@ -48,7 +50,9 @@ class IndexWorkerThread extends Thread{ // done so shut this thread down. stopRequested = true; } - log.debug("Worker number " + threadNum + " exiting."); + setWorkLevel(WorkLevel.IDLE); + + log.debug("Worker number " + threadNum + " exiting."); } protected void addDocsToIndex() { @@ -82,8 +86,8 @@ class IndexWorkerThread extends Thread{ } } }catch(Throwable th){ - //on tomcat shutdown odd exceptions get thrown and log can be null - if( log != null && ! stopRequested ) + //on tomcat shutdown odd exceptions get thrown + if( ! stopRequested ) log.error("Exception during index building",th); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/services/shortview/FakeApplicationOntologyService.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/services/shortview/FakeApplicationOntologyService.java index 794837d24..8096c799c 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/services/shortview/FakeApplicationOntologyService.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/services/shortview/FakeApplicationOntologyService.java @@ -479,10 +479,10 @@ public class FakeApplicationOntologyService { @Override public Map getData(Map pageData) { - Map parms = new HashMap(); - parms.put("uri", new String[] { individualUri }); + Map parms = new HashMap<>(); + parms.put("uri", individualUri); - return doQuery(parms, getModel(ctx, vreq, null)); + return super.getData(parms); } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetter.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetter.java index b1533acb7..e7ded50e9 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetter.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetter.java @@ -29,8 +29,18 @@ import com.hp.hpl.jena.shared.Lock; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary; +import edu.cornell.mannlib.vitro.webapp.dao.jena.QueryUtils; + +public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ + private final static Log log = LogFactory.getLog(SparqlQueryDataGetter.class); + + private static final String queryPropertyURI = "<" + DisplayVocabulary.QUERY + ">"; + private static final String saveToVarPropertyURI= "<" + DisplayVocabulary.SAVE_TO_VAR+ ">"; + private static final String queryModelPropertyURI= "<" + DisplayVocabulary.QUERY_MODEL+ ">"; + + public static final String defaultVarNameForResults = "results"; + private static final String defaultTemplate = "menupage--defaultSparql.ftl"; -public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ String dataGetterURI; String queryText; String saveToVar; @@ -38,11 +48,6 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ VitroRequest vreq; ServletContext context; - - final static Log log = LogFactory.getLog(SparqlQueryDataGetter.class); - //default template - private final static String defaultTemplate = "menupage--defaultSparql.ftl"; - /** * Constructor with display model and data getter URI that will be called by reflection. */ @@ -50,21 +55,11 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ this.configure(vreq, displayModel,dataGetterURI); } - @Override - public Map getData(Map pageData) { - // Merge the pageData with the request parameters. PageData overrides - Map merged = new HashMap(); - merged.putAll(vreq.getParameterMap()); - for (String key: pageData.keySet()) { - merged.put(key, new String[] {String.valueOf(pageData.get(key))}); - } - return doQuery( merged, getModel(context, vreq, modelURI)); - } - - /** + /** * Configure this instance based on the URI and display model. */ - protected void configure(VitroRequest vreq, Model displayModel, String dataGetterURI) { + @SuppressWarnings("hiding") + protected void configure(VitroRequest vreq, Model displayModel, String dataGetterURI) { if( vreq == null ) throw new IllegalArgumentException("VitroRequest may not be null."); if( displayModel == null ) @@ -79,7 +74,6 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ QuerySolutionMap initBindings = new QuerySolutionMap(); initBindings.add("dataGetterURI", ResourceFactory.createResource(this.dataGetterURI)); - int count = 0; Query dataGetterConfigurationQuery = QueryFactory.create(dataGetterQuery) ; displayModel.enterCriticalSection(Lock.READ); try{ @@ -88,7 +82,6 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ ResultSet res = qexec.execSelect(); try{ while( res.hasNext() ){ - count++; QuerySolution soln = res.next(); //query is NOT OPTIONAL @@ -121,56 +114,120 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ } /** - * Do the query and return a result. This is in its own method - * to make testing easy. + * Query to get the definition of the SparqlDataGetter for a given URI. */ - protected Map doQuery(MapparameterMap, Model queryModel){ + private static final String dataGetterQuery = + "PREFIX display: <" + DisplayVocabulary.DISPLAY_NS +"> \n" + + "SELECT ?query ?saveToVar ?queryModel WHERE { \n" + + " ?dataGetterURI "+queryPropertyURI+" ?query . \n" + + " OPTIONAL{ ?dataGetterURI "+saveToVarPropertyURI+" ?saveToVar } \n " + + " OPTIONAL{ ?dataGetterURI "+queryModelPropertyURI+" ?queryModel } \n" + + "}"; - if( this.queryText == null ){ - log.error("no SPARQL query defined for page " + this.dataGetterURI); + + @Override + public Map getData(Map pageData) { + Map merged = mergeParameters(vreq.getParameterMap(), pageData); + + String boundQueryText = bindParameters(queryText, merged); + + if (modelURI != null) { + return doQueryOnModel(boundQueryText, getModel(context, vreq, modelURI)); + } else { + return doQueryOnRDFService(boundQueryText); + } + } + + /** Merge the pageData with the request parameters. PageData overrides. */ + private Map mergeParameters( + Map parameterMap, Map pageData) { + Map merged = new HashMap<>(); + for (String key: parameterMap.keySet()) { + merged.put(key, parameterMap.get(key)[0]); + } + for (String key: pageData.keySet()) { + merged.put(key, String.valueOf(pageData.get(key))); + } + return merged; + } + + /** + * InitialBindings don't always work, and besides, RDFService doesn't accept + * them. So do a text-based substitution. + * + * This assumes that every parameter is a URI. What if we want to substitute + * a string value? + */ + private String bindParameters(String text, Map merged) { + String bound = text; + for (String key : merged.keySet()) { + bound.replace('?' + key, '<' + merged.get(key) + '>'); + } + log.debug("query after binding parameters: " + bound); + return bound; + } + + /** + * Do the query and return a result. This is in its own method, with + * protected access, to make testing easy. + */ + protected Map doQueryOnRDFService(String q) { + log.debug("Going to RDFService with " + q); + ResultSet results = QueryUtils.getQueryResults(q, vreq); + return assembleMap(parseResults(results)); + } + + /** + * Do the query and return a result. This is in its own method, with + * protected access, to make testing easy. + */ + protected Map doQueryOnModel(String q, Model queryModel){ + log.debug("Going to model " + modelURI + " with " + q); + if (q == null) { + return Collections.emptyMap(); + } + + Query query = makeQuery(q); + if (query == null) { return Collections.emptyMap(); } - //this may throw a SPARQL syntax error - Query query = QueryFactory.create( this.queryText ); - - //build query bindings - QuerySolutionMap initialBindings = createBindings( parameterMap); - - //execute query - List> results = executeQuery( query, queryModel, initialBindings); - - //put results in page data, what key to use for results? - Map rmap = new HashMap(); - //also store the variable name within which results will be returned - rmap.put("variableName", this.saveToVar); - rmap.put(this.saveToVar, results); - //This will be overridden at page level in display model if template specified there - rmap.put("bodyTemplate", defaultTemplate); - return rmap; + return assembleMap(executeQuery( query, queryModel)); } - private List> executeQuery(Query query, Model model, - QuerySolutionMap initialBindings) { - - List> rows = new ArrayList>(); - + private Query makeQuery(String q) { + try { + return QueryFactory.create(q); + } catch (Exception e) { + log.error("Failed to build a query from ''", e); + return null; + } + } + + private List> executeQuery(Query query, Model model) { model.enterCriticalSection(Lock.READ); try{ - QueryExecution qexec= QueryExecutionFactory.create(query, model,initialBindings ); + QueryExecution qexec= QueryExecutionFactory.create(query, model ); ResultSet results = qexec.execSelect(); try{ - while (results.hasNext()) { - QuerySolution soln = results.nextSolution(); - rows.add( toRow( soln ) ); - } + return parseResults(results); }finally{ qexec.close(); } }finally{ model.leaveCriticalSection(); } - - return rows; } /** + * Converts a ResultSet into a List of Maps. + */ + private List> parseResults(ResultSet results) { + List> rows = new ArrayList>(); + while (results.hasNext()) { + QuerySolution soln = results.nextSolution(); + rows.add( toRow( soln ) ); + } + return rows; + } + + /** * Converts a row from a QuerySolution to a Map */ private Map toRow(QuerySolution soln) { @@ -186,8 +243,8 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ private String toCell(RDFNode rdfNode) { if( rdfNode == null){ return ""; - }else if( rdfNode.canAs( Literal.class )){ - return ((Literal)rdfNode.as(Literal.class)).getLexicalForm(); + }else if( rdfNode.isLiteral() ){ + return rdfNode.asLiteral().getLexicalForm(); }else if( rdfNode.isResource() ){ Resource resource = (Resource)rdfNode; if( ! resource.isAnon() ){ @@ -200,40 +257,17 @@ public class SparqlQueryDataGetter extends DataGetterBase implements DataGetter{ } } - - - private QuerySolutionMap createBindings(MapparameterMap) { - QuerySolutionMap initBindings = new QuerySolutionMap(); - - //could have bindings from HTTP parameters - for( String var : parameterMap.keySet() ) { - String[] values = parameterMap.get(var); - if( values != null && values.length == 1 ){ - //what do do when we don't want a Resource? - initBindings.add(var, ResourceFactory.createResource(values[0]) ); - }else if( values.length > 1){ - log.error("more than 1 http parameter for " + var); - } - } - return initBindings; - } - - private static final String queryPropertyURI = "<" + DisplayVocabulary.QUERY + ">"; - private static final String saveToVarPropertyURI= "<" + DisplayVocabulary.SAVE_TO_VAR+ ">"; - private static final String queryModelPropertyURI= "<" + DisplayVocabulary.QUERY_MODEL+ ">"; - - public static final String defaultVarNameForResults = "results"; - - /** - * Query to get the definition of the SparqlDataGetter for a given URI. - */ - private static final String dataGetterQuery = - "PREFIX display: <" + DisplayVocabulary.DISPLAY_NS +"> \n" + - "SELECT ?query ?saveToVar ?queryModel WHERE { \n" + - " ?dataGetterURI "+queryPropertyURI+" ?query . \n" + - " OPTIONAL{ ?dataGetterURI "+saveToVarPropertyURI+" ?saveToVar } \n " + - " OPTIONAL{ ?dataGetterURI "+queryModelPropertyURI+" ?queryModel } \n" + - "}"; - - + private Map assembleMap(List> results) { + Map rmap = new HashMap(); + + //put results in page data + rmap.put(this.saveToVar, results); + //also store the variable name within which results will be returned + rmap.put("variableName", this.saveToVar); + //This will be overridden at page level in display model if template specified there + rmap.put("bodyTemplate", defaultTemplate); + + return rmap; + } + } diff --git a/webapp/test/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetterTest.java b/webapp/test/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetterTest.java index 6dd1dc7e4..8e7faafb7 100644 --- a/webapp/test/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetterTest.java +++ b/webapp/test/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SparqlQueryDataGetterTest.java @@ -67,9 +67,9 @@ public class SparqlQueryDataGetterTest extends AbstractTestClass{ String bobURI = "http://example.com/p/bob"; dataModel.add(ResourceFactory.createResource(bobURI), RDF.type, ResourceFactory.createResource("http://xmlns.com/foaf/0.1/Person")); - Map params = Collections.emptyMap(); + Map params = Collections.emptyMap(); - Map mapOut = sdg.doQuery(params, dataModel); + Map mapOut = sdg.doQueryOnModel(sdg.queryText, dataModel); Assert.assertNotNull(mapOut); Assert.assertTrue("should contain key people" , mapOut.containsKey("people"));