litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Updates for adding a data getter that executes a solr query to retrieve individuals for a specific vclass and returns the results - the default template is a copy of the individual list controller template - some code refactoring also breaks out some of the individual controller methods into solr query utils for potential further reuse.

Browse source
This commit is contained in:
hjkhjk54 2012-10-16 20:40:02 +00:00
parent 8c12deacec
commit 52314c4640
6 changed files with 419 additions and 89 deletions
Download patch file Download diff file Expand all files Collapse all files

103
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java
View file

@ -28,6 +28,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Tem
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual;
/**
@ -143,22 +144,14 @@ public class IndividualListController extends FreemarkerHttpServlet {
}
}
//TODO: Remove and update reference within JsonServlet
public static String getAlphaParameter(VitroRequest request){
return request.getParameter("alpha");
return SolrQueryUtils.getAlphaParameter(request);
}
//TODO: Remove and update reference within JsonServlet
public static int getPageParameter(VitroRequest request) {
String pageStr = request.getParameter("page");
if( pageStr != null ){
try{
return Integer.parseInt(pageStr);
}catch(NumberFormatException nfe){
log.debug("could not parse page parameter");
return 1;
}
}else{
return 1;
}
return SolrQueryUtils.getPageParameter(request);
}
public static Map<String,Object> getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context)
@ -166,7 +159,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
Map<String,Object> rvMap = new HashMap<String,Object>();
try{
List<String> classUris = Collections.singletonList(vclassURI);
IndividualListQueryResults results = buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, context, indDao);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, context, indDao);
rvMap = getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha);
} catch (SolrServerException e) {
String msg = "An error occurred retrieving results for vclass query";
@ -182,7 +175,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
public static Map<String,Object> getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao, ServletContext context) {
Map<String,Object> rvMap = new HashMap<String,Object>();
try{
IndividualListQueryResults results = buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, context, indDao);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, context, indDao);
rvMap = getResultsForVClassQuery(results, page, pageSize, alpha);
} catch(Throwable th) {
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
@ -190,18 +183,12 @@ public class IndividualListController extends FreemarkerHttpServlet {
return rvMap;
}
private static IndividualListQueryResults buildAndExecuteVClassQuery(
List<String> vclassURIs, String alpha, int page, int pageSize,
ServletContext context, IndividualDao indDao)
throws SolrServerException {
SolrQuery query = getQuery(vclassURIs, alpha, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs);
}
return results;
}
//TODO: Get rid of this method and utilize SolrQueryUtils - currently appears to be referenced
//only within DataGetterUtils
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
return SolrQueryUtils.getIndividualCount(vclassUris, indDao, context);
}
private static Map<String,Object> getResultsForVClassQuery(IndividualListQueryResults results, int page, int pageSize, String alpha) {
Map<String,Object> rvMap = new HashMap<String,Object>();
@ -222,69 +209,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
return rvMap;
}
//Get count of individuals without actually getting the results
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
SolrQuery query = new SolrQuery(makeMultiClassQuery(vclassUris));
query.setRows(0);
try {
SolrServer solr = SolrSetup.getSolrServer(context);
QueryResponse response = null;
response = solr.query(query);
return response.getResults().getNumFound();
} catch(Exception ex) {
log.error("An error occured in retrieving individual count", ex);
}
return 0;
}
/**
* builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by
* alpha, and just the hits for the page for pageSize.
*/
private static SolrQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
String queryText = "";
try {
queryText = makeMultiClassQuery(vclassUris);
// Add alpha filter if applicable
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
}
SolrQuery query = new SolrQuery(queryText);
//page count starts at 1, row count starts at 0
int startRow = (page-1) * pageSize ;
query.setStart( startRow ).setRows( pageSize );
// Need a single-valued field for sorting
query.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
log.debug("Query is " + query.toString());
return query;
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return new SolrQuery();
}
}
private static String makeMultiClassQuery( List<String> vclassUris){
List<String> queryTypes = new ArrayList<String>();
try {
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
}
return StringUtils.join(queryTypes, " AND ");
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return "";
}
}
public static List<PageRecord> makePagesList( long size, int pageSize, int selectedPage ) {

5
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/DisplayVocabulary.java
View file

@ -138,6 +138,11 @@ public class DisplayVocabulary {
/* URI of property for Fixed HTML Generator */
public static final String FIXED_HTML_VALUE = DISPLAY_NS + "htmlValue";
/* URI of property for Solr Query Generator */
public static final String VCLASSID = DISPLAY_NS + "hasVClassId";
//public static final Individual EVENTS = m_model.createIndividual( NS + "Events", PAGE );
//public static final Individual EVENTS_MENU_ITEM = m_model.createIndividual( NS + "EventsMenuItem", NAVIGATION_ELEMENT );

220
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SolrIndividualsDataGetter.java Normal file
View file

@ -0,0 +1,220 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.utils.dataGetter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.QuerySolutionMap;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController.SearchException;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual;
public class SolrIndividualsDataGetter extends DataGetterBase implements DataGetter{
String dataGetterURI;
List<String> vclassUris = null;
String saveToVar;
VitroRequest vreq;
ServletContext context;
final static Log log = LogFactory.getLog(SolrIndividualsDataGetter.class);
//default template
private final static String defaultTemplate = "menupage--defaultSolrIndividuals.ftl";
/**
* Constructor with display model and data getter URI that will be called by reflection.
*/
public SolrIndividualsDataGetter(VitroRequest vreq, Model displayModel, String dataGetterURI){
this.configure(vreq, displayModel,dataGetterURI);
}
//For now, vClassURI should be passed in within model
//We are also including ability to pass as parameter
@Override
public Map<String, Object> getData(Map<String, Object> pageData) {
// Merge the pageData with the request parameters. PageData overrides
Map<String, String[]> merged = new HashMap<String, String[]>();
merged.putAll(vreq.getParameterMap());
for (String key: pageData.keySet()) {
merged.put(key, new String[] {String.valueOf(pageData.get(key))});
}
return doSolrQuery( merged);
}
/**
* Configure this instance based on the URI and display model.
*/
protected void configure(VitroRequest vreq, Model displayModel, String dataGetterURI) {
if( vreq == null )
throw new IllegalArgumentException("VitroRequest may not be null.");
if( displayModel == null )
throw new IllegalArgumentException("Display Model may not be null.");
if( dataGetterURI == null )
throw new IllegalArgumentException("PageUri may not be null.");
this.vreq = vreq;
this.context = vreq.getSession().getServletContext();
this.dataGetterURI = dataGetterURI;
this.vclassUris = new ArrayList<String>();
QuerySolutionMap initBindings = new QuerySolutionMap();
initBindings.add("dataGetterURI", ResourceFactory.createResource(this.dataGetterURI));
int count = 0;
Query dataGetterConfigurationQuery = QueryFactory.create(dataGetterQuery) ;
displayModel.enterCriticalSection(Lock.READ);
try{
QueryExecution qexec = QueryExecutionFactory.create(
dataGetterConfigurationQuery, displayModel, initBindings) ;
ResultSet res = qexec.execSelect();
try{
while( res.hasNext() ){
count++;
QuerySolution soln = res.next();
//saveToVar is OPTIONAL
Literal saveTo = soln.getLiteral("saveToVar");
if( saveTo != null && saveTo.isLiteral() ){
this.saveToVar = saveTo.asLiteral().getLexicalForm();
}else{
this.saveToVar = defaultVarNameForResults;
}
//vclass uri is OPTIONAL
//Right now, only anticipating one but will need to change this if this is in fact a list
String vclassUriStr = null;
Resource vclassUri = soln.getResource("vclassUri");
if( vclassUri != null && vclassUri.isResource() ){
vclassUriStr = vclassUri.getURI();
}
if(vclassUriStr != null) {
this.vclassUris.add(vclassUriStr);
}
}
}finally{ qexec.close(); }
}finally{ displayModel.leaveCriticalSection(); }
}
//Partially copied from IndividualListController
private Map<String, Object> doSolrQuery( Map<String, String[]> merged) {
if(vclassUris.size() == 0) {
if(merged.containsKey("vclassuri")) {
this.vclassUris = Arrays.asList(merged.get("vclassuri"));
} else {
log.error("No vclass uri found. Solr query will not work");
}
}
Map<String, Object> body = new HashMap<String, Object>();
//Right now, just getting the first vclass uri, but will have to review how to get intersections or multiple classes
if(vclassUris.size() > 0) {
String vClassURI = vclassUris.get(0);
//First, get the vclass object
VClass vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(vClassURI);
if (vclass == null) {
log.error("Couldn't retrieve vclass " + vClassURI);
}
String vclassUri = vclass.getURI();
body.put("vclassId", vclassUri);
vreq.setAttribute("displayType", vclassUri); // used by the template model object
//Am not sure why an attribute is used here instead of just sending this in as data to the template?
// Set title and subtitle.
VClassGroup classGroup = vclass.getGroup();
String title;
if (classGroup == null) {
title = vclass.getName();
} else {
title = classGroup.getPublicName();
body.put("subtitle", vclass.getName());
}
body.put("title", title);
populateSolrQueryResults(vclass, body);
body.put("bodyTemplate", this.defaultTemplate);
} else {
log.error("No VClass URIs found. No query will be executed");
}
return body;
}
private void populateSolrQueryResults(VClass vclass, Map<String, Object> body) {
try {
String alpha = SolrQueryUtils.getAlphaParameter(vreq);
int page = SolrQueryUtils.getPageParameter(vreq);
Map<String,Object> map = IndividualListController.getResultsForVClass(
vclass.getURI(),
page,
alpha,
vreq.getWebappDaoFactory().getIndividualDao(),
vreq.getSession().getServletContext());
body.putAll(map);
@SuppressWarnings("unchecked")
List<Individual> inds = (List<Individual>)map.get("entities");
List<ListedIndividual> indsTm = new ArrayList<ListedIndividual>();
if (inds != null) {
for ( Individual ind : inds ) {
indsTm.add(new ListedIndividual(ind,vreq));
}
}
body.put("individuals", indsTm);
body.put("rdfUrl", UrlBuilder.getUrl("/listrdf", "vclass", vclass.getURI()));
} catch (SearchException ex) {
log.error("Error retrieving results for display.", ex);
}
catch(Exception ex) {
log.error("Error occurred in retrieving results ", ex);
}
}
private static final String saveToVarPropertyURI= "<" + DisplayVocabulary.SAVE_TO_VAR+ ">";
private static final String vclassIdPropertyURI= "<" + DisplayVocabulary.VCLASSID+ ">";
public static final String defaultVarNameForResults = "results";
/**
* Query to get the definition of the Solr individuals data getter for a given URI.
*/
private static final String dataGetterQuery =
"PREFIX display: <" + DisplayVocabulary.DISPLAY_NS +"> \n" +
"SELECT ?vclassUri ?saveToVar WHERE { \n" +
" OPTIONAL{ ?dataGetterURI "+saveToVarPropertyURI+" ?saveToVar } \n " +
" OPTIONAL{ ?dataGetterURI "+vclassIdPropertyURI+" ?vclassUri } \n " +
"}";
}

117
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/solr/SolrQueryUtils.java
View file

@ -4,17 +4,37 @@ package edu.cornell.mannlib.vitro.webapp.utils.solr;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListQueryResults;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController.SearchException;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/**
* Some static method to help in constructing Solr queries and parsing the
* results.
*/
public class SolrQueryUtils {
private static final Log log = LogFactory.getLog(SolrQueryUtils.class.getName());
public enum Conjunction {
AND, OR;
@ -99,5 +119,102 @@ public class SolrQueryUtils {
private static String buildTerm(String fieldName, String word) {
return fieldName + ":\"" + word + "\"";
}
/**
* Methods that can be used in multiple places, such as IndividualListController and SolrIndividualsDataGetter
*/
public static String getAlphaParameter(VitroRequest request){
return request.getParameter("alpha");
}
public static int getPageParameter(VitroRequest request) {
String pageStr = request.getParameter("page");
if( pageStr != null ){
try{
return Integer.parseInt(pageStr);
}catch(NumberFormatException nfe){
log.debug("could not parse page parameter");
return 1;
}
}else{
return 1;
}
}
//Get count of individuals without actually getting the results
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
SolrQuery query = new SolrQuery(makeMultiClassQuery(vclassUris));
query.setRows(0);
try {
SolrServer solr = SolrSetup.getSolrServer(context);
QueryResponse response = null;
response = solr.query(query);
return response.getResults().getNumFound();
} catch(Exception ex) {
log.error("An error occured in retrieving individual count", ex);
}
return 0;
}
/**
* builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by
* alpha, and just the hits for the page for pageSize.
*/
public static SolrQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
String queryText = "";
try {
queryText = makeMultiClassQuery(vclassUris);
// Add alpha filter if applicable
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
}
SolrQuery query = new SolrQuery(queryText);
//page count starts at 1, row count starts at 0
int startRow = (page-1) * pageSize ;
query.setStart( startRow ).setRows( pageSize );
// Need a single-valued field for sorting
query.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
log.debug("Query is " + query.toString());
return query;
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return new SolrQuery();
}
}
public static String makeMultiClassQuery( List<String> vclassUris){
List<String> queryTypes = new ArrayList<String>();
try {
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
}
return StringUtils.join(queryTypes, " AND ");
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return "";
}
}
public static IndividualListQueryResults buildAndExecuteVClassQuery(
List<String> vclassURIs, String alpha, int page, int pageSize,
ServletContext context, IndividualDao indDao)
throws SolrServerException {
SolrQuery query = SolrQueryUtils.getQuery(vclassURIs, alpha, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs);
}
return results;
}
}