litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-2801 Fix SolrIndividualListController to handle a situation where the index contains a number of non-existent individuals

Browse source
This commit is contained in:
ryounes 2011-06-29 21:25:16 +00:00
parent f6a0fdebfe
commit b3abd584dc
3 changed files with 68 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java
View file

@ -184,14 +184,14 @@ public class SolrJsonServlet extends VitroHttpServlet {
VClass vclass=null; VClass vclass=null;
JSONObject rObj = new JSONObject(); JSONObject rObj = new JSONObject();
Map<String, Object> map = getSolrVclassIntersectionResults(vclassURIs, vreq, context); Map<String, Object> map = getSolrVClassIntersectionResults(vclassURIs, vreq, context);
//last parameter indicates single vclass instead of multiple vclasses //last parameter indicates single vclass instead of multiple vclasses
rObj = processVclassResults(map, vreq, context, false); rObj = processVClassResults(map, vreq, context, false);
return rObj; return rObj;
} }
//Accepts multiple vclasses and returns individuals which correspond to the intersection of those classes (i.e. have all those types) // Accepts multiple vclasses and returns individuals which correspond to the intersection of those classes (i.e. have all those types)
private void getSolrIndividualsByVClasses( HttpServletRequest req, HttpServletResponse resp ){ private void getSolrIndividualsByVClasses( HttpServletRequest req, HttpServletResponse resp ){
String errorMessage = null; String errorMessage = null;
JSONObject rObj = null; JSONObject rObj = null;
@ -199,8 +199,7 @@ public class SolrJsonServlet extends VitroHttpServlet {
VitroRequest vreq = new VitroRequest(req); VitroRequest vreq = new VitroRequest(req);
VClass vclass=null; VClass vclass=null;
//Could have multiple vclass Ids sent in // Could have multiple vclass ids sent in
List<String> vclassIds = new ArrayList<String>();
String[] vitroClassIdStr = vreq.getParameterValues("vclassId"); String[] vitroClassIdStr = vreq.getParameterValues("vclassId");
if ( vitroClassIdStr != null && vitroClassIdStr.length > 0){ if ( vitroClassIdStr != null && vitroClassIdStr.length > 0){
for(String vclassId: vitroClassIdStr) { for(String vclassId: vitroClassIdStr) {
@ -214,7 +213,7 @@ public class SolrJsonServlet extends VitroHttpServlet {
log.debug("parameter vclassId URI parameter expected "); log.debug("parameter vclassId URI parameter expected ");
throw new Exception("parameter vclassId URI parameter expected "); throw new Exception("parameter vclassId URI parameter expected ");
} }
vclassIds = Arrays.asList(vitroClassIdStr); List<String> vclassIds = Arrays.asList(vitroClassIdStr);
rObj = getSolrIndividualsByVClasses(vclassIds,req, getServletContext()); rObj = getSolrIndividualsByVClasses(vclassIds,req, getServletContext());
}catch(Exception ex){ }catch(Exception ex){
errorMessage = ex.toString(); errorMessage = ex.toString();
@ -245,19 +244,17 @@ public class SolrJsonServlet extends VitroHttpServlet {
public static JSONObject getSolrIndividualsByVClasses(List<String> vclassURIs, HttpServletRequest req, ServletContext context) throws Exception { public static JSONObject getSolrIndividualsByVClasses(List<String> vclassURIs, HttpServletRequest req, ServletContext context) throws Exception {
VitroRequest vreq = new VitroRequest(req); VitroRequest vreq = new VitroRequest(req);
Map<String, Object> map = getSolrVclassIntersectionResults(vclassURIs, vreq, context); Map<String, Object> map = getSolrVClassIntersectionResults(vclassURIs, vreq, context);
JSONObject rObj = processVclassResults(map, vreq, context, true); JSONObject rObj = processVClassResults(map, vreq, context, true);
return rObj; return rObj;
} }
//Including version for Solr query for Vclass Intersections //Including version for Solr query for Vclass Intersections
private static Map<String,Object> getSolrVclassIntersectionResults(List<String> vclassURIs, VitroRequest vreq, ServletContext context){ private static Map<String,Object> getSolrVClassIntersectionResults(List<String> vclassURIs, VitroRequest vreq, ServletContext context){
String alpha = SolrIndividualListController.getAlphaParameter(vreq); String alpha = SolrIndividualListController.getAlphaParameter(vreq);
int page = SolrIndividualListController.getPageParameter(vreq); int page = SolrIndividualListController.getPageParameter(vreq);
Map<String,Object> map = null; Map<String,Object> map = null;
try { try {
//TODO: This needs to be moved or copied to Solr Individual List Controller
//Note the method below does use Solr search
map = SolrIndividualListController.getResultsForVClassIntersections( map = SolrIndividualListController.getResultsForVClassIntersections(
vclassURIs, vclassURIs,
page, page,
@ -273,38 +270,11 @@ public class SolrJsonServlet extends VitroHttpServlet {
//Factoring out to allow for results to be processed from query for both lucene and solr //Factoring out to allow for results to be processed from query for both lucene and solr
//Map given to process method includes the actual individuals returned from the search //Map given to process method includes the actual individuals returned from the search
public static JSONObject processVclassResults(Map<String, Object> map, VitroRequest vreq, ServletContext context, boolean multipleVclasses) throws Exception{ public static JSONObject processVClassResults(Map<String, Object> map, VitroRequest vreq, ServletContext context, boolean multipleVclasses) throws Exception{
JSONObject rObj = DataGetterUtils.processVclassResultsJSON(map, vreq, multipleVclasses); JSONObject rObj = DataGetterUtils.processVclassResultsJSON(map, vreq, multipleVclasses);
return rObj; return rObj;
} }
private static String getVClassName(Individual ind, String moniker,
WebappDaoFactory fullWdf) {
/* so the moniker frequently has a vclass name in it. Try to return
* the vclass name that is the same as the moniker so that the templates
* can detect this. */
if( (moniker == null || moniker.isEmpty()) ){
if( ind.getVClass() != null && ind.getVClass().getName() != null )
return ind.getVClass().getName();
else
return "";
}
List<VClass> vcList = ind.getVClasses();
for( VClass vc : vcList){
if( vc != null && moniker.equals( vc.getName() ))
return moniker;
}
// if we get here, then we didn't find a moniker that matched a vclass,
// so just return any vclass.name
if( ind.getVClass() != null && ind.getVClass().getName() != null )
return ind.getVClass().getName();
else
return "";
}
static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){ static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){
List<Literal> values = wdf.getDataPropertyStatementDao() List<Literal> values = wdf.getDataPropertyStatementDao()
.getDataPropertyValuesForIndividualByProperty(ind, dp); .getDataPropertyValuesForIndividualByProperty(ind, dp);

100
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java
View file

@ -43,9 +43,9 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(SolrIndividualListController.class.getName()); private static final Log log = LogFactory.getLog(SolrIndividualListController.class.getName());
public static final int ENTITY_LIST_CONTROLLER_MAX_RESULTS = 30000; private static final int INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS = 30000;
public static final int INDIVIDUALS_PER_PAGE = 30; private static final int INDIVIDUALS_PER_PAGE = 30;
public static final int MAX_PAGES = 40; //must be even private static final int MAX_PAGES = 40; // must be even
private static final String TEMPLATE_DEFAULT = "individualList.ftl"; private static final String TEMPLATE_DEFAULT = "individualList.ftl";
@ -167,7 +167,7 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
throws IOException, ServletException{ throws IOException, ServletException{
Map<String,Object> rvMap = new HashMap<String,Object>(); Map<String,Object> rvMap = new HashMap<String,Object>();
try{ try{
SolrQuery query = getQuery(vclassURIs, alpha, page); SolrQuery query = getQuery(vclassURIs, alpha);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) if (individuals == null)
@ -185,7 +185,7 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
//make query for this rdf:type //make query for this rdf:type
List<String> classUris = new ArrayList<String>(); List<String> classUris = new ArrayList<String>();
classUris.add(vclassURI); classUris.add(vclassURI);
SolrQuery query = getQuery(classUris, alpha, page); SolrQuery query = getQuery(classUris, alpha);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) if (individuals == null)
@ -201,14 +201,13 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
Map<String,Object> rvMap = new HashMap<String,Object>(); Map<String,Object> rvMap = new HashMap<String,Object>();
try{ try{
// make query for multiple rdf types // make query for multiple rdf types
SolrQuery query = getQuery(vclassURIs, alpha, page); SolrQuery query = getQuery(vclassURIs, alpha);
//get results corresponding to this query
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) if (individuals == null)
log.debug("entities list is null for vclass " + vclassURIs.toString() ); log.debug("entities list is null for vclass " + vclassURIs.toString() );
} catch(Throwable th) { } catch(Throwable th) {
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th); log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
} }
return rvMap; return rvMap;
} }
@ -231,33 +230,42 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
} }
if ( response == null ) { if ( response == null ) {
throw new ServletException("Could not run search in IndividualListController"); throw new ServletException("Could not run search in SolrIndividualListController");
} }
SolrDocumentList docs = response.getResults(); SolrDocumentList docs = response.getResults();
if (docs == null) { if (docs == null) {
throw new ServletException("Could not run search in IndividualListController"); throw new ServletException("Could not run search in SorlIndividualListController");
} }
// get list of individuals for the search results // get list of individuals for the search results
long size = docs.getNumFound(); long hitCount = docs.getNumFound();
log.debug("Number of search results: " + size); log.debug("Number of search results: " + hitCount);
List<Individual> individuals = new ArrayList<Individual>(); List<Individual> individuals = new ArrayList<Individual>(INDIVIDUALS_PER_PAGE);
for (SolrDocument doc : docs) { int individualsAdded = 0;
String uri = doc.get(VitroSearchTermNames.URI).toString(); int index = (page-1) * INDIVIDUALS_PER_PAGE;
Individual individual = indDao.getIndividualByURI( uri ); while (individualsAdded < INDIVIDUALS_PER_PAGE && index < hitCount) {
if (individual != null) { SolrDocument doc = docs.get(index);
individuals.add(individual); if (doc != null) {
String uri = doc.get(VitroSearchTermNames.URI).toString();
Individual individual = indDao.getIndividualByURI( uri );
if (individual != null) {
individualsAdded++;
individuals.add(individual);
log.debug("Adding individual " + uri + " to individuals for display");
} else {
log.debug("No existing individual for search document with uri = " + uri);
}
} }
index++;
} }
rvMap.put("count", size); // Test index < hitCount ensures that there are still some docs left
if ( hitCount > INDIVIDUALS_PER_PAGE && index < hitCount ){
if( size > INDIVIDUALS_PER_PAGE ){
rvMap.put("showPages", Boolean.TRUE); rvMap.put("showPages", Boolean.TRUE);
List<PageRecord> pageRecords = makePagesList(size, INDIVIDUALS_PER_PAGE, page); List<PageRecord> pageRecords = makePagesList(hitCount, INDIVIDUALS_PER_PAGE, page);
rvMap.put("pages", pageRecords); rvMap.put("pages", pageRecords);
}else{ }else{
rvMap.put("showPages", Boolean.FALSE); rvMap.put("showPages", Boolean.FALSE);
@ -266,45 +274,47 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
rvMap.put("alpha",alpha); rvMap.put("alpha",alpha);
rvMap.put("totalCount", size); rvMap.put("totalCount", hitCount);
rvMap.put("entities",individuals); rvMap.put("entities",individuals);
return rvMap; return rvMap;
} }
private static SolrQuery getQuery(List<String> vclassUris, String alpha, int page){ private static SolrQuery getQuery(List<String> vclassUris, String alpha){
// Solr requires these values, but we don't want them to be the real values for this page
// of results, else the refinement links won't work correctly: each page of results needs to
// show refinement links generated for all results, not just for the results on the current page.
String queryText = ""; String queryText = "";
//Query text should be of form:
List<String> queryTypes = new ArrayList<String>(); List<String> queryTypes = new ArrayList<String>();
try{ try {
//query term for rdf:type - multiple types possible
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) { for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" "); queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
} }
if(queryTypes.size() > 1) { if (queryTypes.size() > 0) {
queryText = StringUtils.join(queryTypes, " AND "); queryText = StringUtils.join(queryTypes, " AND ");
} else {
if(queryTypes.size() > 0) {
queryText = queryTypes.get(0);
}
} }
// Add alpha filter if it is needed // Add alpha filter if applicable
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) { if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
} }
SolrQuery query = new SolrQuery(queryText);
log.debug("Query text is " + queryText); SolrQuery query = new SolrQuery(queryText);
int start = (page-1)*INDIVIDUALS_PER_PAGE; log.debug("Query text is " + queryText);
query.setStart(start)
.setRows(INDIVIDUALS_PER_PAGE) // Get all available results from index rather than just those for the current page.
.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc); // Otherwise, if there are a large number of non-existent individuals in the search
// index, the current page of results might not retrieve any existing individuals,
// and nothing gets returned.
query.setStart(0)
.setRows(INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS)
// Need a single-valued field for sorting
.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
return query; return query;
} catch (Exception ex){ } catch (Exception ex){
log.error(ex,ex); log.error(ex,ex);
return new SolrQuery(); return new SolrQuery();

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java
View file

@ -91,7 +91,7 @@ public class SolrAutocompleteController extends VitroAjaxController {
} }
long hitCount = docs.getNumFound(); long hitCount = docs.getNumFound();
log.debug("Number of hits = " + hitCount); log.debug("Total number of hits = " + hitCount);
if ( hitCount < 1 ) { if ( hitCount < 1 ) {
doNoSearchResults(response); doNoSearchResults(response);
return; return;