litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

NIHVIVO-2801 Fix SolrIndividualListController to handle a situation where the index contains a number of non-existent individuals

Browse source
This commit is contained in:
ryounes 2011-06-29 21:25:16 +00:00
parent f6a0fdebfe
commit b3abd584dc
3 changed files with 68 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java
View file

@ -184,14 +184,14 @@ public class SolrJsonServlet extends VitroHttpServlet {
VClass vclass=null;
JSONObject rObj = new JSONObject();
Map<String, Object> map = getSolrVclassIntersectionResults(vclassURIs, vreq, context);
Map<String, Object> map = getSolrVClassIntersectionResults(vclassURIs, vreq, context);
//last parameter indicates single vclass instead of multiple vclasses
rObj = processVclassResults(map, vreq, context, false);
rObj = processVClassResults(map, vreq, context, false);
return rObj;
}
//Accepts multiple vclasses and returns individuals which correspond to the intersection of those classes (i.e. have all those types)
// Accepts multiple vclasses and returns individuals which correspond to the intersection of those classes (i.e. have all those types)
private void getSolrIndividualsByVClasses( HttpServletRequest req, HttpServletResponse resp ){
String errorMessage = null;
JSONObject rObj = null;
@ -199,8 +199,7 @@ public class SolrJsonServlet extends VitroHttpServlet {
VitroRequest vreq = new VitroRequest(req);
VClass vclass=null;
//Could have multiple vclass Ids sent in
List<String> vclassIds = new ArrayList<String>();
// Could have multiple vclass ids sent in
String[] vitroClassIdStr = vreq.getParameterValues("vclassId");
if ( vitroClassIdStr != null && vitroClassIdStr.length > 0){
for(String vclassId: vitroClassIdStr) {
@ -214,7 +213,7 @@ public class SolrJsonServlet extends VitroHttpServlet {
log.debug("parameter vclassId URI parameter expected ");
throw new Exception("parameter vclassId URI parameter expected ");
}
vclassIds = Arrays.asList(vitroClassIdStr);
List<String> vclassIds = Arrays.asList(vitroClassIdStr);
rObj = getSolrIndividualsByVClasses(vclassIds,req, getServletContext());
}catch(Exception ex){
errorMessage = ex.toString();
@ -245,19 +244,17 @@ public class SolrJsonServlet extends VitroHttpServlet {
public static JSONObject getSolrIndividualsByVClasses(List<String> vclassURIs, HttpServletRequest req, ServletContext context) throws Exception {
VitroRequest vreq = new VitroRequest(req);
Map<String, Object> map = getSolrVclassIntersectionResults(vclassURIs, vreq, context);
JSONObject rObj = processVclassResults(map, vreq, context, true);
Map<String, Object> map = getSolrVClassIntersectionResults(vclassURIs, vreq, context);
JSONObject rObj = processVClassResults(map, vreq, context, true);
return rObj;
}
//Including version for Solr query for Vclass Intersections
private static Map<String,Object> getSolrVclassIntersectionResults(List<String> vclassURIs, VitroRequest vreq, ServletContext context){
private static Map<String,Object> getSolrVClassIntersectionResults(List<String> vclassURIs, VitroRequest vreq, ServletContext context){
String alpha = SolrIndividualListController.getAlphaParameter(vreq);
int page = SolrIndividualListController.getPageParameter(vreq);
Map<String,Object> map = null;
try {
//TODO: This needs to be moved or copied to Solr Individual List Controller
//Note the method below does use Solr search
map = SolrIndividualListController.getResultsForVClassIntersections(
vclassURIs,
page,
@ -273,37 +270,10 @@ public class SolrJsonServlet extends VitroHttpServlet {
//Factoring out to allow for results to be processed from query for both lucene and solr
//Map given to process method includes the actual individuals returned from the search
public static JSONObject processVclassResults(Map<String, Object> map, VitroRequest vreq, ServletContext context, boolean multipleVclasses) throws Exception{
public static JSONObject processVClassResults(Map<String, Object> map, VitroRequest vreq, ServletContext context, boolean multipleVclasses) throws Exception{
JSONObject rObj = DataGetterUtils.processVclassResultsJSON(map, vreq, multipleVclasses);
return rObj;
}
private static String getVClassName(Individual ind, String moniker,
WebappDaoFactory fullWdf) {
/* so the moniker frequently has a vclass name in it. Try to return
* the vclass name that is the same as the moniker so that the templates
* can detect this. */
if( (moniker == null || moniker.isEmpty()) ){
if( ind.getVClass() != null && ind.getVClass().getName() != null )
return ind.getVClass().getName();
else
return "";
}
List<VClass> vcList = ind.getVClasses();
for( VClass vc : vcList){
if( vc != null && moniker.equals( vc.getName() ))
return moniker;
}
// if we get here, then we didn't find a moniker that matched a vclass,
// so just return any vclass.name
if( ind.getVClass() != null && ind.getVClass().getName() != null )
return ind.getVClass().getName();
else
return "";
}
static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){
List<Literal> values = wdf.getDataPropertyStatementDao()

106
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java
View file

@ -43,9 +43,9 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(SolrIndividualListController.class.getName());
public static final int ENTITY_LIST_CONTROLLER_MAX_RESULTS = 30000;
public static final int INDIVIDUALS_PER_PAGE = 30;
public static final int MAX_PAGES = 40; //must be even
private static final int INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS = 30000;
private static final int INDIVIDUALS_PER_PAGE = 30;
private static final int MAX_PAGES = 40; // must be even
private static final String TEMPLATE_DEFAULT = "individualList.ftl";
@ -167,7 +167,7 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
throws IOException, ServletException{
Map<String,Object> rvMap = new HashMap<String,Object>();
try{
SolrQuery query = getQuery(vclassURIs, alpha, page);
SolrQuery query = getQuery(vclassURIs, alpha);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null)
@ -185,7 +185,7 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
//make query for this rdf:type
List<String> classUris = new ArrayList<String>();
classUris.add(vclassURI);
SolrQuery query = getQuery(classUris, alpha, page);
SolrQuery query = getQuery(classUris, alpha);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null)
@ -201,14 +201,13 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
Map<String,Object> rvMap = new HashMap<String,Object>();
try{
// make query for multiple rdf types
SolrQuery query = getQuery(vclassURIs, alpha, page);
//get results corresponding to this query
SolrQuery query = getQuery(vclassURIs, alpha);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null)
log.debug("entities list is null for vclass " + vclassURIs.toString() );
} catch(Throwable th) {
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
}
return rvMap;
}
@ -231,33 +230,42 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
}
if ( response == null ) {
throw new ServletException("Could not run search in IndividualListController");
throw new ServletException("Could not run search in SolrIndividualListController");
}
SolrDocumentList docs = response.getResults();
if (docs == null) {
throw new ServletException("Could not run search in IndividualListController");
throw new ServletException("Could not run search in SorlIndividualListController");
}
// get list of individuals for the search results
long size = docs.getNumFound();
log.debug("Number of search results: " + size);
long hitCount = docs.getNumFound();
log.debug("Number of search results: " + hitCount);
List<Individual> individuals = new ArrayList<Individual>();
for (SolrDocument doc : docs) {
String uri = doc.get(VitroSearchTermNames.URI).toString();
Individual individual = indDao.getIndividualByURI( uri );
if (individual != null) {
individuals.add(individual);
}
List<Individual> individuals = new ArrayList<Individual>(INDIVIDUALS_PER_PAGE);
int individualsAdded = 0;
int index = (page-1) * INDIVIDUALS_PER_PAGE;
while (individualsAdded < INDIVIDUALS_PER_PAGE && index < hitCount) {
SolrDocument doc = docs.get(index);
if (doc != null) {
String uri = doc.get(VitroSearchTermNames.URI).toString();
Individual individual = indDao.getIndividualByURI( uri );
if (individual != null) {
individualsAdded++;
individuals.add(individual);
log.debug("Adding individual " + uri + " to individuals for display");
} else {
log.debug("No existing individual for search document with uri = " + uri);
}
}
index++;
}
rvMap.put("count", size);
if( size > INDIVIDUALS_PER_PAGE ){
// Test index < hitCount ensures that there are still some docs left
if ( hitCount > INDIVIDUALS_PER_PAGE && index < hitCount ){
rvMap.put("showPages", Boolean.TRUE);
List<PageRecord> pageRecords = makePagesList(size, INDIVIDUALS_PER_PAGE, page);
List<PageRecord> pageRecords = makePagesList(hitCount, INDIVIDUALS_PER_PAGE, page);
rvMap.put("pages", pageRecords);
}else{
rvMap.put("showPages", Boolean.FALSE);
@ -266,45 +274,47 @@ public class SolrIndividualListController extends FreemarkerHttpServlet {
rvMap.put("alpha",alpha);
rvMap.put("totalCount", size);
rvMap.put("totalCount", hitCount);
rvMap.put("entities",individuals);
return rvMap;
}
private static SolrQuery getQuery(List<String> vclassUris, String alpha, int page){
// Solr requires these values, but we don't want them to be the real values for this page
// of results, else the refinement links won't work correctly: each page of results needs to
// show refinement links generated for all results, not just for the results on the current page.
private static SolrQuery getQuery(List<String> vclassUris, String alpha){
String queryText = "";
//Query text should be of form:
List<String> queryTypes = new ArrayList<String>();
try{
//query term for rdf:type - multiple types possible
try {
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
}
if(queryTypes.size() > 1) {
if (queryTypes.size() > 0) {
queryText = StringUtils.join(queryTypes, " AND ");
} else {
if(queryTypes.size() > 0) {
queryText = queryTypes.get(0);
}
}
}
// Add alpha filter if it is needed
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
}
SolrQuery query = new SolrQuery(queryText);
log.debug("Query text is " + queryText);
int start = (page-1)*INDIVIDUALS_PER_PAGE;
query.setStart(start)
.setRows(INDIVIDUALS_PER_PAGE)
.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
// Add alpha filter if applicable
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
}
SolrQuery query = new SolrQuery(queryText);
log.debug("Query text is " + queryText);
// Get all available results from index rather than just those for the current page.
// Otherwise, if there are a large number of non-existent individuals in the search
// index, the current page of results might not retrieve any existing individuals,
// and nothing gets returned.
query.setStart(0)
.setRows(INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS)
// Need a single-valued field for sorting
.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
return query;
} catch (Exception ex){
log.error(ex,ex);
return new SolrQuery();

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java
View file

@ -91,7 +91,7 @@ public class SolrAutocompleteController extends VitroAjaxController {
}
long hitCount = docs.getNumFound();
log.debug("Number of hits = " + hitCount);
log.debug("Total number of hits = " + hitCount);
if ( hitCount < 1 ) {
doNoSearchResults(response);
return;