From 1176568eb5f1681b5f476238f57c0b7b2e7ef7a2 Mon Sep 17 00:00:00 2001 From: ryounes Date: Mon, 23 May 2011 15:30:38 +0000 Subject: [PATCH] NIHVIVO-2437 Generate /individuallist page results in SolrIndividualListController --- .../SolrIndividualListController.java | 311 ++++++++---------- .../SolrAutocompleteController.java | 1 + .../controller/SolrPagedSearchController.java | 6 +- 3 files changed, 148 insertions(+), 170 deletions(-) diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java index eec707bec..0dab98de1 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/SolrIndividualListController.java @@ -17,16 +17,12 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServer; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.VClass; @@ -36,8 +32,8 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Exc import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; -import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc; -import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexFactory; +import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc.VitroLuceneTermNames; +import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individual.ListedIndividualTemplateModel; import freemarker.ext.beans.BeansWrapper; import freemarker.template.TemplateModel; @@ -175,163 +171,148 @@ public class SolrIndividualListController extends FreemarkerHttpServlet { * This method is now called in a couple of places. It should be refactored * into a DAO or similar object. */ - public static Map getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context) - throws CorruptIndexException, IOException, ServletException{ - Map rvMap = new HashMap(); - - //make lucene query for this rdf:type - Query query = getQuery(vclassURI, alpha); - - //execute lucene query for individuals of the specified type - IndexSearcher index = LuceneIndexFactory.getIndexSearcher(context); - TopDocs docs = null; - try{ - docs = index.search(query, null, - ENTITY_LIST_CONTROLLER_MAX_RESULTS, - new Sort(Entity2LuceneDoc.term.NAME_LOWERCASE)); - }catch(Throwable th){ - log.error("Could not run search. " + th.getMessage()); - docs = null; - } - - if( docs == null ) - throw new ServletException("Could not run search in IndividualListController"); - - //get list of individuals for the search results - int size = docs.totalHits; - log.debug("Number of search results: " + size); - - // don't get all the results, only get results for the requestedSize - List individuals = new ArrayList(INDIVIDUALS_PER_PAGE); - int individualsAdded = 0; - int ii = (page-1)*INDIVIDUALS_PER_PAGE; - while( individualsAdded < INDIVIDUALS_PER_PAGE && ii < size ){ - ScoreDoc hit = docs.scoreDocs[ii]; - if (hit != null) { - Document doc = index.doc(hit.doc); - if (doc != null) { - String uri = doc.getField(Entity2LuceneDoc.term.URI).stringValue(); - Individual ind = indDao.getIndividualByURI( uri ); - if( ind != null ){ - individuals.add( ind ); - individualsAdded++; - } - } else { - log.warn("no document found for lucene doc id " + hit.doc); - } - } else { - log.debug("hit was null"); - } - ii++; - } - - rvMap.put("count", size); - - if( size > INDIVIDUALS_PER_PAGE ){ - rvMap.put("showPages", Boolean.TRUE); - List pageRecords = makePagesList(size, INDIVIDUALS_PER_PAGE, page); - rvMap.put("pages", pageRecords); - }else{ - rvMap.put("showPages", Boolean.FALSE); - rvMap.put("pages", Collections.emptyList()); - } - - rvMap.put("alpha",alpha); - - rvMap.put("totalCount", size); - rvMap.put("entities",individuals); - if (individuals == null) - log.debug("entities list is null for vclass " + vclassURI ); - - return rvMap; - } - - private static BooleanQuery getQuery(String vclassUri, String alpha){ - BooleanQuery query = new BooleanQuery(); - try{ - //query term for rdf:type - query.add( - new TermQuery( new Term(Entity2LuceneDoc.term.RDFTYPE, vclassUri)), - BooleanClause.Occur.MUST ); - - //Add alpha filter if it is needed - Query alphaQuery = null; - if( alpha != null && !"".equals(alpha) && alpha.length() == 1){ - alphaQuery = - new PrefixQuery(new Term(Entity2LuceneDoc.term.NAME_LOWERCASE, alpha.toLowerCase())); - query.add(alphaQuery,BooleanClause.Occur.MUST); - } - - log.debug("Query: " + query); - return query; - } catch (Exception ex){ - log.error(ex,ex); - return new BooleanQuery(); - } - } + public static Map getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context) + throws CorruptIndexException, IOException, ServletException { + Map rvMap = new HashMap(); - public static List makePagesList( int count, int pageSize, int selectedPage){ - - List records = new ArrayList( MAX_PAGES + 1 ); - int requiredPages = count/pageSize ; - int remainder = count % pageSize ; - if( remainder > 0 ) - requiredPages++; - - if( selectedPage < MAX_PAGES && requiredPages > MAX_PAGES ){ - //the selected pages is within the first maxPages, just show the normal pages up to maxPages. - for(int page = 1; page < requiredPages && page <= MAX_PAGES ; page++ ){ - records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); + // Make solr query for this rdf:type + SolrQuery query = getQuery(vclassURI, alpha, page); + SolrServer solr = SolrSetup.getSolrServer(context); + QueryResponse response = null; + + // Execute lucene query for individuals of the specified type + try { + response = solr.query(query); + } catch (Throwable t) { + log.error(t, t); + } + + if ( response == null ) { + throw new ServletException("Could not run search in IndividualListController"); + } + + SolrDocumentList docs = response.getResults(); + + if (docs == null) { + throw new ServletException("Could not run search in IndividualListController"); + } + + // get list of individuals for the search results + long size = docs.getNumFound(); + log.debug("Number of search results: " + size); + + List individuals = new ArrayList((int)size); + for (SolrDocument doc : docs) { + String uri = doc.get(VitroLuceneTermNames.URI).toString(); + Individual individual = indDao.getIndividualByURI( uri ); + if (individual != null) { + individuals.add(individual); } - records.add( new PageRecord( "page="+ (MAX_PAGES+1), Integer.toString(MAX_PAGES+1), "more...", false)); - }else if( requiredPages > MAX_PAGES && selectedPage+1 > MAX_PAGES && selectedPage < requiredPages - MAX_PAGES){ - //the selected pages is in the middle of the list of page - int startPage = selectedPage - MAX_PAGES / 2; - int endPage = selectedPage + MAX_PAGES / 2; - for(int page = startPage; page <= endPage ; page++ ){ - records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); - } - records.add( new PageRecord( "page="+ endPage+1, Integer.toString(endPage+1), "more...", false)); - }else if ( requiredPages > MAX_PAGES && selectedPage > requiredPages - MAX_PAGES ){ - //the selected page is in the end of the list - int startPage = requiredPages - MAX_PAGES; - double max = Math.ceil(count/pageSize); - for(int page = startPage; page <= max; page++ ){ - records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); - } - }else{ - //there are fewer than maxPages pages. - for(int i = 1; i <= requiredPages; i++ ){ - records.add( new PageRecord( "page=" + i, Integer.toString(i), Integer.toString(i), selectedPage == i ) ); - } - } - return records; - } + } + + rvMap.put("count", size); + + if( size > INDIVIDUALS_PER_PAGE ){ + rvMap.put("showPages", Boolean.TRUE); + List pageRecords = makePagesList(size, INDIVIDUALS_PER_PAGE, page); + rvMap.put("pages", pageRecords); + }else{ + rvMap.put("showPages", Boolean.FALSE); + rvMap.put("pages", Collections.emptyList()); + } + + rvMap.put("alpha",alpha); + + rvMap.put("totalCount", size); + rvMap.put("entities",individuals); + + if (individuals.isEmpty()) + log.debug("entities list is empty for vclass " + vclassURI ); + + return rvMap; + } - public static class PageRecord { - public PageRecord(String param, String index, String text, boolean selected) { - this.param = param; - this.index = index; - this.text = text; - this.selected = selected; - } - public String param; - public String index; - public String text; - public boolean selected=false; + private static SolrQuery getQuery(String vclassUri, String alpha, int page){ + + String queryStr = VitroLuceneTermNames.RDFTYPE + ":\"" + vclassUri + "\""; + + // Add alpha filter if it is needed + if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) { + queryStr += VitroLuceneTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; + } + + SolrQuery query = new SolrQuery(queryStr); + + int start = (page-1)*INDIVIDUALS_PER_PAGE; + query.setStart(start) + .setRows(INDIVIDUALS_PER_PAGE); + + log.debug("Query: " + query); + return query; + } + + public static List makePagesList( long size, int pageSize, int selectedPage ) { + + List records = new ArrayList( MAX_PAGES + 1 ); + int requiredPages = (int) (size/pageSize) ; + int remainder = (int) (size % pageSize) ; + if( remainder > 0 ) + requiredPages++; - public String getParam() { - return param; - } - public String getIndex() { - return index; - } - public String getText() { - return text; - } - public boolean getSelected(){ - return selected; - } - } + if( selectedPage < MAX_PAGES && requiredPages > MAX_PAGES ){ + //the selected pages is within the first maxPages, just show the normal pages up to maxPages. + for(int page = 1; page < requiredPages && page <= MAX_PAGES ; page++ ){ + records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); + } + records.add( new PageRecord( "page="+ (MAX_PAGES+1), Integer.toString(MAX_PAGES+1), "more...", false)); + }else if( requiredPages > MAX_PAGES && selectedPage+1 > MAX_PAGES && selectedPage < requiredPages - MAX_PAGES){ + //the selected pages is in the middle of the list of page + int startPage = selectedPage - MAX_PAGES / 2; + int endPage = selectedPage + MAX_PAGES / 2; + for(int page = startPage; page <= endPage ; page++ ){ + records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); + } + records.add( new PageRecord( "page="+ endPage+1, Integer.toString(endPage+1), "more...", false)); + }else if ( requiredPages > MAX_PAGES && selectedPage > requiredPages - MAX_PAGES ){ + //the selected page is in the end of the list + int startPage = requiredPages - MAX_PAGES; + double max = Math.ceil(size/pageSize); + for(int page = startPage; page <= max; page++ ){ + records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) ); + } + }else{ + //there are fewer than maxPages pages. + for(int i = 1; i <= requiredPages; i++ ){ + records.add( new PageRecord( "page=" + i, Integer.toString(i), Integer.toString(i), selectedPage == i ) ); + } + } + return records; + } + + public static class PageRecord { + public PageRecord(String param, String index, String text, boolean selected) { + this.param = param; + this.index = index; + this.text = text; + this.selected = selected; + } + public String param; + public String index; + public String text; + public boolean selected=false; + + public String getParam() { + return param; + } + public String getIndex() { + return index; + } + public String getText() { + return text; + } + public boolean getSelected(){ + return selected; + } + } } diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java index 95c7fbf9b..663467256 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java @@ -187,6 +187,7 @@ public class SolrAutocompleteController extends VitroAjaxController { // RY 5/18/2011 For now, just doing untokenized query, due to the interactions of wildcard // query and stemming described below. Need to find a way to do this in Solr. // Should take the same approach if we can figure out how to do a disjunction. + // Probably just add an explicit "OR" between the terms. // String stemParam = (String) request.getParameter("stem"); // boolean stem = "true".equals(stemParam); diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrPagedSearchController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrPagedSearchController.java index f546cc796..4b7e4e44c 100644 --- a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrPagedSearchController.java +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrPagedSearchController.java @@ -174,11 +174,7 @@ public class SolrPagedSearchController extends FreemarkerHttpServlet { log.debug("Query text is \""+ qtxt + "\""); - SolrQuery query = getQuery(qtxt, maxHitCount, vreq); - - // ** For xml requested, add version=2.2 for xml version - // is that enough, or do we also have to add wt param? - + SolrQuery query = getQuery(qtxt, maxHitCount, vreq); SolrServer solr = SolrSetup.getSolrServer(getServletContext()); QueryResponse response = null;