merge 5911 from nihvivo-rel-1.1-maint to trunk

This commit is contained in:
bdc34 2010-10-06 20:02:20 +00:00
parent 4549e8e610
commit f0d55d6875
10 changed files with 683 additions and 232 deletions

View file

@ -3,7 +3,11 @@
package edu.cornell.mannlib.vitro.webapp.controller;
import java.io.IOException;
import java.util.Collection;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
@ -12,9 +16,28 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RangeQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.joda.time.DateTime;
import edu.cornell.mannlib.vitro.webapp.auth.policy.JenaNetidPolicy.ContextSetup;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.Tab;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.TabDao;
import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexFactory;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexer;
import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils;
import edu.cornell.mannlib.vitro.webapp.web.TabWebUtil;
/**
@ -28,6 +51,11 @@ public class TabEntitiesController extends VitroHttpServlet {
private static final Log log = LogFactory.getLog(TabEntitiesController.class.getName());
public static int TAB_DEPTH_CUTOFF = 3;
public static int MAX_PAGES = 40; //must be even
public static int DEFAULT_NUMBER_INDIVIDUALS_ON_TAB = 8;
private static int MAX_RESULTS=40000;
private static int NON_PAGED_LIMIT=1000;
private static Random random = new Random();
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
@ -50,29 +78,21 @@ public class TabEntitiesController extends VitroHttpServlet {
which we are doing entities.
leadingTab = 1, child of leadingTab = 2, etc.
"alpha" if set to an uppercase letter entities will be filtered
"alpha" if set to a letter entities will be filtered
to have only that initial.
bdc34 2006-01-12 created
*
bdc34 2010-09-17 modified to use lucene for some tasks.
*/
public void doGet( HttpServletRequest req, HttpServletResponse response )
throws IOException, ServletException {
//this will setup the portal
super.doGet(req,response);
try{
VitroRequest request = new VitroRequest(req);
TabDao tabDao = request.getWebappDaoFactory().getTabDao();
String obj = null;
try {
obj = request.getParameter("tabDepth");
if( obj == null ){
String e="TabEntitesController expects that request parameter 'tabDepth' be set"
+", use 1 as the leading tab's depth.";
throw new ServletException(e);
}
int depth = Integer.parseInt((String)obj);
int depth = getTabDepth(request);
if( depth >= TAB_DEPTH_CUTOFF){
String tabId = request.getParameter("tabId");
log.debug("\ttab "+tabId+" is at, "+ depth+" below "+ TAB_DEPTH_CUTOFF);
@ -88,86 +108,259 @@ public void doGet( HttpServletRequest req, HttpServletResponse response )
Tab tab = TabWebUtil.findStashedTab(tabId,request);
if( tab == null ){
String e="TabEntitiesController expects that tab"+tabId+" will be in the request attribute. "
+"It should have been placed there by a call to TabWebUtil.stashTabsInRequest in "
+"tabPrimary.jsp";
+"It should have been placed there by a call to TabWebUtil.stashTabsInRequest in tabPrimary.jsp";
throw new ServletException(e);
}
req.setAttribute("tabId", tab.getTabId());
request.setAttribute("controllerParam","primary=" + tab.getTabId());
String alpha = request.getParameter("alpha");
boolean doAlphaFilter = false;
if(( alpha != null && alpha.length() == 1) || tab.getGalleryRows()>1)
/* bjl23 20061006:
* The tab.getGalleryRows()>1 is a hack to use this field as
* a switch to turn on alpha filter display in
* non-gallery tabs. We need to add a db field for this. */
if(( alpha != null && alpha.length() == 1) ){
doAlphaFilter = true;
request.setAttribute("alpha", alpha.toUpperCase());
}
//now we have the parameteres from the request,
//branch to the different types of ways to handle things
if(depth == 1 && tab.isGallery() && !doAlphaFilter){
doGallery(tab, request,response);
}else if( tab.isGallery() || doAlphaFilter ){
doAlphaFiltered(alpha,tab,request,response);
}else if( tab.isManualLinked() ){
doManual(tab, request, response);
}else if( tab.isAutoLinked() ){
doAutoLinked( tab, request, response);
}else if( tab.isMixedLinked() ){
doAutoLinked( tab, request, response);
boolean doPagedFilter = request.getParameter("page") != null;
boolean showPaged = true;
if( tab.getGalleryRows() != 1 ){
/* bjl23 20061006: The tab.getGalleryRows()>1 is a hack to use this field as a switch to turn on
* alpha filter display in non-gallery tabs. We need to add a db field for this. */
request.setAttribute("showAlpha","1");
showPaged = true;
}
List<String> manuallyLinkedUris = Collections.emptyList();
List<String> autoLinkedUris = Collections.emptyList();
if( tab.isAutoLinked() || tab.isMixedLinked())
autoLinkedUris = request.getWebappDaoFactory().getTabDao().getTabAutoLinkedVClassURIs(tab.getTabId());
if( tab.isManualLinked() || tab.isMixedLinked())
manuallyLinkedUris = request.getWebappDaoFactory().getTabDao().getTabManuallyLinkedEntityURIs(tab.getTabId());
//If a tab is not linked to any types or individuals, don't show alpha or page filter links
boolean tabNotLinked = tabNotLinked( tab, autoLinkedUris, manuallyLinkedUris);
if( tabNotLinked ){
request.setAttribute("showAlpha", "0");
showPaged = false;
}
int page = getPage(request);
int entsPerTab = getSizeForNonGalleryTab(tab, showPaged);
int size = 0;
if( ! tabNotLinked ){
try{
//a lot of the work happens in this next line
size = addLinkedIndividualsForTab (tab,request, autoLinkedUris, manuallyLinkedUris,
alpha, page, entsPerTab, depth, doAlphaFilter, doPagedFilter);
}catch(Exception e){
log.warn(e,e);
}
}
//only show page list when the tab is depth 1.
if( showPaged && depth == 1 && size > 0 && size > entsPerTab ){
request.setAttribute("showPages",Boolean.TRUE);
//make a data structure to hold information about paged tabs
request.setAttribute("pages", makePagesList(size, entsPerTab, page ));
}else{
//what to do here when the entity link mod is unknown?
log.debug("TabEntitiesController: doing none for tabtypeid: "+ tab.getTabtypeId() +" and link mode: " + tab.getEntityLinkMethod());
request.setAttribute("showPages",Boolean.FALSE);
}
if( tab.isAutoLinked() || tab.isGallery() ){
if( doAlphaFilter ){
doAlphaFiltered(alpha, tab, request, response, tabDao, size);
}else{
doAutoLinked( tab, request, response, tabDao, size);
}
}else if( tab.isMixedLinked() || tab.isManualLinked() ){
doAutoLinked(tab,request,response,tabDao,size);
}else{
log.debug("TabEntitiesController: doing none for tabtypeid: "
+ tab.getTabtypeId() +" and link mode: " + tab.getEntityLinkMethod());
}
} catch (Throwable e) {
request.setAttribute("javax.servlet.jsp.jspException",e);
RequestDispatcher rd = request.getRequestDispatcher("/error.jsp");
rd.include(request, response);
req.setAttribute("javax.servlet.jsp.jspException",e);
RequestDispatcher rd = req.getRequestDispatcher("/error.jsp");
rd.include(req, response);
}
}
/**
* This build a lucene query for the tab, runs the query, gets individuals for that query,
* and adds the individuals to the tab.
*
* @param tab - this parameter is mutated by this method
* @param request
* @param autoLinkedUris
* @param manuallyLinkedUris
* @param alpha - letter for alpha filtering
* @param page - page for page filtering
* @param entsPerTab
* @param depth
* @param doAlphaFilter
* @param doPagedFilter
* @return total number of ents associated with tab regardless of filtering
*/
private int addLinkedIndividualsForTab(Tab tab, VitroRequest request,
List<String> autoLinkedUris, List<String> manuallyLinkedUris,
String alpha, int page, int entsPerTab, int depth,
boolean doAlphaFilter , boolean doPagedFilter) {
//try to get the URIs of the required individuals from the lucene index
IndexSearcher index = LuceneIndexFactory.getIndexSearcher(getServletContext());
boolean isSinglePortal = request.getWebappDaoFactory().getPortalDao().isSinglePortal();
BooleanQuery query = null;
if( tab.isAutoLinked() ){
query = getQuery(tab, autoLinkedUris, null, alpha, isSinglePortal);
}else if (tab.isManualLinked() ){
query = getQuery(tab, null, manuallyLinkedUris, alpha, isSinglePortal);
}else if ( tab.isMixedLinked() ){
query = getQuery(tab, autoLinkedUris, manuallyLinkedUris, alpha, isSinglePortal);
}else{
log.error("Tab " + tab.getTabId() + " is neither manually, auto nor mixed. ");
}
if( tab.getDayLimit() != 0 ){
query = addDayLimit( query, tab );
}
boolean onlyWithThumbImg = false;
if( depth > 1 && tab.getImageWidth() > 0 ){
onlyWithThumbImg = true;
}
IndividualDao indDao = request.getWebappDaoFactory().getIndividualDao();
int size = 0;
try {
String sortField = tab.getEntitySortField();
Sort sort = null;
if( sortField != null && !doAlphaFilter && !doPagedFilter ){
if( sortField.equalsIgnoreCase("timekey") || tab.getDayLimit() > 0){
sort = new Sort(Entity2LuceneDoc.term.TIMEKEY);
}else if( sortField.equalsIgnoreCase("sunrise") || tab.getDayLimit() < 0 ){
sort = new Sort(Entity2LuceneDoc.term.SUNRISE, true);
}else if( sortField.equalsIgnoreCase("sunset") ){
sort = new Sort(Entity2LuceneDoc.term.SUNSET);
}else{
sort = new Sort(Entity2LuceneDoc.term.NAMEUNANALYZED);
}
} else {
sort = new Sort(Entity2LuceneDoc.term.NAMEUNANALYZED);
}
if( depth > 1 && "rand()".equalsIgnoreCase(sortField) ){
sort = null;
}
TopDocs docs;
if( sort != null )
docs = index.search(query, null, MAX_RESULTS, sort);
else
docs = index.search(query, null, MAX_RESULTS);
if( docs == null ){
log.error("Search of lucene index returned null");
return 0;
}
size = docs.totalHits;
// don't get all the results, only get results for the requestedSize
List<Individual> results = new ArrayList<Individual>(entsPerTab);
int entsAddedToTab = 0;
int ii = (page-1)*entsPerTab;
boolean doingRandom = false;
if( !doAlphaFilter && !doPagedFilter && depth > 1 && size > 1 && "rand()".equalsIgnoreCase(tab.getEntitySortField())){
doingRandom = true;
ii = random.nextInt( size );
}
boolean looped = false;
while( entsAddedToTab < entsPerTab && ii < docs.scoreDocs.length ){
ScoreDoc hit = docs.scoreDocs[ii];
if (hit != null) {
Document doc = index.doc(hit.doc);
if (doc != null) {
if( onlyWithThumbImg && "0".equals(doc.getField(Entity2LuceneDoc.term.THUMBNAIL).stringValue()) ){
//do Nothing
}else{
String uri = doc.getField(Entity2LuceneDoc.term.URI).stringValue();
Individual ind = indDao.getIndividualByURI( uri );
results.add( ind );
entsAddedToTab++;
}
} else {
log.warn("no document found for lucene doc id " + hit.doc);
}
} else {
log.debug("hit was null");
}
if( doingRandom && ii >= docs.scoreDocs.length && ! looped){
ii=0;
looped = true;
}else{
ii++;
}
}
//mutate state of tab
tab.setRelatedEntityList(results);
}catch(IOException ex){
log.warn(ex,ex);
return size;
}
return size;
}
private boolean tabNotLinked(Tab tab, List<String> autoLinkedUris,
List<String> manuallyLinkedUris) {
if( tab.isManualLinked() )
return manuallyLinkedUris.isEmpty();
else if( tab.isAutoLinked() )
return autoLinkedUris.isEmpty();
else
return autoLinkedUris.isEmpty() && manuallyLinkedUris.isEmpty();
}
private BooleanQuery addDayLimit(BooleanQuery query, Tab tab) {
DateTime now = new DateTime();
if( tab.getDayLimit() > 0 ){
String start = now.toString(LuceneIndexer.DATE_FORMAT);
String future = now.plusDays( tab.getDayLimit() ).toString(LuceneIndexer.DATE_FORMAT);
query.add( new RangeQuery(new Term(Entity2LuceneDoc.term.TIMEKEY,start),new Term(Entity2LuceneDoc.term.TIMEKEY,future), true) , BooleanClause.Occur.MUST);
}else{
String end = now.toString(LuceneIndexer.DATE_FORMAT);
String past = now.minusDays( tab.getDayLimit() ).toString(LuceneIndexer.DATE_FORMAT);
query.add( new RangeQuery(new Term(Entity2LuceneDoc.term.SUNRISE,past),new Term(Entity2LuceneDoc.term.SUNRISE,end), true) , BooleanClause.Occur.MUST);
}
return query;
}
private void doAlphaFiltered(String alpha, Tab tab,
HttpServletRequest request, HttpServletResponse response)
VitroRequest request, HttpServletResponse response, TabDao tabDao, int size)
throws ServletException, IOException {
Collection ents = tab.getRelatedEntityList(alpha);
if( ents != null ) {
request.setAttribute("entities", ents);
request.setAttribute("alpha",alpha);
request.setAttribute("count",tab.grabEntityFactory().getRelatedEntityCount()+"");
log.debug("in doAlphaFitlered");
request.setAttribute("entities", tab.getRelatedEntities());
request.setAttribute("alpha", alpha);
request.setAttribute("count", Integer.toString(size) );
request.setAttribute("tabParam",tab.getTabDepthName()+"="+tab.getTabId());
request.setAttribute("letters",tab.grabEntityFactory().getLettersOfEnts());
request.setAttribute("letters",Controllers.getLetters());
//request.setAttribute("letters",tab.grabEntityFactory().getLettersOfEnts());
request.setAttribute("servlet",Controllers.TAB);
String jsp = Controllers.ENTITY_LIST_FOR_TABS_JSP;
RequestDispatcher rd =
request.getRequestDispatcher(jsp);
rd.include(request, response);
}else{
//no entities, do nothing
}
}
private void doGallery(Tab tab, HttpServletRequest request, HttpServletResponse response)
private void doAutoLinked(Tab tab, VitroRequest request, HttpServletResponse response, TabDao tabDao, int size)
throws ServletException, IOException {
Collection ents = tab.getRelatedEntityList(null);
if( ents != null && ents.size() > 0){
request.setAttribute("entities", ents);
request.setAttribute("rows", tab.getGalleryRows());
request.setAttribute("columns", tab.getGalleryCols());
String jsp = Controllers.TAB_ENTITIES_LIST_GALLERY_JSP;
RequestDispatcher rd =
request.getRequestDispatcher(jsp);
rd.include(request, response);
}else{
doAutoLinked(tab,request,response);
}
}
log.debug("in doAutoLinked");
private void doAutoLinked(Tab tab, HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Collection ents = tab.getRelatedEntityList(null);
if( ents != null && ents.size() > 0 ) {
request.setAttribute("entities", ents);
request.setAttribute("alpha","none");
request.setAttribute("count",tab.getAlphaUnfilteredEntityCount()+"");
request.setAttribute("entities", tab.getRelatedEntities());
request.setAttribute("count", Integer.toString(size));
request.setAttribute("tabParam",tab.getTabDepthName()+"="+tab.getTabId());
request.setAttribute("letters",Controllers.getLetters());
request.setAttribute("servlet",Controllers.TAB);
@ -175,22 +368,224 @@ public void doGet( HttpServletRequest req, HttpServletResponse response )
RequestDispatcher rd =
request.getRequestDispatcher(jsp);
rd.include(request, response);
}else{
//no entities, do nothing
}
private int getPage(VitroRequest request) {
String p = request.getParameter("page") ;
if( p == null )
return 1;
try{
return Integer.parseInt(p);
}catch(Exception e){
return 1;
}
}
private void doManual(Tab tab, HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
Collection ents = tab.getRelatedEntityList(null);
if( ents != null && ents.size() > 0 ) {
request.setAttribute("entities", ents);
String jsp = Controllers.ENTITY_LIST_FOR_TABS_JSP;
RequestDispatcher rd =
request.getRequestDispatcher(jsp);
rd.include(request, response);
/**
* Mixed and auto linked tabs get their individuals via
* a query to the lucene index. This is to allow sorting and alpha
* filtering of the combined list. Manually linked tabs do not
* get their individuals using lucene, see doManual().
*
*/
private BooleanQuery getQuery(Tab tab, List<String> vclassUris, List<String> manualUris, String alpha , boolean isSinglePortal){
BooleanQuery query = new BooleanQuery();
try{
//make the type query if needed
BooleanQuery typeQuery = null;
if( tab.isAutoLinked() || tab.isMixedLinked() ){
typeQuery = new BooleanQuery();
BooleanQuery queryForTypes = new BooleanQuery();
//setup up type queries in their own sub query
for( String vclassUri : vclassUris ){
if( vclassUri != null && !"".equals(vclassUri)){
queryForTypes.add(
new TermQuery( new Term(Entity2LuceneDoc.term.RDFTYPE, vclassUri)),
BooleanClause.Occur.SHOULD );
}
}
typeQuery.add(queryForTypes, BooleanClause.Occur.MUST);
//check for portal filtering only on auto linked queries
if( ! isSinglePortal ){
int tabPortal = tab.getPortalId();
if( tabPortal < 16 ){ //could be a normal portal
typeQuery.add(
new TermQuery( new Term(Entity2LuceneDoc.term.PORTAL, Integer.toString(1 << tab.getPortalId() ))),
BooleanClause.Occur.MUST);
}else{ //could be a combined portal
BooleanQuery tabQueries = new BooleanQuery();
Long[] ids= FlagMathUtils.numeric2numerics(tabPortal);
for( Long id : ids){
tabQueries.add(
new TermQuery( new Term(Entity2LuceneDoc.term.PORTAL,id.toString()) ),
BooleanClause.Occur.SHOULD);
}
typeQuery.add(tabQueries,BooleanClause.Occur.MUST);
}
}
String flag2Set = tab.getFlag2Set();
if( tab.getFlag2Set() != null && ! tab.getFlag2Set().isEmpty()){
if( flag2Set != null && ! "".equals(flag2Set)){
BooleanQuery flag2Query = new BooleanQuery();
for( String flag2Value : flag2Set.split(",")){
if( flag2Value != null ){
String value = flag2Value.replace(",", "");
if(!value.isEmpty()){
flag2Query.add(new TermQuery(new Term(Entity2LuceneDoc.term.FLAG2,value)),
BooleanClause.Occur.SHOULD);
}
}
}
typeQuery.add(flag2Query, BooleanClause.Occur.MUST);
}
}
}
//make query for manually linked individuals
BooleanQuery manualQuery = null;
boolean foundManual = false;
if( tab.isManualLinked() || tab.isMixedLinked()){
manualQuery = new BooleanQuery();
BooleanQuery queryForManual = new BooleanQuery();
for( String indURI : manualUris){
if( indURI != null ){
queryForManual.add(
new TermQuery( new Term(Entity2LuceneDoc.term.URI, indURI)),
BooleanClause.Occur.SHOULD );
foundManual = true;
}
}
if( foundManual )
manualQuery.add( queryForManual, BooleanClause.Occur.MUST);
}
if( tab.isAutoLinked() || !foundManual ){
query = typeQuery;
}else if ( tab.isManualLinked() ){
query = manualQuery;
}else{
//no entities, do nothing
BooleanQuery orQuery = new BooleanQuery();
orQuery.add( typeQuery, BooleanClause.Occur.SHOULD);
orQuery.add( manualQuery, BooleanClause.Occur.SHOULD);
query.add( orQuery, BooleanClause.Occur.MUST);
}
//Add alpha filter if it is needed
Query alphaQuery = null;
if( alpha != null && !"".equals(alpha) && alpha.length() == 1){
alphaQuery =
new PrefixQuery(new Term(Entity2LuceneDoc.term.NAMEUNANALYZED, alpha.toLowerCase()));
query.add(alphaQuery,BooleanClause.Occur.MUST);
}
log.debug("Query for tab " + tab.getTabId() + ": " + query);
return query;
}catch (Exception ex){
log.error(ex,ex);
return new BooleanQuery();
}
}
private int getSizeForGalleryTab(Tab tab){
int rows = tab.getGalleryRows() != 0 ? tab.getGalleryRows() : 8;
int col = tab.getGalleryCols() != 0 ? tab.getGalleryCols() : 8;
return rows * col;
}
private int getSizeForNonGalleryTab(Tab tab, boolean showPaged ){
if( showPaged )
if( tab.getGalleryCols() == 0 || tab.getGalleryRows() == 0 )
return 8;
else
return getSizeForGalleryTab(tab);
else
return NON_PAGED_LIMIT;
}
private int getTabDepth(VitroRequest request){
String obj = null;
try {
obj = request.getParameter("tabDepth");
if( obj == null ){
String e="TabEntitesController expects that request parameter 'tabDepth' be set"
+", use 1 as the leading tab's depth.";
throw new ServletException(e);
}
return Integer.parseInt((String)obj);
}catch(Exception ex){
return 1;
}
}
public static List<PageRecord> makePagesList( int count, int pageSize, int selectedPage){
List<PageRecord> records = new ArrayList<PageRecord>( MAX_PAGES + 1 );
int requiredPages = count/pageSize ;
int remainder = count % pageSize ;
if( remainder > 0 )
requiredPages++;
if( selectedPage < MAX_PAGES && requiredPages > MAX_PAGES ){
//the selected pages is within the first maxPages, just show the normal pages up to maxPages.
for(int page = 1; page < requiredPages && page <= MAX_PAGES ; page++ ){
records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) );
}
records.add( new PageRecord( "page="+ MAX_PAGES+1, Integer.toString(MAX_PAGES+1), "more...", false));
}else if( requiredPages > MAX_PAGES && selectedPage+1 > MAX_PAGES && selectedPage < requiredPages - MAX_PAGES){
//the selected pages is in the middle of the list of page
int startPage = selectedPage - MAX_PAGES / 2;
int endPage = selectedPage + MAX_PAGES / 2;
for(int page = startPage; page <= endPage ; page++ ){
records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) );
}
records.add( new PageRecord( "page="+ endPage+1, Integer.toString(endPage+1), "more...", false));
}else if ( requiredPages > MAX_PAGES && selectedPage > requiredPages - MAX_PAGES ){
//the selected page is in the end of the list
int startPage = requiredPages - MAX_PAGES;
double max = Math.ceil(count/pageSize);
for(int page = startPage; page <= max; page++ ){
records.add( new PageRecord( "page=" + page, Integer.toString(page), Integer.toString(page), selectedPage == page ) );
}
}else{
//there are fewer than maxPages pages.
for(int i = 1; i <= requiredPages; i++ ){
records.add( new PageRecord( "page=" + i, Integer.toString(i), Integer.toString(i), selectedPage == i ) );
}
}
return records;
}
public static class PageRecord {
public PageRecord(String param, String index, String text, boolean selected) {
this.param = param;
this.index = index;
this.text = text;
this.selected = selected;
}
public String param;
public String index;
public String text;
public boolean selected=false;
public String getParam() {
return param;
}
public String getIndex() {
return index;
}
public String getText() {
return text;
}
public boolean getSelected(){
return selected;
}
}
}

View file

@ -29,6 +29,7 @@ import com.hp.hpl.jena.util.iterator.ClosableIterator;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Tab;
import edu.cornell.mannlib.vitro.webapp.beans.TabIndividualRelation;
import edu.cornell.mannlib.vitro.webapp.dao.TabDao;
import edu.cornell.mannlib.vitro.webapp.dao.TabEntityFactory;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
@ -262,30 +263,16 @@ public class TabDaoJena extends JenaBaseDao implements TabDao {
/**
* returns a list of URI strings of Entities manually linked to tab (tabId)
*/
public List<String> getTabManuallyLinkedEntityURIs(int tabId) {
public List<String> getTabManuallyLinkedEntityURIs(int tab_id) {
List<String> entityURIs = new LinkedList<String>();
// getOntModel().enterCriticalSection(Lock.READ);
// try {
// Resource tab = getOntModel().getResource(DEFAULT_NAMESPACE+"tab"+tabId);
// if (tab != null && TAB_LINKEDENTITY != null) {
// ClosableIterator entityIt = getOntModel().listStatements(tab, TAB_LINKEDENTITY, (Resource)null);
// try {
// while (entityIt.hasNext()) {
// Statement st = (Statement) entityIt.next();
// Resource entity = (Resource) st.getObject();
// if (entity != null) {
// entityURIs.add(entity.getURI());
// }
// }
// } catch (Exception e) {
// e.printStackTrace();
// } finally {
// entityIt.close();
// }
// }
// } finally {
// getOntModel().leaveCriticalSection();
// }
TabIndividualRelationDaoJena tabToIndDao = new TabIndividualRelationDaoJena( getWebappDaoFactory() );
List<TabIndividualRelation> tabsToInd = tabToIndDao.getTabIndividualRelationsByTabURI( DEFAULT_NAMESPACE+"tab"+tab_id );
if( tabsToInd != null ){
for( TabIndividualRelation rel : tabsToInd){
if( rel != null && rel.getEntURI() != null)
entityURIs.add( rel.getEntURI() );
}
}
return entityURIs;
}

View file

@ -19,6 +19,7 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryFactory;
*
* @author bdc34
*
*@deprecated Use LuceneIndexFactory instead
*/
public interface Searcher {

View file

@ -26,6 +26,7 @@ import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
*
* @author bdc34
*
* @deprecated there is no replacement
*/
public abstract class VitroHighlighter extends UnaryFunctor<String,String> {

View file

@ -79,7 +79,9 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryWrapper;
* are different ways of doing this. We will just get an object out
* of the application scope. This object was set when the context
* started up and can be set in the web.xml. See LuceneSetup.java
* for an exmple.
* for an example.
*
* @deprecated Use PagedSearchController instead.
*/
public class SearchController extends VitroHttpServlet{
private static final Log log = LogFactory.getLog(SearchController.class.getName());

View file

@ -20,6 +20,8 @@ public class IndexBuilderThread extends Thread{
public IndexBuilderThread(IndexBuilder ib){
super("IndexBuilderThread");
if( ib == null )
log.error("IndexBuilderThread needs an IndexBuilder, search is not configured.");
this.indexBuilder = ib;
}
@ -31,8 +33,11 @@ public class IndexBuilderThread extends Thread{
return;
}
if( indexBuilder == null )
log.warn("IndexBuilderThread needs a IndexBuilder, search may not be configured.");
try{
if( indexBuilder.isReindexRequested() ){
if( indexBuilder != null && indexBuilder.isReindexRequested() ){
log.debug("full re-index requested");
indexBuilder.indexRebuild();
}else{

View file

@ -2,7 +2,9 @@
package edu.cornell.mannlib.vitro.webapp.search.lucene;
import java.util.HashSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@ -12,15 +14,12 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.joda.time.DateTime;
import org.openrdf.model.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.docbuilder.Obj2DocIface;
import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils;
@ -51,11 +50,15 @@ public class Entity2LuceneDoc implements Obj2DocIface{
public static String NAMEUNANALYZED = "nameunanalyzed" ;
/** Name of entity, unstemmed */
public static String NAMEUNSTEMMED = "nameunstemmed";
/** Name of portal */
/** portal ( 2 ^ portalId ) */
public static String PORTAL = "portal";
/** Flag 2 (legacy, only used at Cornell) */
public static String FLAG2 = "FLAG2";
/** time of index in msec since epoc */
public static String INDEXEDTIME= "indexedTime";
/** time of sunset/end of entity */
/** timekey of entity in yyyymmddhhmm */
public static String TIMEKEY="TIMEKEY";
/** time of sunset/end of entity in yyyymmddhhmm */
public static String SUNSET="SUNSET";
/** time of sunrise/start of entity in yyyymmddhhmm */
public static String SUNRISE="SUNRISE";
@ -66,6 +69,8 @@ public class Entity2LuceneDoc implements Obj2DocIface{
public static String ALLTEXTUNSTEMMED = "ALLTEXTUNSTEMMED";
/** keywords */
public static final String KEYWORDS = "KEYWORDS";
/** Does the individual have a thumbnail image? 1=yes 0=no */
public static final String THUMBNAIL = "THUMBNAIL";
}
private static final Log log = LogFactory.getLog(Entity2LuceneDoc.class.getName());
@ -108,10 +113,12 @@ public class Entity2LuceneDoc implements Obj2DocIface{
doc.add( new Field(term.JCLASS, entClassName, Field.Store.YES, Field.Index.NOT_ANALYZED));
//Entity Name
if( ent.getName() != null )
value=ent.getName();
else
value="";
if( ent.getRdfsLabel() != null )
value=ent.getRdfsLabel();
else{
log.debug("Skipping individual without rdfs:label " + ent.getURI());
return null;
}
Field name =new Field(term.NAME, value,
Field.Store.YES, Field.Index.ANALYZED);
name.setBoost( NAME_BOOST );
@ -123,7 +130,7 @@ public class Entity2LuceneDoc implements Obj2DocIface{
doc.add( nameUn );
Field nameUnanalyzed = new Field(term.NAMEUNANALYZED, value.toLowerCase(),
Field.Store.NO, Field.Index.NOT_ANALYZED);
Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add( nameUnanalyzed );
//boost for entity
@ -200,6 +207,26 @@ public class Entity2LuceneDoc implements Obj2DocIface{
doc.add(new Field(term.SUNSET, latestTime, Field.Store.YES, Field.Index.NOT_ANALYZED));
}
try{
value = null;
if( ent.getTimekey() != null ){
value = (new DateTime(ent.getTimekey().getTime())).toString(LuceneIndexer.DATE_FORMAT);
doc.add(new Field(term.TIMEKEY, value, Field.Store.YES, Field.Index.NOT_ANALYZED));
}
}catch(Exception ex){
log.error("could not save timekey " + ex);
}
try{
value = null;
if( ent.getThumbUrl() != null )
doc.add(new Field(term.THUMBNAIL, "1", Field.Store.YES, Field.Index.NOT_ANALYZED));
else
doc.add(new Field(term.THUMBNAIL, "0", Field.Store.YES, Field.Index.NOT_ANALYZED));
}catch(Exception ex){
log.debug("could not index thumbnail: " + ex);
}
//time of index in millis past epoc
Object anon[] = { new Long((new DateTime() ).getMillis()) };
doc.add( new Field(term.INDEXEDTIME, String.format( "%019d", anon ),
@ -208,6 +235,8 @@ public class Entity2LuceneDoc implements Obj2DocIface{
//portal Flags
doPortalFlags(ent, doc);
//do flag 2 legacy, only used at Cornell
doFlag2( ent, doc );
//ALLTEXT, all of the 'full text'
String t=null;
@ -252,6 +281,23 @@ public class Entity2LuceneDoc implements Obj2DocIface{
return doc;
}
/**
* Flag two is a legacy field that is used only by Cornell.
* It is related to the old portal filtering.
*/
private void doFlag2(Individual ent, Document doc) {
String flag2Set = ent.getFlag2Set();
if( flag2Set != null && ! "".equals(flag2Set)){
for( String flag2Value : flag2Set.split(",")){
if( flag2Value != null ){
String value = flag2Value.replace(",", "");
if(!value.isEmpty())
doc.add( new Field(term.FLAG2, value, Field.Store.NO, Field.Index.ANALYZED));
}
}
}
}
/**
* Splits up the entity's flag1 value into portal id and then
* adds the id to the doc.
@ -283,17 +329,20 @@ public class Entity2LuceneDoc implements Obj2DocIface{
if( portalIds == null || portalIds.length == 0)
return;
// System.out.print('\n'+"numeric: " + ent.getFlag1Numeric()
// + " " + Arrays.toString(portalIds) +" = ");
//
log.debug("Flag 1 numeric: " + ent.getFlag1Numeric() + " for " + ent.getURI());
long id = -1;
for( Long idLong : portalIds){
if( idLong != null ){
id = idLong.longValue();
String numericPortal = Long.toString(id);
if( numericPortal != null ){
doc.add( new Field(term.PORTAL,numericPortal,
Field.Store.NO, Field.Index.NOT_ANALYZED));
// System.out.print(numericPortal+" ");
}/* end of portal id code */
log.debug("adding portal " + numericPortal + " to " + ent.getURI());
}
}
}
}
@SuppressWarnings("static-access")

View file

@ -5,6 +5,7 @@ package edu.cornell.mannlib.vitro.webapp.search.lucene;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
@ -38,10 +39,11 @@ public class LuceneIndexer implements IndexerIface {
LinkedList<Obj2DocIface> obj2DocList = new LinkedList<Obj2DocIface>();
String indexDir = null;
Analyzer analyzer = null;
List<Searcher> searchers = null;
List<Searcher> searchers = Collections.EMPTY_LIST;
IndexWriter writer = null;
boolean indexing = false;
HashSet<String> urisIndexed;
private LuceneIndexFactory luceneIndexFactory;
//JODA timedate library can use java date format strings.
//http://java.sun.com/j2se/1.3/docs/api/java/text/SimpleDateFormat.html
@ -71,6 +73,7 @@ public class LuceneIndexer implements IndexerIface {
public LuceneIndexer(String indexDir, List<Searcher> searchers, Analyzer analyzer ) throws IOException{
this.indexDir = indexDir;
this.analyzer = analyzer;
if( searchers != null )
this.searchers = searchers;
makeIndexIfNone();
}
@ -167,6 +170,9 @@ public class LuceneIndexer implements IndexerIface {
for( Searcher s : searchers){
s.close();
}
//this is the call that replaces Searcher.close()
luceneIndexFactory.forceNewIndexSearcher();
} catch (IOException e) {
log.error("LuceneIndexer.endIndexing() - "
+ "unable to optimize lucene index: \n" + e);
@ -216,7 +222,8 @@ public class LuceneIndexer implements IndexerIface {
log.debug("added " + ind.getName() + " " + ind.getURI());
}
}else{
log.debug("could not translate " + ind.getURI());
log.debug("could not translate, removing from index " + ind.getURI());
writer.deleteDocuments((Term)obj2doc.getIndexId(ind));
}
}
}
@ -251,23 +258,18 @@ public class LuceneIndexer implements IndexerIface {
* clear the index by deleting the directory and make a new empty index.
*/
public synchronized void clearIndex() throws IndexingException{
// if( indexing )
// throw new IndexingException("Cannot clear search index because an" +
// "index rebuild in in progress.");
log.debug("Clearing the index at "+indexDir);
closeModifier();
deleteDir(new File(indexDir));
//might not be thread safe since searchers can try to open a new index
// for(LuceneSearcher s : searchers){
// s.close();
// }
try {
makeNewIndex();
for(Searcher s : searchers){
s.close();
}
//this is the call that replaces Searcher.close()
luceneIndexFactory.forceNewIndexSearcher();
} catch (IOException e) {
throw new IndexingException(e.getMessage());
}
@ -330,5 +332,8 @@ public class LuceneIndexer implements IndexerIface {
return dir.delete();
}
public void setLuceneIndexFactory(LuceneIndexFactory lif) {
luceneIndexFactory = lif;
}
}

View file

@ -75,22 +75,17 @@ public class LuceneSetupCJK implements javax.servlet.ServletContextListener {
objectPropertyBlacklist.add("http://www.w3.org/2002/07/owl#differentFrom");
context.setAttribute(LuceneSetup.SEARCH_OBJECTPROPERTY_BLACKLIST, objectPropertyBlacklist);
//Here we want to put the LuceneSearcher in the application scope.
// the queries need to know the analyzer to use so that the same one can be used
// to analyze the fields in the incoming user query terms.
LuceneSearcher searcher = new LuceneSearcher(
new LuceneQueryFactory(getAnalyzer(), Entity2LuceneDoc.term.ALLTEXT),
indexDir);
searcher.addObj2Doc(new Entity2LuceneDoc());
context.setAttribute(Searcher.class.getName(), searcher);
//here we want to put the LuceneIndex object into the application scope
LuceneIndexer indexer = new LuceneIndexer(indexDir, null, getAnalyzer());
indexer.addSearcher(searcher);
context.setAttribute(LuceneSetup.ANALYZER, getAnalyzer());
context.setAttribute(LuceneSetup.INDEX_DIR, indexDir);
indexer.addObj2Doc(new Entity2LuceneDoc());
//This is where to get a LucenIndex from. The indexer will
//need to reference this to notify it of updates to the index
LuceneIndexFactory lif = LuceneIndexFactory.getLuceneIndexFactoryFromContext(context);
indexer.setLuceneIndexFactory(lif);
//This is where the builder gets the list of places to try to
//get objects to index. It is filtered so that non-public text
//does not get into the search index.

View file

@ -10,7 +10,7 @@
request attributres:
'alpha' - set to currently displaying alpha, 'none' or 'all'
'tabParam' - parameter for tab
'controllerParam' - parameter for controller
'count' - count of entites in the index
'letters' - List of STrings, letters for index.
'servlet' - name of servlet to put in links.
@ -35,14 +35,25 @@
%>
<c:set var="portalId" value="<%=portalId%>" />
<c:if test="${requestScope.alpha != 'none'}">
<c:if test="${ requestScope.showAlpha == 1 }">
<div class='alphaIndex'>
<c:forEach items='${requestScope.letters}' var='letter'>
<a <c:if test="${letter == requestScope.alpha}">class='selected' </c:if> href='<c:url value=".${requestScope.servlet}?home=${portalId}&amp;alpha=${letter}&amp;${requestScope.tabParam}"/>'>${letter} </a>
</c:forEach>
<c:if test='${not empty requestScope.count}'>
(${requestScope.count})
<c:if test="${letter == requestScope.alpha}"> ${requestScope.alpha }&nbsp;</c:if>
<c:if test="${letter != requestScope.alpha}">
<c:url var="url" value=".${requestScope.servlet}">
<c:param name="alpha">${letter}</c:param>
</c:url>
<a href='<c:url value="${url}&amp;${requestScope.controllerParam}"/>'>${letter} </a>
</c:if>
</c:forEach>
<% if( request.getAttribute("alpha") != null && ! "all".equalsIgnoreCase((String)request.getAttribute("alpha"))) { %>
<a href='<c:url value=".${requestScope.servlet}?&amp;alpha=all&amp;${requestScope.controllerParam}"/>'>all </a>
<c:if test='${not empty requestScope.count }'>
(${requestScope.count} that start with ${requestScope.alpha })
</c:if>
<% }else{ %>
(${requestScope.count})
<% } %>
</div>
</c:if>