litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Changes to lucen index creation NIHVIVO-1128

Browse source
This commit is contained in:
bdc34 2010-10-06 21:51:59 +00:00
parent 5002c3d761
commit 1041759072
5 changed files with 51 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

6
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/IndividualJena.java
View file

@ -585,7 +585,7 @@ public class IndividualJena extends IndividualImpl implements Individual {
try { try {
webappDaoFactory.getLinksDao().addLinksToIndividual( this ); webappDaoFactory.getLinksDao().addLinksToIndividual( this );
} catch (Exception e) { } catch (Exception e) {
log.error(this.getClass().getName()+" could not addLinksToIndividual for "+this.getURI()); log.debug(this.getClass().getName()+" could not addLinksToIndividual for "+this.getURI());
} }
return this.linksList; return this.linksList;
} }
@ -598,7 +598,7 @@ public class IndividualJena extends IndividualImpl implements Individual {
try { try {
webappDaoFactory.getLinksDao().addPrimaryLinkToIndividual( this ); webappDaoFactory.getLinksDao().addPrimaryLinkToIndividual( this );
} catch (Exception e) { } catch (Exception e) {
log.error(this.getClass().getName()+" could not addPrimaryLinkToIndividual for "+this.getURI()); log.debug(this.getClass().getName()+" could not addPrimaryLinkToIndividual for "+this.getURI());
} }
return this.primaryLink; return this.primaryLink;
} }
@ -612,7 +612,7 @@ public class IndividualJena extends IndividualImpl implements Individual {
try { try {
this.setKeywords(webappDaoFactory.getIndividualDao().getKeywordsForIndividual(this.getURI())); this.setKeywords(webappDaoFactory.getIndividualDao().getKeywordsForIndividual(this.getURI()));
} catch (Exception e) { } catch (Exception e) {
log.error(this.getClass().getName()+" could not getKeywords for "+this.getURI()); log.debug(this.getClass().getName()+" could not getKeywords for "+this.getURI());
} }
return this.keywords; return this.keywords;
} }

7
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/LinksDaoJena.java
View file

@ -13,6 +13,9 @@ import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.List; import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.datatypes.xsd.XSDDatatype; import com.hp.hpl.jena.datatypes.xsd.XSDDatatype;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.ObjectProperty; import com.hp.hpl.jena.ontology.ObjectProperty;
@ -33,6 +36,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
public class LinksDaoJena extends JenaBaseDao implements LinksDao { public class LinksDaoJena extends JenaBaseDao implements LinksDao {
protected static final Log log = LogFactory.getLog(LinksDaoJena.class.getName());
public LinksDaoJena(WebappDaoFactoryJena wadf) { public LinksDaoJena(WebappDaoFactoryJena wadf) {
super(wadf); super(wadf);
} }
@ -329,7 +334,7 @@ public class LinksDaoJena extends JenaBaseDao implements LinksDao {
} else if (l.getDatatype()==XSDDatatype.XSDstring) { } else if (l.getDatatype()==XSDDatatype.XSDstring) {
link.setDisplayRank(l.getString()); link.setDisplayRank(l.getString());
} else { } else {
log.error("unexpected literal datatype; saved as displayRank 10"); log.debug("unexpected literal datatype; saved as displayRank 10");
link.setDisplayRank("10"); link.setDisplayRank("10");
} }
} }

65
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java
View file

@ -4,16 +4,12 @@ package edu.cornell.mannlib.vitro.webapp.search.controller;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.ServletConfig; import javax.servlet.ServletConfig;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
@ -35,8 +31,6 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@ -44,12 +38,9 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServ
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.flags.PortalFlag; import edu.cornell.mannlib.vitro.webapp.flags.PortalFlag;
import edu.cornell.mannlib.vitro.webapp.search.SearchException; import edu.cornell.mannlib.vitro.webapp.search.SearchException;
import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroHighlighter;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQuery;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryFactory;
import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc; import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexer; import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexer;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexFactory;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup; import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup;
import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils; import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils;
import freemarker.template.Configuration; import freemarker.template.Configuration;
@ -59,7 +50,7 @@ import freemarker.template.Configuration;
* through a Lucene search. * through a Lucene search.
*/ */
public class AutocompleteController extends FreemarkerHttpServlet implements Searcher{ public class AutocompleteController extends FreemarkerHttpServlet{
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(AutocompleteController.class); private static final Log log = LogFactory.getLog(AutocompleteController.class);
@ -68,24 +59,9 @@ public class AutocompleteController extends FreemarkerHttpServlet implements Sea
private static String QUERY_PARAMETER_NAME = "term"; private static String QUERY_PARAMETER_NAME = "term";
private IndexSearcher searcher = null;
String NORESULT_MSG = ""; String NORESULT_MSG = "";
private int defaultMaxSearchSize= 1000; private int defaultMaxSearchSize= 1000;
public void init(ServletConfig config) throws ServletException {
super.init(config);
LuceneIndexer indexer=(LuceneIndexer)getServletContext()
.getAttribute(LuceneIndexer.class.getName());
indexer.addSearcher(this);
try{
String indexDir = getIndexDir(getServletContext());
getIndexSearcher(indexDir);
}catch(Exception ex){
}
}
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException { throws ServletException, IOException {
doGet(request, response); doGet(request, response);
@ -125,9 +101,9 @@ public class AutocompleteController extends FreemarkerHttpServlet implements Sea
doNoQuery(map, config, response); doNoQuery(map, config, response);
return; return;
} }
IndexSearcher searcherForRequest = LuceneIndexFactory.getIndexSearcher(getServletContext());
IndexSearcher searcherForRequest = getIndexSearcher(indexDir);
TopDocs topDocs = null; TopDocs topDocs = null;
try{ try{
topDocs = searcherForRequest.search(query,null,maxHitSize); topDocs = searcherForRequest.search(query,null,maxHitSize);
@ -428,21 +404,6 @@ public class AutocompleteController extends FreemarkerHttpServlet implements Sea
return qp; return qp;
} }
private synchronized IndexSearcher getIndexSearcher(String indexDir) {
if( searcher == null ){
try {
Directory fsDir = FSDirectory.getDirectory(indexDir);
searcher = new IndexSearcher(fsDir);
} catch (IOException e) {
log.error("LuceneSearcher: could not make indexSearcher "+e);
log.error("It is likely that you have not made a directory for the lucene index. "+
"Create the directory indicated in the error and set permissions/ownership so"+
" that the tomcat server can read/write to it.");
//The index directory is created by LuceneIndexer.makeNewIndex()
}
}
return searcher;
}
private void doNoQuery(Map<String, Object> map, Configuration config, HttpServletResponse response) { private void doNoQuery(Map<String, Object> map, Configuration config, HttpServletResponse response) {
@ -491,24 +452,6 @@ public class AutocompleteController extends FreemarkerHttpServlet implements Sea
} }
} }
/**
* Need to accept notification from indexer that the index has been changed.
*/
public void close() {
searcher = null;
}
public VitroHighlighter getHighlighter(VitroQuery q) {
throw new Error("AutocompleteController.getHighlighter() is unimplemented");
}
public VitroQueryFactory getQueryFactory() {
throw new Error("AutocompleteController.getQueryFactory() is unimplemented");
}
public List search(VitroQuery query) throws SearchException {
throw new Error("AutocompleteController.search() is unimplemented");
}
} }

232
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java
View file

@ -16,7 +16,6 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import javax.servlet.RequestDispatcher; import javax.servlet.RequestDispatcher;
import javax.servlet.ServletConfig;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
@ -38,39 +37,25 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import edu.cornell.mannlib.vitro.webapp.beans.DataProperty;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.Portal; import edu.cornell.mannlib.vitro.webapp.beans.Portal;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.Controllers; import edu.cornell.mannlib.vitro.webapp.controller.Controllers;
import edu.cornell.mannlib.vitro.webapp.controller.VitroHttpServlet; import edu.cornell.mannlib.vitro.webapp.controller.VitroHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.dao.DataPropertyDao;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.ObjectPropertyDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao; import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao; import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupDao;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary; import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.flags.PortalFlag; import edu.cornell.mannlib.vitro.webapp.flags.PortalFlag;
import edu.cornell.mannlib.vitro.webapp.search.SearchException; import edu.cornell.mannlib.vitro.webapp.search.SearchException;
import edu.cornell.mannlib.vitro.webapp.search.beans.Searcher;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroHighlighter;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQuery; import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQuery;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryFactory;
import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc; import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexer; import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexFactory;
import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup; import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup;
import edu.cornell.mannlib.vitro.webapp.search.lucene.SimpleLuceneHighlighter;
import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils; import edu.cornell.mannlib.vitro.webapp.utils.FlagMathUtils;
import edu.cornell.mannlib.vitro.webapp.utils.Html2Text;
/** /**
* PagedSearchController is the new search controller that interacts * PagedSearchController is the new search controller that interacts
@ -79,27 +64,12 @@ import edu.cornell.mannlib.vitro.webapp.utils.Html2Text;
* @author bdc34 * @author bdc34
* *
*/ */
public class PagedSearchController extends VitroHttpServlet implements Searcher{ public class PagedSearchController extends VitroHttpServlet {
private IndexSearcher searcher = null;
private static final Log log = LogFactory.getLog(PagedSearchController.class.getName()); private static final Log log = LogFactory.getLog(PagedSearchController.class.getName());
String NORESULT_MSG = "The search returned no results."; String NORESULT_MSG = "The search returned no results.";
private int defaultHitsPerPage = 25; private int defaultHitsPerPage = 25;
private int defaultMaxSearchSize= 1000; private int defaultMaxSearchSize= 1000;
public void init(ServletConfig config) throws ServletException {
super.init(config);
LuceneIndexer indexer=(LuceneIndexer)getServletContext()
.getAttribute(LuceneIndexer.class.getName());
indexer.addSearcher(this);
try{
String indexDir = getIndexDir(getServletContext());
getIndexSearcher(indexDir);
}catch(Exception ex){
}
}
public void doPost(HttpServletRequest request, HttpServletResponse response) public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException { throws ServletException, IOException {
doGet(request, response); doGet(request, response);
@ -130,7 +100,8 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
startIndex = Integer.parseInt(request.getParameter("startIndex")); startIndex = Integer.parseInt(request.getParameter("startIndex"));
}catch (Throwable e) { }catch (Throwable e) {
startIndex = 0; startIndex = 0;
} }
log.debug("startIndex is " + startIndex);
int hitsPerPage = defaultHitsPerPage; int hitsPerPage = defaultHitsPerPage;
try{ try{
@ -138,6 +109,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
} catch (Throwable e) { } catch (Throwable e) {
hitsPerPage = defaultHitsPerPage; hitsPerPage = defaultHitsPerPage;
} }
log.debug("hitsPerPage is " + hitsPerPage);
int maxHitSize = defaultMaxSearchSize; int maxHitSize = defaultMaxSearchSize;
if( startIndex >= defaultMaxSearchSize - hitsPerPage ) if( startIndex >= defaultMaxSearchSize - hitsPerPage )
@ -146,15 +118,14 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
maxHitSize = maxHitSize * 2; maxHitSize = maxHitSize * 2;
hitsPerPage = maxHitSize; hitsPerPage = maxHitSize;
} }
log.debug("maxHitSize is " + maxHitSize);
String indexDir = getIndexDir(getServletContext());
String qtxt = vreq.getParameter(VitroQuery.QUERY_PARAMETER_NAME); String qtxt = vreq.getParameter(VitroQuery.QUERY_PARAMETER_NAME);
Analyzer analyzer = getAnalyzer(getServletContext()); Analyzer analyzer = getAnalyzer(getServletContext());
Query query = null; Query query = null;
try { try {
query = getQuery(vreq, portalFlag, analyzer, indexDir, qtxt); query = getQuery(vreq, portalFlag, analyzer, qtxt);
} catch (ParseException e) { } catch (ParseException e) {
log.warn("Query parse exception: " + e); log.warn("Query parse exception: " + e);
doBadQuery(qtxt, request, response); doBadQuery(qtxt, request, response);
@ -163,7 +134,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
log.debug("query for '" + qtxt +"' is " + query.toString()); log.debug("query for '" + qtxt +"' is " + query.toString());
IndexSearcher searcherForRequest = getIndexSearcher(indexDir); IndexSearcher searcherForRequest = LuceneIndexFactory.getIndexSearcher(getServletContext());
TopDocs topDocs = null; TopDocs topDocs = null;
try{ try{
@ -181,7 +152,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
doFailedSearch(request, response, msg, qtxt); doFailedSearch(request, response, msg, qtxt);
return; return;
} }
} }
if( topDocs == null || topDocs.scoreDocs == null){ if( topDocs == null || topDocs.scoreDocs == null){
log.error("topDocs for a search was null"); log.error("topDocs for a search was null");
@ -202,18 +173,25 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
lastHitToShow = hitsLength; lastHitToShow = hitsLength;
else else
lastHitToShow = startIndex + hitsPerPage - 1; lastHitToShow = startIndex + hitsPerPage - 1;
List<Individual> beans = new LinkedList<Individual>(); List<Individual> beans = new LinkedList<Individual>();
for(int i=startIndex; i<topDocs.scoreDocs.length ;i++){ for(int i=startIndex; i<topDocs.scoreDocs.length ;i++){
try{ try{
if( (i >= startIndex) && (i <= lastHitToShow) ){ if( (i >= startIndex) && (i <= lastHitToShow) ){
Document doc = searcherForRequest.doc(topDocs.scoreDocs[i].doc); Document doc = searcherForRequest.doc(topDocs.scoreDocs[i].doc);
String uri = doc.get(Entity2LuceneDoc.term.URI); String uri = doc.get(Entity2LuceneDoc.term.URI);
Individual ent = new IndividualImpl(); Individual ent = iDao.getIndividualByURI(uri);
ent.setURI(uri); if(ent != null ){
ent = iDao.getIndividualByURI(uri); List<VClass>vcs = ent.getVClasses() ;
if(ent!=null) if( vcs != null && !vcs.isEmpty() ){
beans.add(ent); beans.add(ent);
log.debug("found individual for search hit in model" + uri );
}else{
log.debug("filtered out classless individual from search results" + uri);
}
}else{
log.debug("could not find individual for search hit in model " + uri);
}
} }
}catch(Exception e){ }catch(Exception e){
log.error("problem getting usable Individuals from search " + log.error("problem getting usable Individuals from search " +
@ -225,7 +203,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
if( request.getParameter("classgroup") == null && request.getParameter("type") == null){ if( request.getParameter("classgroup") == null && request.getParameter("type") == null){
request.setAttribute("classgroups", request.setAttribute("classgroups",
getClassGroups(grpDao, topDocs, searcherForRequest)); getClassGroups(grpDao, topDocs, searcherForRequest));
request.setAttribute("refinement", ""); request.setAttribute("refinment", "");
}else{ }else{
//search request for a classgroup so add rdf:type search refinement links //search request for a classgroup so add rdf:type search refinement links
@ -233,23 +211,18 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
if( request.getParameter("classgroup") != null && request.getParameter("type") == null ){ if( request.getParameter("classgroup") != null && request.getParameter("type") == null ){
request.setAttribute("types", request.setAttribute("types",
getVClasses(vclassDao,topDocs,searcherForRequest)); getVClasses(vclassDao,topDocs,searcherForRequest));
request.setAttribute("refinement", "&classgroup=" request.setAttribute("refinment", "&classgroup="
+ URLEncoder.encode(request.getParameter("classgroup"),"UTF-8")); + URLEncoder.encode(request.getParameter("classgroup"),"UTF-8"));
}else{ }else{
if( alphaFilter != null && !"".equals(alphaFilter)){ if( alphaFilter != null && !"".equals(alphaFilter)){
request.setAttribute("alphas", getAlphas(topDocs, searcherForRequest)); request.setAttribute("alphas", getAlphas(topDocs, searcherForRequest));
alphaSortIndividuals(beans); alphaSortIndividuals(beans);
}else{ }else{
request.setAttribute("refinement", "&type=" request.setAttribute("refinment", "&type="
+ URLEncoder.encode(request.getParameter("type"),"UTF-8")); + URLEncoder.encode(request.getParameter("type"),"UTF-8"));
} }
} }
} }
beans = highlightBeans( beans ,
vreq.getWebappDaoFactory().getDataPropertyDao(),
vreq.getWebappDaoFactory().getObjectPropertyDao(),
new SimpleLuceneHighlighter(query,analyzer) );
//stick the results in the requestScope and prepare to forward to JSP //stick the results in the requestScope and prepare to forward to JSP
request.setAttribute("beans", beans); request.setAttribute("beans", beans);
@ -412,15 +385,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
} }
} }
return typesInHits; return typesInHits;
} }
private String getIndexDir(ServletContext servletContext) throws SearchException {
Object obj = servletContext.getAttribute(LuceneSetup.INDEX_DIR);
if( obj == null || !(obj instanceof String) )
throw new SearchException("Could not get IndexDir for luecene index");
else
return (String)obj;
}
private Analyzer getAnalyzer(ServletContext servletContext) throws SearchException { private Analyzer getAnalyzer(ServletContext servletContext) throws SearchException {
Object obj = servletContext.getAttribute(LuceneSetup.ANALYZER); Object obj = servletContext.getAttribute(LuceneSetup.ANALYZER);
@ -431,7 +396,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
} }
private Query getQuery(VitroRequest request, PortalFlag portalState, private Query getQuery(VitroRequest request, PortalFlag portalState,
Analyzer analyzer, String indexDir, String querystr ) throws SearchException, ParseException { Analyzer analyzer, String querystr ) throws SearchException, ParseException {
Query query = null; Query query = null;
try{ try{
//String querystr = request.getParameter(VitroQuery.QUERY_PARAMETER_NAME); //String querystr = request.getParameter(VitroQuery.QUERY_PARAMETER_NAME);
@ -445,14 +410,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
return null; return null;
} }
QueryParser parser = getQueryParser(analyzer); QueryParser parser = getQueryParser(analyzer);
// Prevent org.apache.lucene.queryParser.ParseException:
// Cannot parse 'mary *': '*' or '?' not allowed as first character in WildcardQuery
// The * is redundant in this case anyway, so just remove it.
log.debug("Query string is '" + querystr + "'");
querystr = querystr.replaceAll("([\\s^])[?*]", "$1");
log.debug("Cleaned query string is '" + querystr + "'");
query = parser.parse(querystr); query = parser.parse(querystr);
String alpha = request.getParameter("alpha"); String alpha = request.getParameter("alpha");
@ -576,113 +534,7 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
//we have no flags set, so no flag filtering //we have no flags set, so no flag filtering
return null; return null;
} }
} }
private synchronized IndexSearcher getIndexSearcher(String indexDir) {
if( searcher == null ){
try {
Directory fsDir = FSDirectory.getDirectory(indexDir);
searcher = new IndexSearcher(fsDir);
} catch (IOException e) {
log.error("LuceneSearcher: could not make indexSearcher "+e);
log.error("It is likely that you have not made a directory for the lucene index. "+
"Create the directory indicated in the error and set permissions/ownership so"+
" that the tomcat server can read/write to it.");
//The index directory is created by LuceneIndexer.makeNewIndex()
}
}
return searcher;
}
private List<Individual> highlightBeans(List<Individual> beans,
DataPropertyDao dpDao, ObjectPropertyDao opDao, VitroHighlighter highlighter) {
if( beans == null ){
log.debug("List of beans passed to highlightBeans() was null");
return Collections.EMPTY_LIST;
}else if( highlighter == null ){
log.debug("Null highlighter passed to highlightBeans()");
return beans;
}
Iterator<Individual> it = beans.iterator();
while(it.hasNext()){
Individual ent = it.next();
try{
dpDao.fillDataPropertiesForIndividual(ent);
opDao.fillObjectPropertiesForIndividual(ent);
fragmentHighlight(ent, highlighter);
}catch( Exception ex ){
log.debug("Error while doing search highlighting" , ex);
}
}
return beans;
}
/**
* Highlights the name and then replaces the description with
* highlighted fragments.
* @param ent
* @param highlighter
*/
public void fragmentHighlight(Individual ent, VitroHighlighter hl){
try{
if( ent == null ) return;
Html2Text h2t = new Html2Text();
StringBuffer sb = new StringBuffer("");
if(ent.getBlurb() != null)
sb.append(ent.getBlurb()).append(' ');
if(ent.getDescription() != null )
sb.append(ent.getDescription()).append(' ');
if(ent.getDataPropertyList() != null) {
Iterator edIt = ent.getDataPropertyList().iterator();
while (edIt.hasNext()) {
try{
DataProperty dp = (DataProperty)edIt.next();
if( getDataPropertyBlacklist().contains(dp.getURI()))
continue;
for(DataPropertyStatement dps : dp.getDataPropertyStatements()){
sb.append(dp.getPublicName()).append(' ')
.append(dps.getData()).append(' ');
}
}catch(Throwable e){
log.debug("Error highlighting data property statment " +
"for individual "+ent.getURI());
}
}
}
if(ent.getObjectPropertyList() != null) {
Iterator edIt = ent.getObjectPropertyList().iterator();
String t = null;
while (edIt.hasNext()) {
try {
ObjectProperty op = (ObjectProperty)edIt.next();
if( getObjectPropertyBlacklist().contains(op.getURI()))
continue;
for( ObjectPropertyStatement stmt : op.getObjectPropertyStatements()){
sb.append( ( (t = op.getDomainPublic()) != null) ? t : "" );
sb.append(' ');
sb.append( ( (t = stmt.getObject().getName()) != null) ? t : "" );
sb.append(' ');
}
} catch (Throwable e) {
log.debug("Error highlighting object property " +
"statement for individual "+ent.getURI());
}
}
}
String keywords = ent.getKeywordString();
if( keywords != null )
sb.append(keywords);
ent.setDescription(hl.getHighlightFragments( h2t.stripHtml( sb.toString() )));
}catch(Throwable th){
log.debug("could not hightlight for entity " + ent.getURI(),th);
}
}
private void doBadQuery(String queryStr, HttpServletRequest request, private void doBadQuery(String queryStr, HttpServletRequest request,
HttpServletResponse response) HttpServletResponse response)
@ -785,24 +637,4 @@ public class PagedSearchController extends VitroHttpServlet implements Searcher{
private final String defaultSearchField = "ALLTEXT"; private final String defaultSearchField = "ALLTEXT";
public static final int MAX_QUERY_LENGTH = 500; public static final int MAX_QUERY_LENGTH = 500;
/**
* Need to accept notification from indexer that the index has been changed.
*/
public void close() {
searcher = null;
}
public VitroHighlighter getHighlighter(VitroQuery q) {
throw new Error("PagedSearchController.getHighlighter() is unimplemented");
}
public VitroQueryFactory getQueryFactory() {
throw new Error("PagedSearchController.getQueryFactory() is unimplemented");
}
public List search(VitroQuery query) throws SearchException {
throw new Error("PagedSearchController.search() is unimplemented");
}
} }

9
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -200,6 +200,7 @@ public class IndexBuilder {
listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) ); listOfIterators.add( (new IndexBuilder.BuilderObjectSource(updatedInds)).getUpdatedSinceIterator(0) );
doBuild( listOfIterators, deletedInds, false, UPDATE_DOCS ); doBuild( listOfIterators, deletedInds, false, UPDATE_DOCS );
log.debug("Ending updateIndex()");
} }
/** /**
@ -342,15 +343,17 @@ public class IndexBuilder {
for( VClass vclass : ind.getVClasses() ){ for( VClass vclass : ind.getVClasses() ){
if( classesProhibitedFromSearch.isClassProhibited(vclass.getURI()) ){ if( classesProhibitedFromSearch.isClassProhibited(vclass.getURI()) ){
prohibitedClass = true; prohibitedClass = true;
log.debug("not indexing " + ind.getURI() + " because class " + vclass.getURI() + " is on prohibited list."); log.debug("removing " + ind.getURI() + " from index because class " + vclass.getURI() + " is on prohibited list.");
break; break;
} }
} }
} }
if( !prohibitedClass ) if( !prohibitedClass ){
log.debug("Indexing '" + ind.getName() + "' URI: " + ind.getURI());
indexer.index(ind, newDoc); indexer.index(ind, newDoc);
else }else{
indexer.removeFromIndex(ind); indexer.removeFromIndex(ind);
}
}catch(Throwable ex){ }catch(Throwable ex){
log.debug("IndexBuilder.indexItem() Error indexing " log.debug("IndexBuilder.indexItem() Error indexing "