From 97cfec876b865fec1d4455e1f928ca2fb8d3e279 Mon Sep 17 00:00:00 2001 From: ryounes Date: Thu, 12 May 2011 21:30:43 +0000 Subject: [PATCH] NIHVIVO-2458, NIHVIVO-2459 Add commented-out servlet definitions of new solr-based servlets for autocomplete and json data services. --- webapp/config/web.xml | 12 + .../webapp/controller/SolrJsonServlet.java | 564 ++++++++++++++++++ .../SolrAutocompleteController.java | 328 ++++++++++ 3 files changed, 904 insertions(+) create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java create mode 100644 webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java diff --git a/webapp/config/web.xml b/webapp/config/web.xml index e6cb0573a..a12a19079 100644 --- a/webapp/config/web.xml +++ b/webapp/config/web.xml @@ -927,6 +927,12 @@ AutocompleteController edu.cornell.mannlib.vitro.webapp.search.controller.AutocompleteController + AutocompleteController /autocomplete @@ -976,6 +982,12 @@ JSON Service edu.cornell.mannlib.vitro.webapp.controller.JSONServlet + JSON Service /dataservice diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java new file mode 100644 index 000000000..04f6f48e4 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/controller/SolrJsonServlet.java @@ -0,0 +1,564 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.controller; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.Writer; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.servlet.http.HttpSession; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.hp.hpl.jena.ontology.OntModel; +import com.hp.hpl.jena.rdf.model.Literal; + +import edu.cornell.mannlib.vitro.webapp.beans.DataProperty; +import edu.cornell.mannlib.vitro.webapp.beans.Individual; +import edu.cornell.mannlib.vitro.webapp.beans.VClass; +import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListController.PageRecord; +import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder; +import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary; +import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary; +import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; +import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache; +import edu.cornell.mannlib.vitro.webapp.edit.n3editing.EditConfiguration; +import edu.cornell.mannlib.vitro.webapp.edit.n3editing.SelectListGenerator; +import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch; +import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individual.IndividualTemplateModel; + +/** + * This servlet is for servicing requests for JSON objects/data. + * It could be generalized to get other types of data ex. XML, HTML etc + * @author bdc34 + * + */ +public class SolrJsonServlet extends VitroHttpServlet { + + @Override + protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + super.doPost(req, resp); + } + + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + super.doGet(req, resp); + VitroRequest vreq = new VitroRequest(req); + + try{ + if(vreq.getParameter("getEntitiesByVClass") != null ){ + if( vreq.getParameter("resultKey") == null) { + getEntitiesByVClass(req, resp); + return; + } else { + getEntitiesByVClassContinuation( req, resp); + return; + } + }else if( vreq.getParameter("getN3EditOptionList") != null ){ + doN3EditOptionList(req,resp); + return; + }else if( vreq.getParameter("getLuceneIndividualsByVClass") != null ){ + getLuceneIndividualsByVClass(req,resp); + return; + }else if( vreq.getParameter("getVClassesForVClassGroup") != null ){ + getVClassesForVClassGroup(req,resp); + return; + } + }catch(Exception ex){ + log.warn(ex,ex); + } + } + + private void getVClassesForVClassGroup(HttpServletRequest req, HttpServletResponse resp) throws IOException, JSONException { + JSONObject map = new JSONObject(); + VitroRequest vreq = new VitroRequest(req); + String vcgUri = vreq.getParameter("classgroupUri"); + if( vcgUri == null ){ + log.debug("no URI passed for classgroupUri"); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + VClassGroupCache vcgc = VClassGroupCache.getVClassGroupCache(getServletContext()); + VClassGroup vcg = vcgc.getGroup(vcgUri); + if( vcg == null ){ + log.debug("Could not find vclassgroup: " + vcgUri); + resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + return; + } + + ArrayList classes = new ArrayList(vcg.size()); + for( VClass vc : vcg){ + JSONObject vcObj = new JSONObject(); + vcObj.put("name", vc.getName()); + vcObj.put("URI", vc.getURI()); + vcObj.put("entityCount", vc.getEntityCount()); + classes.add(vcObj); + } + map.put("classes", classes); + map.put("classGroupName", vcg.getPublicName()); + map.put("classGroupUri", vcg.getURI()); + + resp.setCharacterEncoding("UTF-8"); + resp.setContentType("application/json;charset=UTF-8"); + Writer writer = resp.getWriter(); + writer.write(map.toString()); + } + + private void getLuceneIndividualsByVClass( HttpServletRequest req, HttpServletResponse resp ){ + String errorMessage = null; + JSONObject rObj = null; + try{ + VitroRequest vreq = new VitroRequest(req); + VClass vclass=null; + + + String vitroClassIdStr = vreq.getParameter("vclassId"); + if ( vitroClassIdStr != null && !vitroClassIdStr.isEmpty()){ + vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(vitroClassIdStr); + if (vclass == null) { + log.debug("Couldn't retrieve vclass "); + throw new Exception (errorMessage = "Class " + vitroClassIdStr + " not found"); + } + }else{ + log.debug("parameter vclassId URI parameter expected "); + throw new Exception("parameter vclassId URI parameter expected "); + } + rObj = getLuceneIndividualsByVClass(vclass.getURI(),req, getServletContext()); + }catch(Exception ex){ + errorMessage = ex.toString(); + log.error(ex,ex); + } + + if( rObj == null ) + rObj = new JSONObject(); + + try{ + resp.setCharacterEncoding("UTF-8"); + resp.setContentType("application/json;charset=UTF-8"); + + if( errorMessage != null ){ + rObj.put("errorMessage", errorMessage); + resp.setStatus(500 /*HttpURLConnection.HTTP_SERVER_ERROR*/); + }else{ + rObj.put("errorMessage", ""); + } + Writer writer = resp.getWriter(); + writer.write(rObj.toString()); + }catch(JSONException jse){ + log.error(jse,jse); + } catch (IOException e) { + log.error(e,e); + } + + } + + public static JSONObject getLuceneIndividualsByVClass(String vclassURI, HttpServletRequest req, ServletContext context) throws Exception { + + VitroRequest vreq = new VitroRequest(req); + VClass vclass=null; + JSONObject rObj = new JSONObject(); + + DataProperty fNameDp = (new DataProperty()); + fNameDp.setURI("http://xmlns.com/foaf/0.1/firstName"); + DataProperty lNameDp = (new DataProperty()); + lNameDp.setURI("http://xmlns.com/foaf/0.1/lastName"); + DataProperty monikerDp = (new DataProperty()); + monikerDp.setURI( VitroVocabulary.MONIKER); + //this property is vivo specific + DataProperty preferredTitleDp = (new DataProperty()); + preferredTitleDp.setURI("http://vivoweb.org/ontology/core#preferredTitle"); + + + if( log.isDebugEnabled() ){ + Enumeration e = vreq.getParameterNames(); + while(e.hasMoreElements()){ + String name = (String)e.nextElement(); + log.debug("parameter: " + name); + for( String value : vreq.getParameterValues(name) ){ + log.debug("value for " + name + ": '" + value + "'"); + } + } + } + + //need an unfiltered dao to get firstnames and lastnames + WebappDaoFactory fullWdf = vreq.getFullWebappDaoFactory(); + + + String vitroClassIdStr = vreq.getParameter("vclassId"); + if ( vitroClassIdStr != null && !vitroClassIdStr.isEmpty()){ + vclass = vreq.getWebappDaoFactory().getVClassDao().getVClassByURI(vitroClassIdStr); + if (vclass == null) { + log.debug("Couldn't retrieve vclass "); + throw new Exception ("Class " + vitroClassIdStr + " not found"); + } + }else{ + log.debug("parameter vclassId URI parameter expected "); + throw new Exception("parameter vclassId URI parameter expected "); + } + + rObj.put("vclass", + new JSONObject().put("URI",vclass.getURI()) + .put("name",vclass.getName())); + + if (vclass != null) { + String alpha = IndividualListController.getAlphaParameter(vreq); + int page = IndividualListController.getPageParameter(vreq); + Map map = IndividualListController.getResultsForVClass( + vclass.getURI(), + page, + alpha, + vreq.getWebappDaoFactory().getIndividualDao(), + context); + + rObj.put("totalCount", map.get("totalCount")); + rObj.put("alpha", map.get("alpha")); + + List inds = (List)map.get("entities"); + List indsTm = new ArrayList(); + JSONArray jInds = new JSONArray(); + for(Individual ind : inds ){ + JSONObject jo = new JSONObject(); + jo.put("URI", ind.getURI()); + jo.put("label",ind.getRdfsLabel()); + jo.put("name",ind.getName()); + jo.put("thumbUrl", ind.getThumbUrl()); + jo.put("imageUrl", ind.getImageUrl()); + jo.put("profileUrl", UrlBuilder.getIndividualProfileUrl(ind, vreq.getWebappDaoFactory())); + + String moniker = getDataPropertyValue(ind, monikerDp, fullWdf); + jo.put("moniker", moniker); + jo.put("vclassName", getVClassName(ind,moniker,fullWdf)); + + jo.put("preferredTitle", getDataPropertyValue(ind, preferredTitleDp, fullWdf)); + jo.put("firstName", getDataPropertyValue(ind, fNameDp, fullWdf)); + jo.put("lastName", getDataPropertyValue(ind, lNameDp, fullWdf)); + + jInds.put(jo); + } + rObj.put("individuals", jInds); + + JSONArray wpages = new JSONArray(); + List pages = (List)map.get("pages"); + for( PageRecord pr: pages ){ + JSONObject p = new JSONObject(); + p.put("text", pr.text); + p.put("param", pr.param); + p.put("index", pr.index); + wpages.put( p ); + } + rObj.put("pages",wpages); + + JSONArray jletters = new JSONArray(); + List letters = Controllers.getLetters(); + for( String s : letters){ + JSONObject jo = new JSONObject(); + jo.put("text", s); + jo.put("param", "alpha=" + URLEncoder.encode(s, "UTF-8")); + jletters.put( jo ); + } + rObj.put("letters", jletters); + } + + return rObj; + } + + + private static String getVClassName(Individual ind, String moniker, + WebappDaoFactory fullWdf) { + /* so the moniker frequently has a vclass name in it. Try to return + * the vclass name that is the same as the moniker so that the templates + * can detect this. */ + if( (moniker == null || moniker.isEmpty()) ){ + if( ind.getVClass() != null && ind.getVClass().getName() != null ) + return ind.getVClass().getName(); + else + return ""; + } + + List vcList = ind.getVClasses(); + for( VClass vc : vcList){ + if( vc != null && moniker.equals( vc.getName() )) + return moniker; + } + + // if we get here, then we didn't find a moniker that matched a vclass, + // so just return any vclass.name + if( ind.getVClass() != null && ind.getVClass().getName() != null ) + return ind.getVClass().getName(); + else + return ""; + } + + static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){ + List values = wdf.getDataPropertyStatementDao() + .getDataPropertyValuesForIndividualByProperty(ind, dp); + if( values == null || values.isEmpty() ) + return ""; + else{ + if( values.get(0) != null ) + return values.get(0).getLexicalForm(); + else + return ""; + } + + } + + /** + * Gets an option list for a given EditConfiguration and Field. + * Requires following HTTP query parameters: + * editKey + * field + */ + private void doN3EditOptionList(HttpServletRequest req, HttpServletResponse resp) throws IOException { + log.debug("in doN3EditOptionList()"); + String field = req.getParameter("field"); + if( field == null ){ + log.debug("could not find query parameter 'field' for doN3EditOptionList"); + throw new IllegalArgumentException(" getN3EditOptionList requires parameter 'field'"); + } + + HttpSession sess = req.getSession(false); + EditConfiguration editConfig = EditConfiguration.getConfigFromSession(sess, req); + if( editConfig == null ) { + log.debug("could not find query parameter 'editKey' for doN3EditOptionList"); + throw new IllegalArgumentException(" getN3EditOptionList requires parameter 'editKey'"); + } + + if( log.isDebugEnabled() ) + log.debug(" attempting to get option list for field '" + field + "'"); + + // set ProhibitedFromSearch object so picklist doesn't show + // individuals from classes that should be hidden from list views + OntModel displayOntModel = + (OntModel) getServletConfig().getServletContext() + .getAttribute("displayOntModel"); + if (displayOntModel != null) { + ProhibitedFromSearch pfs = new ProhibitedFromSearch( + DisplayVocabulary.PRIMARY_LUCENE_INDEX_URI, displayOntModel); + editConfig.setProhibitedFromSearch(pfs); + } + + Map options = SelectListGenerator.getOptions(editConfig, field, (new VitroRequest(req)).getFullWebappDaoFactory()); + resp.setContentType("application/json"); + ServletOutputStream out = resp.getOutputStream(); + + out.println("["); + for(String key : options.keySet()){ + JSONArray jsonObj = new JSONArray(); + jsonObj.put( options.get(key)); + jsonObj.put( key); + out.println(" " + jsonObj.toString() + ","); + } + out.println("]"); + } + + private void getEntitiesByVClassContinuation(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + log.debug("in getEntitiesByVClassContinuation()"); + VitroRequest vreq = new VitroRequest(req); + String resKey = vreq.getParameter("resultKey"); + if( resKey == null ) + throw new ServletException("Could not get resultKey"); + HttpSession session = vreq.getSession(); + if( session == null ) + throw new ServletException("there is no session to get the pervious results from"); + List entsInVClass = (List) session.getAttribute(resKey); + if( entsInVClass == null ) + throw new ServletException("Could not find List for resultKey " + resKey); + + List entsToReturn = new ArrayList(REPLY_SIZE); + boolean more = false; + int count = 0; + int size = REPLY_SIZE; + /* we have a large number of items to send back so we need to stash the list in the session scope */ + if( entsInVClass.size() > REPLY_SIZE){ + more = true; + ListIterator entsFromVclass = entsInVClass.listIterator(); + while ( entsFromVclass.hasNext() && count <= REPLY_SIZE ){ + entsToReturn.add( entsFromVclass.next()); + entsFromVclass.remove(); + count++; + } + if( log.isDebugEnabled() ) log.debug("getEntitiesByVClassContinuation(): Creating reply with continue token," + + " sending in this reply: " + count +", remaing to send: " + entsInVClass.size() ); + } else { + //send out reply with no continuation + entsToReturn = entsInVClass; + count = entsToReturn.size(); + session.removeAttribute(resKey); + if( log.isDebugEnabled()) log.debug("getEntitiesByVClassContinuation(): sending " + count + " Ind without continue token"); + } + + //put all the entities on the JSON array + JSONArray ja = individualsToJson( entsToReturn ); + + //put the responseGroup number on the end of the JSON array + if( more ){ + try{ + JSONObject obj = new JSONObject(); + obj.put("resultGroup", "true"); + obj.put("size", count); + + StringBuffer nextUrlStr = req.getRequestURL(); + nextUrlStr.append("?") + .append("getEntitiesByVClass").append( "=1&" ) + .append("resultKey=").append( resKey ); + obj.put("nextUrl", nextUrlStr.toString()); + + ja.put(obj); + }catch(JSONException je ){ + throw new ServletException(je.getMessage()); + } + } + resp.setContentType("application/json"); + ServletOutputStream out = resp.getOutputStream(); + out.print( ja.toString() ); + log.debug("done with getEntitiesByVClassContinuation()"); + } + + + + /** + * Gets a list of entities that are members of the indicated vClass. + * + * If the list is large then we will pass some token indicating that there is more + * to come. The results are sent back in 250 entity blocks. To get all of the + * entities for a VClass just keep requesting lists until there are not more + * continue tokens. + * + * If there are more entities the last item on the returned array will be an object + * with no id property. It will look like this: + * + * {"resultGroup":0, + * "resultKey":"2WEK2306", + * "nextUrl":"http://caruso.mannlib.cornell.edu:8080/vitro/dataservice?getEntitiesByVClass=1&resultKey=2WEK2306&resultGroup=1&vclassId=null", + * "entsInVClass":1752, + * "nextResultGroup":1, + * "standardReplySize":256} + * + */ + private void getEntitiesByVClass(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException{ + log.debug("in getEntitiesByVClass()"); + VitroRequest vreq = new VitroRequest(req); + String vclassURI = vreq.getParameter("vclassURI"); + WebappDaoFactory daos = (new VitroRequest(req)).getFullWebappDaoFactory(); + resp.setCharacterEncoding("UTF-8"); + + // ServletOutputStream doesn't support UTF-8 + PrintWriter out = resp.getWriter(); + resp.getWriter(); + + if( vclassURI == null ){ + log.debug("getEntitiesByVClass(): no value for 'vclassURI' found in the HTTP request"); + out.print( (new JSONArray()).toString() ); return; + } + + VClass vclass = daos.getVClassDao().getVClassByURI( vclassURI ); + if( vclass == null ){ + log.debug("getEntitiesByVClass(): could not find vclass for uri '"+ vclassURI + "'"); + out.print( (new JSONArray()).toString() ); return; + } + + List entsInVClass = daos.getIndividualDao().getIndividualsByVClass( vclass ); + if( entsInVClass == null ){ + log.debug("getEntitiesByVClass(): null List retruned by getIndividualsByVClass() for "+vclassURI); + out.print( (new JSONArray().toString() )); return ; + } + int numberOfEntsInVClass = entsInVClass.size(); + + List entsToReturn = new ArrayList( REPLY_SIZE ); + String requestHash = null; + int count = 0; + boolean more = false; + /* we have a large number of items to send back so we need to stash the list in the session scope */ + if( entsInVClass.size() > REPLY_SIZE){ + more = true; + HttpSession session = vreq.getSession(true); + requestHash = Integer.toString((vclassURI + System.currentTimeMillis()).hashCode()); + session.setAttribute(requestHash, entsInVClass ); + + ListIterator entsFromVclass = entsInVClass.listIterator(); + while ( entsFromVclass.hasNext() && count < REPLY_SIZE ){ + entsToReturn.add( entsFromVclass.next()); + entsFromVclass.remove(); + count++; + } + if( log.isDebugEnabled() ){ log.debug("getEntitiesByVClass(): Creating reply with continue token, found " + numberOfEntsInVClass + " Individuals"); } + }else{ + if( log.isDebugEnabled() ) log.debug("getEntitiesByVClass(): sending " + numberOfEntsInVClass +" Individuals without continue token"); + entsToReturn = entsInVClass; + count = entsToReturn.size(); + } + + + //put all the entities on the JSON array + JSONArray ja = individualsToJson( entsToReturn ); + + //put the responseGroup number on the end of the JSON array + if( more ){ + try{ + JSONObject obj = new JSONObject(); + obj.put("resultGroup", "true"); + obj.put("size", count); + obj.put("total", numberOfEntsInVClass); + + StringBuffer nextUrlStr = req.getRequestURL(); + nextUrlStr.append("?") + .append("getEntitiesByVClass").append( "=1&" ) + .append("resultKey=").append( requestHash ); + obj.put("nextUrl", nextUrlStr.toString()); + + ja.put(obj); + }catch(JSONException je ){ + throw new ServletException("unable to create continuation as JSON: " + je.getMessage()); + } + } + + resp.setContentType("application/json"); + out.print( ja.toString() ); + + log.debug("done with getEntitiesByVClass()"); + + } + + private JSONArray individualsToJson(List individuals) throws ServletException { + JSONArray ja = new JSONArray(); + Iterator it = individuals.iterator(); + try{ + while(it.hasNext()){ + Individual ent = (Individual) it.next(); + JSONObject entJ = new JSONObject(); + entJ.put("name", ent.getName()); + entJ.put("URI", ent.getURI()); + ja.put( entJ ); + } + }catch(JSONException ex){ + throw new ServletException("could not convert list of Individuals into JSON: " + ex); + } + + return ja; + } + + private static final int REPLY_SIZE = 256; + + private static final Log log = LogFactory.getLog(JSONServlet.class.getName()); +} diff --git a/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java new file mode 100644 index 000000000..146f2ab29 --- /dev/null +++ b/webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SolrAutocompleteController.java @@ -0,0 +1,328 @@ +/* $This file is distributed under the terms of the license in /doc/license.txt$ */ + +package edu.cornell.mannlib.vitro.webapp.search.controller; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.WildcardQuery; +import org.apache.lucene.util.Version; +import org.json.JSONArray; + +import com.hp.hpl.jena.sparql.lib.org.json.JSONObject; + +import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions; +import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.usepages.UseBasicAjaxControllers; +import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; +import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController; +import edu.cornell.mannlib.vitro.webapp.search.SearchException; +import edu.cornell.mannlib.vitro.webapp.search.lucene.Entity2LuceneDoc.VitroLuceneTermNames; +import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneIndexFactory; +import edu.cornell.mannlib.vitro.webapp.search.lucene.LuceneSetup; + +/** + * AutocompleteController generates autocomplete content + * through a Lucene search. + */ +public class SolrAutocompleteController extends VitroAjaxController { + + private static final long serialVersionUID = 1L; + private static final Log log = LogFactory.getLog(SolrAutocompleteController.class); + + //private static final String TEMPLATE_DEFAULT = "autocompleteResults.ftl"; + + private static String QUERY_PARAMETER_NAME = "term"; + + String NORESULT_MSG = ""; + private int defaultMaxSearchSize= 1000; + + @Override + protected Actions requiredActions(VitroRequest vreq) { + return new Actions(new UseBasicAjaxControllers()); + } + + @Override + protected void doRequest(VitroRequest vreq, HttpServletResponse response) + throws IOException, ServletException { + + try { + + int maxHitSize = defaultMaxSearchSize; + + String qtxt = vreq.getParameter(QUERY_PARAMETER_NAME); + Analyzer analyzer = getAnalyzer(getServletContext()); + + Query query = getQuery(vreq, analyzer, qtxt); + if (query == null ) { + log.debug("query for '" + qtxt +"' is null."); + doNoQuery(response); + return; + } + log.debug("query for '" + qtxt +"' is " + query.toString()); + + IndexSearcher searcherForRequest = LuceneIndexFactory.getIndexSearcher(getServletContext()); + + TopDocs topDocs = null; + try{ + topDocs = searcherForRequest.search(query,null,maxHitSize); + }catch(Throwable t){ + log.error("in first pass at search: " + t); + // this is a hack to deal with odd cases where search and index threads interact + try{ + wait(150); + topDocs = searcherForRequest.search(query,null,maxHitSize); + }catch (Exception e){ + log.error(e, e); + doNoSearchResults(response); + return; + } + } + + if( topDocs == null || topDocs.scoreDocs == null){ + log.error("topDocs for a search was null"); + doNoSearchResults(response); + return; + } + + int hitsLength = topDocs.scoreDocs.length; + if ( hitsLength < 1 ){ + doNoSearchResults(response); + return; + } + log.debug("found "+hitsLength+" hits"); + + List results = new ArrayList(); + for(int i=0; i MAX_QUERY_LENGTH ){ + log.debug("The search was too long. The maximum " + + "query length is " + MAX_QUERY_LENGTH ); + return null; + } + + query = makeNameQuery(querystr, analyzer, vreq); + + // Filter by type + { + BooleanQuery boolQuery = new BooleanQuery(); + String typeParam = (String) vreq.getParameter("type"); + boolQuery.add( new TermQuery( + new Term(VitroLuceneTermNames.RDFTYPE, + typeParam)), + BooleanClause.Occur.MUST); + boolQuery.add(query, BooleanClause.Occur.MUST); + query = boolQuery; + } + + } catch (Exception ex){ + throw new SearchException(ex.getMessage()); + } + + return query; + } + + private Query makeNameQuery(String querystr, Analyzer analyzer, HttpServletRequest request) { + + String tokenizeParam = (String) request.getParameter("tokenize"); + boolean tokenize = "true".equals(tokenizeParam); + + // Note: Stemming is only relevant if we are tokenizing: an untokenized name + // query will not be stemmed. So we don't look at the stem parameter until we get to + // makeTokenizedNameQuery(). + if (tokenize) { + return makeTokenizedNameQuery(querystr, analyzer, request); + } else { + return makeUntokenizedNameQuery(querystr); + } + } + + private Query makeTokenizedNameQuery(String querystr, Analyzer analyzer, HttpServletRequest request) { + + String stemParam = (String) request.getParameter("stem"); + boolean stem = "true".equals(stemParam); + String termName = stem ? VitroLuceneTermNames.NAME : VitroLuceneTermNames.NAMEUNSTEMMED; + + BooleanQuery boolQuery = new BooleanQuery(); + + // Use the query parser to analyze the search term the same way the indexed text was analyzed. + // For example, text is lowercased, and function words are stripped out. + QueryParser parser = getQueryParser(termName, analyzer); + + // The wildcard query doesn't play well with stemming. Query term name:tales* doesn't match + // "tales", which is indexed as "tale", while query term name:tales does. Obviously we need + // the wildcard for name:tal*, so the only way to get them all to match is use a disjunction + // of wildcard and non-wildcard queries. The query will look have only an implicit disjunction + // operator: e.g., +(name:tales name:tales*) + try { + log.debug("Adding non-wildcard query for " + querystr); + Query query = parser.parse(querystr); + boolQuery.add(query, BooleanClause.Occur.SHOULD); + + // Prevent ParseException here when adding * after a space. + // If there's a space at the end, we don't need the wildcard query. + if (! querystr.endsWith(" ")) { + log.debug("Adding wildcard query for " + querystr); + Query wildcardQuery = parser.parse(querystr + "*"); + boolQuery.add(wildcardQuery, BooleanClause.Occur.SHOULD); + } + + log.debug("Name query is: " + boolQuery.toString()); + } catch (ParseException e) { + log.warn(e, e); + } + + return boolQuery; + } + + private Query makeUntokenizedNameQuery(String querystr) { + + querystr = querystr.toLowerCase(); + String termName = VitroLuceneTermNames.NAMELOWERCASE; + BooleanQuery query = new BooleanQuery(); + log.debug("Adding wildcard query on unanalyzed name"); + query.add( + new WildcardQuery(new Term(termName, querystr + "*")), + BooleanClause.Occur.MUST); + + return query; + } + + private QueryParser getQueryParser(String searchField, Analyzer analyzer){ + // searchField indicates which field to search against when there is no term + // indicated in the query string. + // The analyzer is needed so that we use the same analyzer on the search queries as + // was used on the text that was indexed. + QueryParser qp = new QueryParser(Version.LUCENE_29, searchField,analyzer); + //this sets the query parser to AND all of the query terms it finds. + qp.setDefaultOperator(QueryParser.AND_OPERATOR); + return qp; + } + + private void doNoQuery(HttpServletResponse response) throws IOException { + // For now, we are not sending an error message back to the client because with the default autocomplete configuration it + // chokes. + doNoSearchResults(response); + } + + private void doSearchError(HttpServletResponse response) throws IOException { + // For now, we are not sending an error message back to the client because with the default autocomplete configuration it + // chokes. + doNoSearchResults(response); + } + + private void doNoSearchResults(HttpServletResponse response) throws IOException { + response.getWriter().write("[]"); + } + + public static final int MAX_QUERY_LENGTH = 500; + + public class SearchResult implements Comparable { + private String label; + private String uri; + + SearchResult(String label, String uri) { + this.label = label; + this.uri = uri; + } + + public String getLabel() { + return label; + } + + public String getJsonLabel() { + return JSONObject.quote(label); + } + + public String getUri() { + return uri; + } + + public String getJsonUri() { + return JSONObject.quote(uri); + } + + Map toMap() { + Map map = new HashMap(); + map.put("label", label); + map.put("uri", uri); + return map; + } + + public int compareTo(Object o) throws ClassCastException { + if ( !(o instanceof SearchResult) ) { + throw new ClassCastException("Error in SearchResult.compareTo(): expected SearchResult object."); + } + SearchResult sr = (SearchResult) o; + return label.compareToIgnoreCase(sr.getLabel()); + } + } + +}