litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-742 Change client code to use SearchEngine interface.

Browse source 
Removed SolrQueryTest until I can see how to update it. Everything compiles and tests run, but haven't tried running VIVO yet.
This commit is contained in:
Jim Blake 2014-04-22 13:49:27 -04:00
parent 823848123f
commit 90886c564e
38 changed files with 398 additions and 1109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/IndividualListRdfController.java
View file

@ -8,11 +8,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -21,8 +16,13 @@ import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
public class IndividualListRdfController extends VitroHttpServlet {
@ -31,21 +31,22 @@ public class IndividualListRdfController extends VitroHttpServlet {
public static final int ENTITY_LIST_CONTROLLER_MAX_RESULTS = 30000;
public void doGet (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
@Override
public void doGet (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
// Make the query
String vclassUri = req.getParameter("vclass");
String queryStr = VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\"";
SolrQuery query = new SolrQuery(queryStr);
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryStr);
query.setStart(0)
.setRows(ENTITY_LIST_CONTROLLER_MAX_RESULTS)
.setFields(VitroSearchTermNames.URI);
.addFields(VitroSearchTermNames.URI);
// For now, we're only displaying the url, so no need to sort.
//.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED);
//.addSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED);
// Execute the query
SolrServer solr = SolrSetup.getSolrServer(getServletContext());
QueryResponse response = null;
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchResponse response = null;
try {
response = solr.query(query);
@ -57,17 +58,17 @@ public class IndividualListRdfController extends VitroHttpServlet {
throw new ServletException("Could not run search in IndividualListRdfController");
}
SolrDocumentList docs = response.getResults();
SearchResultDocumentList docs = response.getResults();
if (docs == null) {
throw new ServletException("Could not run search in IndividualListRdfController");
}
Model model = ModelFactory.createDefaultModel();
for (SolrDocument doc : docs) {
String uri = doc.get(VitroSearchTermNames.URI).toString();
for (SearchResultDocument doc : docs) {
String uri = doc.getStringValue(VitroSearchTermNames.URI);
Resource resource = ResourceFactory.createResource(uri);
RDFNode node = (RDFNode) ResourceFactory.createResource(vclassUri);
RDFNode node = ResourceFactory.createResource(vclassUri);
model.add(resource, RDF.type, node);
}
@ -75,7 +76,8 @@ public class IndividualListRdfController extends VitroHttpServlet {
model.write(res.getOutputStream(), "RDF/XML");
}
public void doPost (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException{
@Override
public void doPost (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException{
doGet(req,res);
}

36
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/accounts/admin/ajax/ProfileAutoCompleter.java
View file

@ -16,17 +16,11 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.json.JSONException;
import com.hp.hpl.jena.ontology.OntModel;
@ -39,11 +33,16 @@ import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Literal;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.SelfEditingConfiguration;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
@ -121,8 +120,8 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
}
try {
SolrQuery query = buildSolrQuery();
QueryResponse queryResponse = executeSolrQuery(query);
SearchQuery query = buildSearchQuery();
SearchResponse queryResponse = executeSearchQuery(query);
List<Map<String, String>> maps = SolrQueryUtils
.parseAndFilterResponse(queryResponse, RESPONSE_FIELDS,
@ -133,17 +132,17 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
String response = assembleJsonResponse(maps);
log.debug(response);
return response;
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
log.error("Failed to get basic profile info", e);
return EMPTY_RESPONSE;
}
}
private SolrQuery buildSolrQuery() {
SolrQuery q = new SolrQuery();
q.setFields(NAME_RAW, URI);
q.setSortField(NAME_LOWERCASE_SINGLE_VALUED, ORDER.asc);
q.setFilterQueries(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE,
private SearchQuery buildSearchQuery() {
SearchQuery q = ApplicationUtils.instance().getSearchEngine().createQuery();
q.addFields(NAME_RAW, URI);
q.addSortField(NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
q.addFilterQuery(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE,
profileTypes, OR));
q.setStart(0);
q.setRows(10000);
@ -151,10 +150,9 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
return q;
}
private QueryResponse executeSolrQuery(SolrQuery query)
throws SolrServerException {
ServletContext ctx = servlet.getServletContext();
SolrServer solr = SolrSetup.getSolrServer(ctx);
private SearchResponse executeSearchQuery(SearchQuery query)
throws SearchEngineException {
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
return solr.query(query);
}

34
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/accounts/manageproxies/ajax/BasicProfilesGetter.java
View file

@ -1,7 +1,6 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.controller.accounts.manageproxies.ajax;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.AC_NAME_STEMMED;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_RAW;
@ -14,23 +13,22 @@ import java.io.IOException;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.json.JSONException;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
@ -88,10 +86,9 @@ public class BasicProfilesGetter extends AbstractAjaxResponder {
}
try {
ServletContext ctx = servlet.getServletContext();
SolrServer solr = SolrSetup.getSolrServer(ctx);
SolrQuery query = buildSolrQuery();
QueryResponse queryResponse = solr.query(query);
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchQuery query = buildSearchQuery();
SearchResponse queryResponse = solr.query(query);
List<Map<String, String>> parsed = SolrQueryUtils
.parseResponse(queryResponse, RESPONSE_FIELDS);
@ -99,18 +96,17 @@ public class BasicProfilesGetter extends AbstractAjaxResponder {
String response = assembleJsonResponse(parsed);
log.debug(response);
return response;
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
log.error("Failed to get basic profile info", e);
return EMPTY_RESPONSE;
}
}
private SolrQuery buildSolrQuery() {
SolrQuery q = new SolrQuery();
q.setFields(NAME_RAW, URI);
q.setSortField(NAME_LOWERCASE_SINGLE_VALUED, ORDER.asc);
q.setFilterQueries(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE,
profileTypes, OR));
private SearchQuery buildSearchQuery() {
SearchQuery q = ApplicationUtils.instance().getSearchEngine().createQuery();
q.addFields(NAME_RAW, URI);
q.addSortField(NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
q.addFilterQuery(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE, profileTypes, OR));
q.setStart(0);
q.setRows(30);
q.setQuery(searchWords.assembleQuery(NAME_UNSTEMMED, AC_NAME_STEMMED));

25
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java
View file

@ -8,12 +8,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrServerException;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -24,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Res
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.individuallist.IndividualListResults;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual;
@ -94,8 +92,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
vclass.getURI(),
page,
alpha,
vreq.getWebappDaoFactory().getIndividualDao(),
getServletContext());
vreq.getWebappDaoFactory().getIndividualDao());
body.putAll(vcResults.asFreemarkerMap());
List<Individual> inds = vcResults.getEntities();
@ -149,13 +146,13 @@ public class IndividualListController extends FreemarkerHttpServlet {
return SolrQueryUtils.getPageParameter(request);
}
public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context)
public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao)
throws SearchException{
try{
List<String> classUris = Collections.singletonList(vclassURI);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, context, indDao);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, indDao);
return getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha);
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
String msg = "An error occurred retrieving results for vclass query";
log.error(msg, e);
// Throw this up to processRequest, so the template gets the error message.
@ -166,9 +163,9 @@ public class IndividualListController extends FreemarkerHttpServlet {
}
}
public static IndividualListResults getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao, ServletContext context) {
public static IndividualListResults getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao) {
try{
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, context, indDao);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, indDao);
return getResultsForVClassQuery(results, page, pageSize, alpha);
} catch(Throwable th) {
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
@ -176,10 +173,10 @@ public class IndividualListController extends FreemarkerHttpServlet {
}
}
public static IndividualListResults getRandomResultsForVClass(String vclassURI, int page, int pageSize, IndividualDao indDao, ServletContext context) {
public static IndividualListResults getRandomResultsForVClass(String vclassURI, int page, int pageSize, IndividualDao indDao) {
try{
List<String> classUris = Collections.singletonList(vclassURI);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteRandomVClassQuery(classUris, page, pageSize, context, indDao);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteRandomVClassQuery(classUris, page, pageSize, indDao);
return getResultsForVClassQuery(results, page, pageSize, "");
} catch(Throwable th) {
log.error("An error occurred retrieving random results for vclass query", th);
@ -189,8 +186,8 @@ public class IndividualListController extends FreemarkerHttpServlet {
//TODO: Get rid of this method and utilize SolrQueryUtils - currently appears to be referenced
//only within DataGetterUtils
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
return SolrQueryUtils.getIndividualCount(vclassUris, indDao, context);
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao) {
return SolrQueryUtils.getIndividualCount(vclassUris, indDao);
}
private static IndividualListResults getResultsForVClassQuery(IndividualListQueryResults results, int page, int pageSize, String alpha) {

32
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListQueryResults.java
View file

@ -5,21 +5,19 @@ package edu.cornell.mannlib.vitro.webapp.controller.freemarker;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/**
* Holds the Individuals that were found in a Solr search query.
@ -37,12 +35,12 @@ public class IndividualListQueryResults {
// Convenience method
// ----------------------------------------------------------------------
public static IndividualListQueryResults runQuery(SolrQuery query,
IndividualDao indDao, ServletContext context)
throws SolrServerException {
public static IndividualListQueryResults runQuery(SearchQuery query,
IndividualDao indDao)
throws SearchEngineException {
SolrServer solr = SolrSetup.getSolrServer(context);
QueryResponse response = null;
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchResponse response = null;
response = solr.query(query);
if (response == null) {
@ -50,7 +48,7 @@ public class IndividualListQueryResults {
return EMPTY_RESULT;
}
SolrDocumentList docs = response.getResults();
SearchResultDocumentList docs = response.getResults();
if (docs == null) {
log.debug("results from search query response was null");
return EMPTY_RESULT;
@ -61,8 +59,8 @@ public class IndividualListQueryResults {
log.debug("Number of search results: " + hitCount);
List<Individual> individuals = new ArrayList<Individual>(docs.size());
for (SolrDocument doc : docs) {
String uri = doc.get(VitroSearchTermNames.URI).toString();
for (SearchResultDocument doc : docs) {
String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual individual = indDao.getIndividualByURI(uri);
if (individual == null) {
log.debug("No individual for search document with uri = " + uri);

67
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/grefine/JSONReconcileServlet.java
View file

@ -18,23 +18,21 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.controller.VitroHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
/**
* This servlet is for servicing JSON requests from Google Refine's
@ -125,7 +123,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
try {
for (int i = 0; i < queries.size(); i++) {
String queryStr = (String) queries.get(i);
String queryStr = queries.get(i);
JSONObject json = new JSONObject(queryStr);
if (json.has("query")) { // single query
@ -138,7 +136,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
} else { // multiple queries
for (Iterator<String> iter = json.keys(); iter.hasNext();) {
ArrayList<JSONObject> jsonList = new ArrayList<JSONObject>();
String key = (String) iter.next();
String key = iter.next();
Object obj = json.get(key);
JSONObject jsonLvl2 = (JSONObject) obj;
if (jsonLvl2.has("query")) {
@ -234,7 +232,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
for (Map.Entry<String, JSONObject> entry : currMap.entrySet()) {
JSONObject resultAllJson = new JSONObject();
String key = entry.getKey();
JSONObject json = (JSONObject) entry.getValue();
JSONObject json = entry.getValue();
String queryVal = json.getString("query");
// System.out.println("query: " + json.toString());
@ -274,16 +272,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
JSONArray resultJsonArr = new JSONArray();
// Solr
SolrQuery query = getQuery(queryVal, searchType, limit, propertiesList);
QueryResponse queryResponse = null;
SearchQuery query = getQuery(queryVal, searchType, limit, propertiesList);
SearchResponse queryResponse = null;
if (query != null) {
SolrServer solr = SolrSetup.getSolrServer(getServletContext());
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
queryResponse = solr.query(query);
} else {
log.error("Query for a search was null");
}
SolrDocumentList docs = null;
SearchResultDocumentList docs = null;
if (queryResponse != null) {
docs = queryResponse.getResults();
} else {
@ -293,29 +291,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
if (docs != null) {
List<SearchResult> results = new ArrayList<SearchResult>();
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
try {
String uri = doc.get(VitroSearchTermNames.URI).toString();
// RY 7/1/2011
// Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() returns a list.
// Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() could return a list
// But in fact: I'm no longer seeing any lists returned for individuals with multiple labels. Not sure
// if this is new behavior or what. ???
Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
String name = null;
if (nameRaw instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> nameRawList = (List<String>) nameRaw;
name = nameRawList.get(0);
} else {
name = (String) nameRaw;
}
String uri = doc.getStringValue(VitroSearchTermNames.URI);
String name = doc.getStringValue(VitroSearchTermNames.NAME_RAW);
SearchResult result = new SearchResult(name, uri);
// populate result for Google Refine
JSONObject resultJson = new JSONObject();
resultJson.put("score", doc.getFieldValue("score"));
resultJson.put("score", doc.getFirstValue("score"));
String modUri = result.getUri().replace("#", "%23");
resultJson.put("id", modUri);
resultJson.put("name", result.getLabel());
@ -361,16 +346,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet JSONException: "
+ ex);
} catch (SolrServerException ex) {
} catch (SearchEngineException ex) {
log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet SolrServerException: "
throw new ServletException("JSONReconcileServlet SearchEngineException: "
+ ex);
}
return qJson;
}
protected SolrQuery getQuery(String queryStr, String searchType, int limit, ArrayList<String[]> propertiesList) {
protected SearchQuery getQuery(String queryStr, String searchType, int limit, ArrayList<String[]> propertiesList) {
if ( queryStr == null) {
log.error("There was no parameter '"+ PARAM_QUERY
@ -383,10 +368,10 @@ public class JSONReconcileServlet extends VitroHttpServlet {
}
/// original
///SolrQuery query = new SolrQuery();
///SearchQuery query = new SearchQuery();
/// test
SolrQuery query = new SolrQuery(queryStr.toLowerCase());
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryStr.toLowerCase());
// original code:
// query.setStart(0).setRows(DEFAULT_MAX_HIT_COUNT);
@ -403,17 +388,17 @@ public class JSONReconcileServlet extends VitroHttpServlet {
}
// Added score to original code:
query.setFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, "*", "score"); // fields to retrieve
query.addFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, "*", "score"); // fields to retrieve
// if propertiesList has elements, add extra queries to query
Iterator<String[]> it = propertiesList.iterator();
while (it.hasNext()) {
String[] pvPair = it.next();
query.addFilterQuery(tokenizeNameQuery(pvPair[1]), VitroSearchTermNames.RDFTYPE + ":\"" + pvPair[0] + "\"");
query.addFilterQueries(tokenizeNameQuery(pvPair[1]), VitroSearchTermNames.RDFTYPE + ":\"" + pvPair[0] + "\"");
}
// Can't sort on multivalued field, so we sort the results in Java when we get them.
// query.setSortField(VitroSearchTermNames.NAME_LOWERCASE, SolrQuery.ORDER.asc);
// query.addSortField(VitroSearchTermNames.NAME_LOWERCASE, Order.ASC);
return query;
}

6
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/json/JsonServlet.java
View file

@ -113,8 +113,7 @@ public class JsonServlet extends VitroHttpServlet {
vclassURIs,
page, INDIVIDUALS_PER_PAGE,
alpha,
vreq.getWebappDaoFactory().getIndividualDao(),
context);
vreq.getWebappDaoFactory().getIndividualDao());
} catch(Exception ex) {
log.error("Error in retrieval of search results for VClass " + vclassURIs.toString(), ex);
return IndividualListResults.EMPTY;
@ -149,8 +148,7 @@ public class JsonServlet extends VitroHttpServlet {
vclassURI,
page,
pageSize,
vreq.getWebappDaoFactory().getIndividualDao(),
context);
vreq.getWebappDaoFactory().getIndividualDao());
} catch(Exception ex) {
log.error("Error in retrieval of search results for VClass " + vclassURI, ex);
return IndividualListResults.EMPTY;

66
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java
View file

@ -14,12 +14,6 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.listeners.StatementListener;
@ -30,6 +24,7 @@ import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
@ -40,11 +35,17 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerSetup;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
@ -188,14 +189,14 @@ public class VClassGroupCache implements IndexingEventListener {
int attempts = 0;
int maxTries = 3;
SolrServerException exception = null;
SearchEngineException exception = null;
while( attempts < maxTries ){
try {
attempts++;
rebuildCacheUsingSolr(this);
break;
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
exception = e;
try { Thread.sleep(250); }
catch (InterruptedException e1) {/*ignore interrupt*/}
@ -203,7 +204,7 @@ public class VClassGroupCache implements IndexingEventListener {
}
if( exception != null )
log.error("Could not rebuild cache. " + exception.getRootCause().getMessage() );
log.error("Could not rebuild cache. " + exception.getCause().getMessage() );
}
/**
@ -247,17 +248,13 @@ public class VClassGroupCache implements IndexingEventListener {
*
* If ProhibitedFromSearch is not found in the context, that will be skipped.
*
* @throws SolrServerException if there are problems with the Solr server.
* @throws SearchEngineException if there are problems with the Solr server.
*/
protected static void rebuildCacheUsingSolr( VClassGroupCache cache ) throws SolrServerException{
protected static void rebuildCacheUsingSolr( VClassGroupCache cache ) throws SearchEngineException{
long start = System.currentTimeMillis();
WebappDaoFactory wdFactory = ModelAccess.on(cache.context).getWebappDaoFactory();
SolrServer solrServer = (SolrServer)cache.context.getAttribute(SolrSetup.SOLR_SERVER);
if( solrServer == null){
log.error("Unable to rebuild cache: could not get solrServer from ServletContext");
return;
}
SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
VitroFilters vFilters = VitroFilterUtils.getPublicFilter(cache.context);
VClassGroupDao vcgDao = new WebappDaoFactoryFiltering(wdFactory, vFilters).getVClassGroupDao();
@ -291,7 +288,7 @@ public class VClassGroupCache implements IndexingEventListener {
* Removes classes from groups that are prohibited from search.
*/
protected static void removeClassesHiddenFromSearch(List<VClassGroup> groups, ServletContext context2) {
ProhibitedFromSearch pfs = (ProhibitedFromSearch)context2.getAttribute(SolrSetup.PROHIBITED_FROM_SEARCH);
ProhibitedFromSearch pfs = (ProhibitedFromSearch)context2.getAttribute(SearchIndexerSetup.PROHIBITED_FROM_SEARCH);
if(pfs==null){
log.debug("Could not get ProhibitedFromSearch from ServletContext");
return;
@ -311,10 +308,10 @@ public class VClassGroupCache implements IndexingEventListener {
/**
* Add the Individual count to classes in groups.
* @throws SolrServerException
* @throws SearchEngineException
*/
protected static void addCountsUsingSolr(List<VClassGroup> groups, SolrServer solrServer)
throws SolrServerException {
protected static void addCountsUsingSolr(List<VClassGroup> groups, SearchEngine solrServer)
throws SearchEngineException {
if( groups == null || solrServer == null )
return;
for( VClassGroup group : groups){
@ -322,23 +319,23 @@ public class VClassGroupCache implements IndexingEventListener {
}
}
protected static void addClassCountsToGroup(VClassGroup group, SolrServer solrServer)
throws SolrServerException {
protected static void addClassCountsToGroup(VClassGroup group, SearchEngine solrServer)
throws SearchEngineException {
if( group == null ) return;
String groupUri = group.getURI();
String facetOnField = VitroSearchTermNames.RDFTYPE;
SolrQuery query = new SolrQuery( ).
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery().
setRows(0).
setQuery(VitroSearchTermNames.CLASSGROUP_URI + ":" + groupUri ).
setFacet(true). //facet on type to get counts for classes in classgroup
addFacetField( facetOnField ).
setFaceting(true). //facet on type to get counts for classes in classgroup
addFacetFields( facetOnField ).
setFacetMinCount(0);
log.debug("query: " + query);
QueryResponse rsp = solrServer.query(query);
SearchResponse rsp = solrServer.query(query);
//Get individual count
long individualCount = rsp.getResults().getNumFound();
@ -346,7 +343,7 @@ public class VClassGroupCache implements IndexingEventListener {
group.setIndividualCount((int) individualCount);
//get counts for classes
FacetField ff = rsp.getFacetField( facetOnField );
SearchFacetField ff = rsp.getFacetField( facetOnField );
if( ff != null ){
List<Count> counts = ff.getValues();
if( counts != null ){
@ -407,7 +404,8 @@ public class VClassGroupCache implements IndexingEventListener {
this.cache = cache;
}
public void run() {
@Override
public void run() {
while (!die) {
int delay;
@ -425,12 +423,12 @@ public class VClassGroupCache implements IndexingEventListener {
log.debug("rebuildGroupCacheThread.run() -- rebuilt cache ");
failedAttempts = 0;
delay = 100;
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
failedAttempts++;
if( failedAttempts >= maxFailedAttempts ){
log.error("Could not build VClassGroupCache. " +
"Could not connect with Solr after " +
failedAttempts + " attempts.", e.getRootCause());
failedAttempts + " attempts.", e.getCause());
rebuildRequested = false;
failedAttempts = 0;
delay = 1000;
@ -479,11 +477,13 @@ public class VClassGroupCache implements IndexingEventListener {
* Listen for changes to what class group classes are in and their display rank.
*/
protected class VClassGroupCacheChangeListener extends StatementListener {
public void addedStatement(Statement stmt) {
@Override
public void addedStatement(Statement stmt) {
checkAndDoUpdate(stmt);
}
public void removedStatement(Statement stmt) {
@Override
public void removedStatement(Statement stmt) {
checkAndDoUpdate(stmt);
}

32
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/VTwo/fields/IndividualsViaSolrQueryOptions.java
View file

@ -8,23 +8,21 @@ import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.fields.FieldUtils;
/*
@ -34,13 +32,11 @@ import edu.cornell.mannlib.vitro.webapp.utils.fields.FieldUtils;
public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions implements FieldOptions {
private Log log = LogFactory.getLog(IndividualsViaSolrQueryOptions.class);
private ServletContext servletContext;
private String subjectUri;
private String predicateUri;
private String objectUri;
public IndividualsViaSolrQueryOptions(ServletContext context, String inputSubjectUri, String inputPredicateUri, String inputObjectUri, String ... vclassURIs) throws Exception {
public IndividualsViaSolrQueryOptions(String inputSubjectUri, String inputPredicateUri, String inputObjectUri, String ... vclassURIs) throws Exception {
super(vclassURIs);
this.servletContext = context;
this.subjectUri = inputSubjectUri;
this.predicateUri = inputPredicateUri;
this.objectUri = inputObjectUri;
@ -50,10 +46,10 @@ public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions
protected Map<String,Individual> getIndividualsForClass(String vclassURI, WebappDaoFactory wDaoFact ){
Map<String, Individual> individualMap = new HashMap<String, Individual>();
try {
SolrServer solrServer = SolrSetup.getSolrServer(servletContext);
SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
//solr query for type count.
SolrQuery query = new SolrQuery();
SearchQuery query = solrServer.createQuery();
if( VitroVocabulary.OWL_THING.equals( vclassURI )){
query.setQuery( "*:*" );
}else{
@ -61,15 +57,15 @@ public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions
}
query.setStart(0)
.setRows(1000);
query.setFields(VitroSearchTermNames.URI); // fields to retrieve
query.addFields(VitroSearchTermNames.URI); // fields to retrieve
QueryResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults();
SearchResponse rsp = solrServer.query(query);
SearchResultDocumentList docs = rsp.getResults();
long found = docs.getNumFound();
if(found > 0) {
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
try {
String uri = doc.get(VitroSearchTermNames.URI).toString();
String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual individual = wDaoFact.getIndividualDao().getIndividualByURI(uri);
if (individual == null) {
log.debug("No individual for search document with uri = " + uri);

36
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/configuration/generators/DefaultObjectPropertyFormGenerator.java
View file

@ -5,24 +5,18 @@ package edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.generators
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpSession;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Literal;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -37,9 +31,13 @@ import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.FieldVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropetyOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.validators.AntiXssValidation;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils.EditMode;
@ -175,9 +173,9 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
return types;
}
private boolean tooManyRangeOptions(VitroRequest vreq, HttpSession session ) throws SolrServerException {
private boolean tooManyRangeOptions(VitroRequest vreq, HttpSession session ) throws SearchEngineException {
List<VClass> rangeTypes = getRangeTypes(vreq);
SolrServer solrServer = SolrSetup.getSolrServer(session.getServletContext());
SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
List<String> types = new ArrayList<String>();
for (VClass vclass : rangeTypes) {
@ -194,15 +192,15 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
long count = 0;
for( String type:types){
//solr query for type count.
SolrQuery query = new SolrQuery();
SearchQuery query = solrServer.createQuery();
if( VitroVocabulary.OWL_THING.equals( type )){
query.setQuery( "*:*" );
}else{
query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type);
}
query.setRows(0);
QueryResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults();
SearchResponse rsp = solrServer.query(query);
SearchResultDocumentList docs = rsp.getResults();
long found = docs.getNumFound();
count = count + found;
if( count > maxNonACRangeIndividualCount )
@ -524,7 +522,7 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
editConfiguration.setFormSpecificData(formSpecificData);
}
public void addFormSpecificDataForAC(EditConfigurationVTwo editConfiguration, VitroRequest vreq, HttpSession session) throws SolrServerException {
public void addFormSpecificDataForAC(EditConfigurationVTwo editConfiguration, VitroRequest vreq, HttpSession session) throws SearchEngineException {
HashMap<String, Object> formSpecificData = new HashMap<String, Object>();
//Get the edit mode
formSpecificData.put("editMode", getEditMode(vreq).toString().toLowerCase());
@ -564,19 +562,19 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
editConfiguration.setFormSpecificData(formSpecificData);
}
private Object rangeIndividualsExist(HttpSession session, List<VClass> types) throws SolrServerException {
SolrServer solrServer = SolrSetup.getSolrServer(session.getServletContext());
private Object rangeIndividualsExist(HttpSession session, List<VClass> types) throws SearchEngineException {
SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
boolean rangeIndividualsFound = false;
for( VClass type:types){
//solr for type count.
SolrQuery query = new SolrQuery();
SearchQuery query =ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type.getURI());
query.setRows(0);
QueryResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults();
SearchResponse rsp = solrServer.query(query);
SearchResultDocumentList docs = rsp.getResults();
if( docs.getNumFound() > 0 ){
rangeIndividualsFound = true;
break;

29
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/configuration/generators/IndividualsByRankFormGenerator.java
View file

@ -2,47 +2,18 @@
package edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.generators;
import static edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpSession;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationUtils;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.FieldVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropertyByRankOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropetyOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.validators.AntiXssValidation;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils.EditMode;
/**
* Generates the edit configuration for a default property form.

21
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/CachingResponseFilter.java
View file

@ -21,15 +21,15 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.EnumerationUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vedit.beans.LoginStatusBean;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.UserAccount;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrResultsParser;
@ -244,13 +244,12 @@ public class CachingResponseFilter implements Filter {
* Ask Solr whether it has an ETAG for this URI.
*/
private String findEtagForIndividual(String individualUri) {
SolrQuery query = new SolrQuery("URI:" + individualUri)
.setFields(ETAG_FIELD);
SolrServer solr = SolrSetup.getSolrServer(ctx);
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchQuery query = solr.createQuery("URI:" + individualUri)
.addFields(ETAG_FIELD);
try {
QueryResponse response = solr.query(query);
SearchResponse response = solr.query(query);
List<Map<String, String>> maps = new SolrResultsParser(response,
parserFieldMap).parse();
log.debug("Solr response for '" + query.getQuery() + "' was "
@ -261,7 +260,7 @@ public class CachingResponseFilter implements Filter {
} else {
return maps.get(0).get(ETAG_FIELD);
}
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
log.warn(
"Solr query '" + query.getQuery() + "' threw an exception",
e);

72
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java
View file

@ -16,20 +16,20 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.json.JSONArray;
import org.json.JSONObject;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/**
* AutocompleteController generates autocomplete content
@ -66,7 +66,7 @@ public class AutocompleteController extends VitroAjaxController {
String qtxt = vreq.getParameter(PARAM_QUERY);
SolrQuery query = getQuery(qtxt, vreq);
SearchQuery query = getQuery(qtxt, vreq);
if (query == null ) {
log.debug("query for '" + qtxt +"' is null.");
doNoQuery(response);
@ -74,8 +74,8 @@ public class AutocompleteController extends VitroAjaxController {
}
log.debug("query for '" + qtxt +"' is " + query.toString());
SolrServer solr = SolrSetup.getSolrServer(getServletContext());
QueryResponse queryResponse = solr.query(query);
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchResponse queryResponse = solr.query(query);
if ( queryResponse == null) {
log.error("Query response for a search was null");
@ -83,7 +83,7 @@ public class AutocompleteController extends VitroAjaxController {
return;
}
SolrDocumentList docs = queryResponse.getResults();
SearchResultDocumentList docs = queryResponse.getResults();
if ( docs == null) {
log.error("Docs for a search was null");
@ -99,33 +99,11 @@ public class AutocompleteController extends VitroAjaxController {
}
List<SearchResult> results = new ArrayList<SearchResult>();
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
try {
String uri = doc.get(VitroSearchTermNames.URI).toString();
// RY 7/1/2011
// Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() returns a list.
// Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() could return a list
// But in fact: I'm no longer seeing any lists returned for individuals with multiple labels. Not sure
// if this is new behavior or what. ???
Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
String name = null;
if (nameRaw instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> nameRawList = (List<String>) nameRaw;
name = nameRawList.get(0);
} else {
name = (String) nameRaw;
}
Object mostSpecificType = doc.get(VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS);
String mst = null;
if (mostSpecificType instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> mstList = (List<String>) mostSpecificType;
mst = mstList.get(0);
} else {
mst = (String) mostSpecificType;
}
String uri = doc.getStringValue(VitroSearchTermNames.URI);
String name = doc.getStringValue(VitroSearchTermNames.NAME_RAW);
String mst = doc.getStringValue(VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS);
SearchResult result = new SearchResult(name, uri, mst);
results.add(result);
@ -150,7 +128,7 @@ public class AutocompleteController extends VitroAjaxController {
}
}
private SolrQuery getQuery(String queryStr, VitroRequest vreq) {
private SearchQuery getQuery(String queryStr, VitroRequest vreq) {
if ( queryStr == null) {
log.error("There was no parameter '"+ PARAM_QUERY
@ -162,26 +140,26 @@ public class AutocompleteController extends VitroAjaxController {
return null;
}
SolrQuery query = new SolrQuery();
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setStart(0)
.setRows(DEFAULT_MAX_HIT_COUNT);
setNameQuery(query, queryStr, vreq);
// Filter by type
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE);
String multipleTypesParam = (String) vreq.getParameter(PARAM_MULTIPLE_RDFTYPE);
String typeParam = vreq.getParameter(PARAM_RDFTYPE);
String multipleTypesParam = vreq.getParameter(PARAM_MULTIPLE_RDFTYPE);
if (typeParam != null) {
addFilterQuery(query, typeParam, multipleTypesParam);
}
query.setFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve
query.addFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve
// Can't sort on multivalued field, so we sort the results in Java when we get them.
// query.setSortField(VitroSearchTermNames.NAME_LOWERCASE, SolrQuery.ORDER.asc);
// query.addSortField(VitroSearchTermNames.NAME_LOWERCASE, Order.ASC);
return query;
}
private void addFilterQuery(SolrQuery query, String typeParam, String multipleTypesParam) {
private void addFilterQuery(SearchQuery query, String typeParam, String multipleTypesParam) {
if(multipleTypesParam == null || multipleTypesParam.equals("null") || multipleTypesParam.isEmpty()) {
//Single type parameter, process as usual
query.addFilterQuery(VitroSearchTermNames.RDFTYPE + ":\"" + typeParam + "\"");
@ -200,12 +178,12 @@ public class AutocompleteController extends VitroAjaxController {
}
}
private void setNameQuery(SolrQuery query, String queryStr, HttpServletRequest request) {
private void setNameQuery(SearchQuery query, String queryStr, HttpServletRequest request) {
if (StringUtils.isBlank(queryStr)) {
log.error("No query string");
}
String tokenizeParam = (String) request.getParameter("tokenize");
String tokenizeParam = request.getParameter("tokenize");
boolean tokenize = "true".equals(tokenizeParam);
// Note: Stemming is only relevant if we are tokenizing: an untokenized name
@ -218,7 +196,7 @@ public class AutocompleteController extends VitroAjaxController {
}
}
private void setTokenizedNameQuery(SolrQuery query, String queryStr, HttpServletRequest request) {
private void setTokenizedNameQuery(SearchQuery query, String queryStr, HttpServletRequest request) {
/* We currently have no use case for a tokenized, unstemmed autocomplete search field, so the option
* has been disabled. If needed in the future, will need to add a new field and field type which
@ -262,7 +240,7 @@ public class AutocompleteController extends VitroAjaxController {
}
private void setUntokenizedNameQuery(SolrQuery query, String queryStr) {
private void setUntokenizedNameQuery(SearchQuery query, String queryStr) {
queryStr = queryStr.trim();
queryStr = makeTermQuery(VitroSearchTermNames.AC_NAME_UNTOKENIZED, queryStr, true);
query.setQuery(queryStr);

17
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/DataAutocompleteController.java
View file

@ -5,28 +5,15 @@ package edu.cornell.mannlib.vitro.webapp.search.controller;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.json.JSONArray;
import org.json.JSONObject;
import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.DatasetFactory;
@ -42,10 +29,8 @@ import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils.AjaxControllerException;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
/**
* DataAutocompleteController generates autocomplete content

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java
View file

@ -15,9 +15,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@ -28,7 +27,6 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Red
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp;
@ -160,7 +158,7 @@ public class IndexController extends FreemarkerHttpServlet {
private Boolean testIndexConnection() {
try {
SolrSetup.getSolrServer(getServletContext()).ping();
ApplicationUtils.instance().getSearchEngine().ping();
return Boolean.TRUE;
} catch (Exception e) {
log.error("Can't connect to the Solr server.", e);

70
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java
View file

@ -21,14 +21,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -47,9 +41,15 @@ import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupsForRequest;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.IndexConstants;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
import edu.ucsf.vitro.opensocial.OpenSocialManager;
@ -169,9 +169,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return doFailedSearch(badQueryMsg, queryText, format, vreq);
}
SolrQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq);
SolrServer solr = SolrSetup.getSolrServer(getServletContext());
QueryResponse response = null;
SearchQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq);
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchResponse response = null;
try {
response = solr.query(query);
@ -186,7 +186,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText, format, vreq);
}
SolrDocumentList docs = response.getResults();
SearchResultDocumentList docs = response.getResults();
if (docs == null) {
log.error("Document list for a search was null");
return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText,format, vreq);
@ -199,11 +199,11 @@ public class PagedSearchController extends FreemarkerHttpServlet {
}
List<Individual> individuals = new ArrayList<Individual>(docs.size());
Iterator<SolrDocument> docIter = docs.iterator();
Iterator<SearchResultDocument> docIter = docs.iterator();
while( docIter.hasNext() ){
try {
SolrDocument doc = docIter.next();
String uri = doc.get(VitroSearchTermNames.URI).toString();
SearchResultDocument doc = docIter.next();
String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual ind = iDao.getIndividualByURI(uri);
if(ind != null) {
ind.setSearchSnippet( getSnippet(doc, response) );
@ -353,12 +353,12 @@ public class PagedSearchController extends FreemarkerHttpServlet {
* Get the class groups represented for the individuals in the documents.
* @param qtxt
*/
private List<VClassGroupSearchLink> getClassGroupsLinks(VitroRequest vreq, VClassGroupDao grpDao, SolrDocumentList docs, QueryResponse rsp, String qtxt) {
private List<VClassGroupSearchLink> getClassGroupsLinks(VitroRequest vreq, VClassGroupDao grpDao, SearchResultDocumentList docs, SearchResponse rsp, String qtxt) {
Map<String,Long> cgURItoCount = new HashMap<String,Long>();
List<VClassGroup> classgroups = new ArrayList<VClassGroup>( );
List<FacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){
List<SearchFacetField> ffs = rsp.getFacetFields();
for(SearchFacetField ff : ffs){
if(VitroSearchTermNames.CLASSGROUP_URI.equals(ff.getName())){
List<Count> counts = ff.getValues();
for( Count ct: counts){
@ -388,13 +388,13 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return classGroupLinks;
}
private List<VClassSearchLink> getVClassLinks(VClassDao vclassDao, SolrDocumentList docs, QueryResponse rsp, String qtxt){
private List<VClassSearchLink> getVClassLinks(VClassDao vclassDao, SearchResultDocumentList docs, SearchResponse rsp, String qtxt){
HashSet<String> typesInHits = getVClassUrisForHits(docs);
List<VClass> classes = new ArrayList<VClass>(typesInHits.size());
Map<String,Long> typeURItoCount = new HashMap<String,Long>();
List<FacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){
List<SearchFacetField> ffs = rsp.getFacetFields();
for(SearchFacetField ff : ffs){
if(VitroSearchTermNames.RDFTYPE.equals(ff.getName())){
List<Count> counts = ff.getValues();
for( Count ct: counts){
@ -435,9 +435,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return vClassLinks;
}
private HashSet<String> getVClassUrisForHits(SolrDocumentList docs){
private HashSet<String> getVClassUrisForHits(SearchResultDocumentList docs){
HashSet<String> typesInHits = new HashSet<String>();
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
try {
Collection<Object> types = doc.getFieldValues(VitroSearchTermNames.RDFTYPE);
if (types != null) {
@ -453,8 +453,8 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return typesInHits;
}
private String getSnippet(SolrDocument doc, QueryResponse response) {
String docId = doc.get(VitroSearchTermNames.DOCID).toString();
private String getSnippet(SearchResultDocument doc, SearchResponse response) {
String docId = doc.getStringValue(VitroSearchTermNames.DOCID);
StringBuffer text = new StringBuffer();
Map<String, Map<String, List<String>>> highlights = response.getHighlighting();
if (highlights != null && highlights.get(docId) != null) {
@ -466,19 +466,19 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return text.toString();
}
private SolrQuery getQuery(String queryText, int hitsPerPage, int startIndex, VitroRequest vreq) {
private SearchQuery getQuery(String queryText, int hitsPerPage, int startIndex, VitroRequest vreq) {
// Lowercase the search term to support wildcard searches: Solr applies no text
// processing to a wildcard search term.
SolrQuery query = new SolrQuery( queryText );
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryText);
query.setStart( startIndex )
.setRows(hitsPerPage);
// ClassGroup filtering param
String classgroupParam = (String) vreq.getParameter(PARAM_CLASSGROUP);
String classgroupParam = vreq.getParameter(PARAM_CLASSGROUP);
// rdf:type filtering param
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE);
String typeParam = vreq.getParameter(PARAM_RDFTYPE);
if ( ! StringUtils.isBlank(classgroupParam) ) {
// ClassGroup filtering
@ -487,9 +487,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
query.addFilterQuery(VitroSearchTermNames.CLASSGROUP_URI + ":\"" + classgroupParam + "\"");
//with ClassGroup filtering we want type facets
query.add("facet","true");
query.add("facet.limit","-1");
query.add("facet.field",VitroSearchTermNames.RDFTYPE);
query.addParameter("facet","true");
query.addParameter("facet.limit","-1");
query.addParameter("facet.field",VitroSearchTermNames.RDFTYPE);
}else if ( ! StringUtils.isBlank(typeParam) ) {
// rdf:type filtering
@ -500,9 +500,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
//with type filtering we don't have facets.
}else{
//When no filtering is set, we want ClassGroup facets
query.add("facet","true");
query.add("facet.limit","-1");
query.add("facet.field",VitroSearchTermNames.CLASSGROUP_URI);
query.addParameter("facet","true");
query.addParameter("facet.limit","-1");
query.addParameter("facet.field",VitroSearchTermNames.CLASSGROUP_URI);
}
log.debug("Query = " + query.toString());

64
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java
View file

@ -7,15 +7,16 @@ import java.util.HashSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
@ -24,7 +25,7 @@ import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualT
public class SolrIndexer implements IndexerIface {
private final static Log log = LogFactory.getLog(SolrIndexer.class);
protected SolrServer server;
protected SearchEngine server;
protected boolean indexing;
protected HashSet<String> urisIndexed;
protected IndividualToSolrDocument individualToSolrDoc;
@ -48,7 +49,7 @@ public class SolrIndexer implements IndexerIface {
*/
protected boolean doingFullIndexRebuild = false;
public SolrIndexer( SolrServer server, IndividualToSolrDocument indToDoc){
public SolrIndexer( SearchEngine server, IndividualToSolrDocument indToDoc){
this.server = server;
this.individualToSolrDoc = indToDoc;
}
@ -59,15 +60,17 @@ public class SolrIndexer implements IndexerIface {
throw new IndexingException("SolrIndexer: must call " +
"startIndexing() before index().");
if( ind == null )
if( ind == null ) {
log.debug("Individual to index was null, ignoring.");
return;
}
try{
if( urisIndexed.contains(ind.getURI()) ){
log.debug("already indexed " + ind.getURI() );
return;
}else{
SolrInputDocument solrDoc = null;
SearchInputDocument solrDoc = null;
synchronized(this){
urisIndexed.add(ind.getURI());
}
@ -80,16 +83,14 @@ public class SolrIndexer implements IndexerIface {
log.debug( solrDoc.toString() );
}
UpdateResponse res = server.add( solrDoc );
log.debug("response after adding docs to server: "+ res);
server.add( solrDoc );
log.debug("Added docs to server.");
}else{
log.debug("removing from index " + ind.getURI());
removeFromIndex(ind.getURI());
}
}
} catch (IOException ex) {
throw new IndexingException(ex.getMessage());
} catch (SolrServerException ex) {
} catch (SearchEngineException ex) {
throw new IndexingException(ex.getMessage());
}
}
@ -111,9 +112,7 @@ public class SolrIndexer implements IndexerIface {
try {
server.deleteById(individualToSolrDoc.getIdForUri(uri));
log.debug("deleted " + " " + uri);
} catch (SolrServerException e) {
log.error( "could not delete individual " + uri, e);
} catch (IOException e) {
} catch (SearchEngineException e) {
log.error( "could not delete individual " + uri, e);
}
}
@ -172,12 +171,9 @@ public class SolrIndexer implements IndexerIface {
try {
server.deleteByQuery("indexedTime:[ * TO " + reindexStart + " ]");
server.commit();
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
if( ! shutdownRequested )
log.error("could not delete documents from before rebuild.",e);
} catch (IOException e) {
if( ! shutdownRequested )
log.error("could not delete documents from before rebuild.",e);
}
}
@ -186,17 +182,17 @@ public class SolrIndexer implements IndexerIface {
public long getModified() {
long modified = 0;
SolrQuery query = new SolrQuery();
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery("*:*");
query.addSortField("indexedTime", SolrQuery.ORDER.desc);
query.addSortField("indexedTime", Order.DESC);
try {
QueryResponse rsp = server.query(query);
SolrDocumentList docs = rsp.getResults();
SearchResponse rsp = server.query(query);
SearchResultDocumentList docs = rsp.getResults();
if(docs!=null){
modified = (Long)docs.get(0).getFieldValue("indexedTime");
modified = (Long)docs.get(0).getFirstValue("indexedTime");
}
} catch (SolrServerException e) {
} catch (SearchEngineException e) {
log.error(e,e);
}
@ -208,16 +204,16 @@ public class SolrIndexer implements IndexerIface {
* and returns false on failure to connect to server.
*/
public boolean isIndexEmpty() {
SolrQuery query = new SolrQuery();
SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery("*:*");
try {
QueryResponse rsp = server.query(query);
SolrDocumentList docs = rsp.getResults();
SearchResponse rsp = server.query(query);
SearchResultDocumentList docs = rsp.getResults();
if(docs==null || docs.size()==0){
return true;
}
} catch (SolrServerException e) {
log.error("Could not connect to solr server" ,e.getRootCause());
} catch (SearchEngineException e) {
log.error("Could not connect to solr server" ,e.getCause());
}
return false;
}

209
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java
View file

@ -1,209 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.search.beans.StatementToURIsToUpdate;
import edu.cornell.mannlib.vitro.webapp.search.indexing.AdditionalUriFinders;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.indexing.SearchReindexingListener;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.DocumentModifier;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnNamespace;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnType;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnTypeNamespace;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeNonFlagVitro;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.NameBoost;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.NameFields;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.SearchIndexExcluder;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.SyncingExcludeBasedOnType;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ThumbnailImageURL;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
public class SolrSetup implements javax.servlet.ServletContextListener{
public static final String SOLR_SERVER = "vitro.local.solr.server";
public static final String PROHIBITED_FROM_SEARCH = "edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch";
/** Exclude from the search index Individuals with types from these namespaces */
private static final String[] TYPE_NS_EXCLUDES = {
VitroVocabulary.PUBLIC
//if you do OWL.NS here you will exclude all of owl:Thing.
};
/** Exclude from the search index individuals who's URIs start with these namespaces. */
private static final String[] INDIVIDUAL_NS_EXCLUDES={
VitroVocabulary.vitroURI,
VitroVocabulary.VITRO_PUBLIC,
VitroVocabulary.PSEUDO_BNODE_NS,
OWL.NS
};
/** Individuals of these types will be excluded from the search index */
private static final String[] OWL_TYPES_EXCLUDES = {
OWL.ObjectProperty.getURI(),
OWL.DatatypeProperty.getURI(),
OWL.AnnotationProperty.getURI()
};
@Override
public void contextInitialized(ServletContextEvent sce) {
ServletContext context = sce.getServletContext();
StartupStatus ss = StartupStatus.getBean(context);
/* setup the http connection with the solr server */
String solrServerUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.solr.url");
if( solrServerUrlString == null ){
ss.fatal(this, "Could not find vitro.local.solr.url in runtime.properties. "+
"Vitro application needs a URL of a solr server that it can use to index its data. " +
"It should be something like http://localhost:${port}" + context.getContextPath() + "solr"
);
return;
}
try {
HttpSolrServer server;
boolean useMultiPartPost = true;
server = new HttpSolrServer( solrServerUrlString );
server.setSoTimeout(10000); // socket read timeout
server.setConnectionTimeout(10000);
server.setDefaultMaxConnectionsPerHost(100);
server.setMaxTotalConnections(100);
server.setMaxRetries(1);
context.setAttribute(SOLR_SERVER, server);
/* set up the individual to solr doc translation */
OntModel jenaOntModel = ModelAccess.on(context).getJenaOntModel();
OntModel displayModel = ModelAccess.on(context).getDisplayModel();
/* try to get context attribute DocumentModifiers
* and use that as the start of the list of DocumentModifier
* objects. This allows other ContextListeners to add to
* the basic set of DocumentModifiers. */
@SuppressWarnings("unchecked")
List<DocumentModifier> modifiersFromContext =
(List<DocumentModifier>)context.getAttribute("DocumentModifiers");
/* try to get context attribute SearchIndexExcludes
* and use that as the start of the list of exclude
* objects. This allows other ContextListeners to add to
* the basic set of SearchIndexExcludes . */
@SuppressWarnings("unchecked")
List<SearchIndexExcluder> searchIndexExcludesFromContext =
(List<SearchIndexExcluder>)context.getAttribute("SearchIndexExcludes");
IndividualToSolrDocument indToSolrDoc =
setupTransltion(jenaOntModel, displayModel,
RDFServiceUtils.getRDFServiceFactory(context),
modifiersFromContext,
searchIndexExcludesFromContext);
/* setup solr indexer */
SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc);
// This is where the builder gets the list of places to try to
// get objects to index. It is filtered so that non-public text
// does not get into the search index.
WebappDaoFactory wadf = ModelAccess.on(context).getWebappDaoFactory();
VitroFilters vf = VitroFilterUtils.getPublicFilter(context);
wadf = new WebappDaoFactoryFiltering(wadf, vf);
// make objects that will find additional URIs for context nodes etc
RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(context).getRDFService();
List<StatementToURIsToUpdate> uriFinders = AdditionalUriFinders.getList(rdfService,wadf.getIndividualDao());
// Make the IndexBuilder
IndexBuilder builder = new IndexBuilder( solrIndexer, wadf, uriFinders );
// Save it to the servlet context so we can access it later in the webapp.
context.setAttribute(IndexBuilder.class.getName(), builder);
// set up listeners so search index builder is notified of changes to model
ServletContext ctx = sce.getServletContext();
SearchReindexingListener srl = new SearchReindexingListener( builder );
ModelContext.registerListenerForChanges(ctx, srl);
ss.info(this, "Setup of Solr index completed.");
} catch (Throwable e) {
ss.fatal(this, "could not setup local solr server",e);
}
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName());
if( builder != null )
builder.stopIndexingThread();
}
public static SolrServer getSolrServer(ServletContext ctx){
return (SolrServer) ctx.getAttribute(SOLR_SERVER);
}
public static IndividualToSolrDocument setupTransltion(
OntModel jenaOntModel,
Model displayModel,
RDFServiceFactory rdfServiceFactory,
List<DocumentModifier> modifiersFromContext,
List<SearchIndexExcluder> searchIndexExcludesFromContext
) {
/* try to get context attribute DocumentModifiers
* and use that as the start of the list of DocumentModifier
* objects. This allows other ContextListeners to add to
* the basic set of DocumentModifiers. */
List<DocumentModifier> modifiers = new ArrayList<DocumentModifier>();
if( modifiersFromContext != null ){
modifiers.addAll( modifiersFromContext);
}
modifiers.add( new NameFields( rdfServiceFactory ));
modifiers.add( new NameBoost( 1.2f ));
modifiers.add( new ThumbnailImageURL( rdfServiceFactory ));
/* try to get context attribute SearchIndexExcludes
* and use that as the start of the list of exclude
* objects. This allows other ContextListeners to add to
* the basic set of SearchIndexExcludes . */
List<SearchIndexExcluder> excludes =
new ArrayList<SearchIndexExcluder>();
if( searchIndexExcludesFromContext != null ){
excludes.addAll(searchIndexExcludesFromContext);
}
excludes.add(new ExcludeBasedOnNamespace( INDIVIDUAL_NS_EXCLUDES ));
excludes.add(new ExcludeBasedOnTypeNamespace( TYPE_NS_EXCLUDES ) );
excludes.add(new ExcludeBasedOnType( OWL_TYPES_EXCLUDES) );
excludes.add(new ExcludeNonFlagVitro() );
excludes.add( new SyncingExcludeBasedOnType( displayModel ) );
return new IndividualToSolrDocument(excludes, modifiers);
}
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ContextNodeFields.java
View file

@ -9,16 +9,14 @@ import java.util.List;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.QuerySolutionMap;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -53,7 +51,7 @@ public class ContextNodeFields implements DocumentModifier{
}
@Override
public void modifyDocument(Individual individual, SolrInputDocument doc, StringBuffer addUri) {
public void modifyDocument(Individual individual, SearchInputDocument doc, StringBuffer addUri) {
if( individual == null )
return;
@ -65,11 +63,11 @@ public class ContextNodeFields implements DocumentModifier{
/* get text from the context nodes and add the to ALLTEXT */
StringBuffer values = executeQueryForValues(individual, queries);
SolrInputField field = doc.getField(VitroSearchTermNames.ALLTEXT);
SearchInputField field = doc.getField(VitroSearchTermNames.ALLTEXT);
if( field == null ){
doc.addField(VitroSearchTermNames.ALLTEXT, values);
}else{
field.addValue(values, field.getBoost());
field.addValues(values, field.getBoost());
}
}

7
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/DocumentModifier.java
View file

@ -2,14 +2,13 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
/**
* This interface represents an object that can add to a SolrInputDocument.
* This interface represents an object that can add to a SearchInputDocument.
*/
public interface DocumentModifier {
public void modifyDocument(Individual individual, SolrInputDocument doc, StringBuffer addUri) throws SkipIndividualException;
public void modifyDocument(Individual individual, SearchInputDocument doc, StringBuffer addUri) throws SkipIndividualException;
//called to inform the DocumentModifier that the system is shutting down
public void shutdown();

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ExcludeBasedOnTypeNamespace.java
View file

@ -7,8 +7,6 @@ import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;

27
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/IndividualToSolrDocument.java
View file

@ -12,19 +12,20 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.joda.time.DateTime;
import org.jsoup.Jsoup;
import com.hp.hpl.jena.shared.JenaException;
import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
@ -46,7 +47,7 @@ public class IndividualToSolrDocument {
}
@SuppressWarnings("static-access")
public SolrInputDocument translate(Individual ind) throws IndexingException{
public SearchInputDocument translate(Individual ind) throws IndexingException{
try{
String excludeMsg = checkExcludes( ind );
if( excludeMsg != DONT_EXCLUDE){
@ -54,7 +55,7 @@ public class IndividualToSolrDocument {
return null;
}
SolrInputDocument doc = new SolrInputDocument();
SearchInputDocument doc = ApplicationUtils.instance().getSearchEngine().createInputDocument();
//DocID
doc.addField(term.DOCID, getIdForUri( ind.getURI() ) );
@ -79,7 +80,7 @@ public class IndividualToSolrDocument {
addObjectPropertyText(ind, doc, objectNames, addUri);
//time of index in msec past epoch
doc.addField(term.INDEXEDTIME, new Long( (new DateTime()).getMillis() ) );
doc.addField(term.INDEXEDTIME, (Object) new DateTime().getMillis() );
addAllText( ind, doc, classPublicNames, objectNames );
@ -125,7 +126,7 @@ public class IndividualToSolrDocument {
protected Map<String,Long> docModClassToTime = new HashMap<String,Long>();
protected long docModCount =0;
protected void runAdditionalDocModifers( Individual ind, SolrInputDocument doc, StringBuffer addUri )
protected void runAdditionalDocModifers( Individual ind, SearchInputDocument doc, StringBuffer addUri )
throws SkipIndividualException{
//run the document modifiers
if( documentModifiers != null && !documentModifiers.isEmpty()){
@ -158,7 +159,7 @@ public class IndividualToSolrDocument {
}
}
protected void addAllText(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames, StringBuffer objectNames) {
protected void addAllText(Individual ind, SearchInputDocument doc, StringBuffer classPublicNames, StringBuffer objectNames) {
String t=null;
//ALLTEXT, all of the 'full text'
StringBuffer allTextValue = new StringBuffer();
@ -209,7 +210,7 @@ public class IndividualToSolrDocument {
* Get the rdfs:labes for objects of statements and put in objectNames.
* Get the URIs for objects of statements and put in addUri.
*/
protected void addObjectPropertyText(Individual ind, SolrInputDocument doc,
protected void addObjectPropertyText(Individual ind, SearchInputDocument doc,
StringBuffer objectNames, StringBuffer addUri) {
try{
@ -245,7 +246,7 @@ public class IndividualToSolrDocument {
* @returns true if prohibited from search
* @throws SkipIndividualException
*/
protected void addClasses(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{
protected void addClasses(Individual ind, SearchInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{
ArrayList<String> superClassNames = null;
List<VClass> vclasses = ind.getVClasses(false);
@ -279,7 +280,7 @@ public class IndividualToSolrDocument {
}
}
protected void addMostSpecificTypeUris(Individual ind, SolrInputDocument doc){
protected void addMostSpecificTypeUris(Individual ind, SearchInputDocument doc){
List<String> mstURIs = ind.getMostSpecificTypeURIs();
if( mstURIs != null ){
for( String typeURI : mstURIs ){
@ -289,7 +290,7 @@ public class IndividualToSolrDocument {
}
}
protected void addLabel(Individual ind, SolrInputDocument doc) {
protected void addLabel(Individual ind, SearchInputDocument doc) {
String value = "";
String label = ind.getRdfsLabel();
if (label != null) {
@ -324,8 +325,8 @@ public class IndividualToSolrDocument {
public Individual unTranslate(Object result) {
Individual ent = null;
if( result != null && result instanceof SolrDocument){
SolrDocument hit = (SolrDocument) result;
if( result instanceof SearchResultDocument){
SearchResultDocument hit = (SearchResultDocument) result;
String uri= (String) hit.getFirstValue(term.URI);
ent = new IndividualImpl();

9
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/NameBoost.java
View file

@ -2,10 +2,9 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
public class NameBoost implements DocumentModifier {
@ -27,11 +26,11 @@ public class NameBoost implements DocumentModifier {
}
@Override
public void modifyDocument(Individual individual, SolrInputDocument doc,
public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) {
for( String fieldName : fieldsToBoost){
SolrInputField field = doc.getField(fieldName);
SearchInputField field = doc.getField(fieldName);
if( field != null ){
field.setBoost(field.getBoost() + boost);
}

4
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/NameFields.java
View file

@ -8,9 +8,9 @@ import java.io.InputStreamReader;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
@ -31,7 +31,7 @@ public class NameFields implements DocumentModifier {
}
@Override
public void modifyDocument(Individual ind, SolrInputDocument doc,
public void modifyDocument(Individual ind, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException {
if( ind == null || ind.getURI() == null ){
return;

5
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/SourceInstitution.java
View file

@ -2,9 +2,8 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
public class SourceInstitution implements DocumentModifier {
@ -22,7 +21,7 @@ public class SourceInstitution implements DocumentModifier {
}
@Override
public void modifyDocument(Individual individual, SolrInputDocument doc,
public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException {
doc.addField(VitroSearchTermNames.SITE_URL, siteURL);

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ThumbnailImageURL.java
View file

@ -9,13 +9,13 @@ import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.RDFNode;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -45,7 +45,7 @@ public class ThumbnailImageURL implements DocumentModifier {
}
@Override
public void modifyDocument(Individual individual, SolrInputDocument doc,
public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException {
//add a field for storing the location of thumbnail for the individual.
@ -56,7 +56,7 @@ public class ThumbnailImageURL implements DocumentModifier {
/**
* Adds if the individual has a thumbnail image or not.
*/
protected void addThumbnailExistence(Individual ind, SolrInputDocument doc) {
protected void addThumbnailExistence(Individual ind, SearchInputDocument doc) {
try{
if(ind.hasThumb())
doc.addField(THUMBNAIL, "1");

7
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java
View file

@ -9,8 +9,6 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.vocabulary.OWL;
@ -23,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -75,7 +74,7 @@ public class SearchIndexerSetup implements ServletContextListener {
public void contextInitialized(ServletContextEvent sce) {
ServletContext context = sce.getServletContext();
StartupStatus ss = StartupStatus.getBean(context);
HttpSolrServer server = (HttpSolrServer) ApplicationUtils.instance().getSearchEngine();
SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try {
/* set up the individual to solr doc translation */
@ -107,7 +106,7 @@ public class SearchIndexerSetup implements ServletContextListener {
modifiersFromContext, searchIndexExcludesFromContext);
/* setup solr indexer */
SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc);
SolrIndexer solrIndexer = new SolrIndexer(searchEngine, indToSolrDoc);
// This is where the builder gets the list of places to try to
// get objects to index. It is filtered so that non-public text

17
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/DataGetterUtils.java
View file

@ -8,7 +8,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang.StringUtils;
@ -56,7 +55,7 @@ public class DataGetterUtils {
* exception if a page has PageDataGetters.
*/
public static List<DataGetter> getDataGettersForPage(VitroRequest vreq, Model displayModel, String pageURI)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException {
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
if( vreq.getAttribute(DATA_GETTERS_FOR_PAGE) != null){
return (List<DataGetter>) vreq.getAttribute(DATA_GETTERS_FOR_PAGE);
@ -74,7 +73,7 @@ public class DataGetterUtils {
* This allows the individual profile for an individual of a specific class to be returned .
*/
public static List<DataGetter> getDataGettersForClass( VitroRequest vreq, Model displayModel, String classURI)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException{
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, classURI);
List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris);
log.debug("getDataGettersForClass: " + dgList);
@ -86,7 +85,7 @@ public class DataGetterUtils {
* @param templateName a filename like "index.ftl", which will be used as a URI like "freemarker:index.ftl".
*/
public static List<DataGetter> getDataGettersForTemplate( VitroRequest vreq, Model displayModel, String templateName)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException{
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
String templateUri = "freemarker:" + templateName;
List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, templateUri);
List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris);
@ -197,7 +196,7 @@ public class DataGetterUtils {
}finally{ displayModel.leaveCriticalSection(); }
return chooseType( types, displayModel, dataGetterURI);
return chooseType( types, dataGetterURI);
}
@ -229,7 +228,7 @@ public class DataGetterUtils {
return dgURIs;
}
private static String chooseType(List<String> types, Model displayModel, String dataGetterURI) throws IllegalAccessException {
private static String chooseType(List<String> types, String dataGetterURI) throws IllegalAccessException {
//currently just get the first one that is not owl:Thing
for(String type : types){
if( ! StringUtils.isEmpty( type ) && !type.equals( OWL.Thing.getURI() ))
@ -270,8 +269,8 @@ public class DataGetterUtils {
/**
* Get Individual count for Solr query for intersection of multiple classes
*/
public static long getIndividualCountForIntersection(VitroRequest vreq, ServletContext context, List<String> classUris) {
return IndividualListController.getIndividualCount(classUris, vreq.getWebappDaoFactory().getIndividualDao(), context);
public static long getIndividualCountForIntersection(VitroRequest vreq, List<String> classUris) {
return IndividualListController.getIndividualCount(classUris, vreq.getWebappDaoFactory().getIndividualDao());
}
//Return data getter type to be employed in display model
@ -410,7 +409,7 @@ public class DataGetterUtils {
/*
* Copied from JSONServlet as expect this to be related to VitroClassGroup
*/
public static JSONObject processVClassGroupJSON(VitroRequest vreq, ServletContext context, VClassGroup vcg) {
public static JSONObject processVClassGroupJSON(VClassGroup vcg) {
JSONObject map = new JSONObject();
try {
ArrayList<JSONObject> classes = new ArrayList<JSONObject>(vcg.size());

8
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/IndividualsForClassesDataGetter.java
View file

@ -163,7 +163,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
protected void processClassesAndRestrictions(VitroRequest vreq, ServletContext context,
HashMap<String, Object> data, List<String> classes, List<String> restrictClasses ) {
processClassesForDisplay(vreq, context, data, classes);
processRestrictionClasses(vreq, context, data, restrictClasses);
processRestrictionClasses(vreq, data, restrictClasses);
processIntersections(vreq, context, data);
}
@ -217,7 +217,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
for(VClass r: restrictClasses) {
classUris.add(r.getURI());
}
long count = DataGetterUtils.getIndividualCountForIntersection(vreq, context, classUris);
long count = DataGetterUtils.getIndividualCountForIntersection(vreq, classUris);
return new Long(count).intValue();
}
@ -254,7 +254,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
data.put("vClassGroup", classesGroup);
}
private void processRestrictionClasses(VitroRequest vreq, ServletContext context,
private void processRestrictionClasses(VitroRequest vreq,
HashMap<String, Object> data, List<String> restrictClasses) {
try {
VClassGroup restrictClassesGroup = new VClassGroup();
@ -305,7 +305,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
}
}
public static VClassGroupTemplateModel getClassGroup(String classGroupUri, ServletContext context, VitroRequest vreq){
public static VClassGroupTemplateModel getClassGroup(String classGroupUri, VitroRequest vreq){
VClassGroupsForRequest vcgc = VClassGroupCache.getVClassGroups(vreq);
List<VClassGroup> vcgList = vcgc.getGroups();

3
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SolrIndividualsDataGetter.java
View file

@ -176,8 +176,7 @@ public class SolrIndividualsDataGetter extends DataGetterBase implements DataGet
vclass.getURI(),
page,
alpha,
vreq.getWebappDaoFactory().getIndividualDao(),
vreq.getSession().getServletContext());
vreq.getWebappDaoFactory().getIndividualDao());
body.putAll(vcResults.asFreemarkerMap());
List<Individual> inds = vcResults.getEntities();

61
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/solr/SolrQueryUtils.java
View file

@ -7,21 +7,20 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListQueryResults;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/**
* Some static method to help in constructing Solr queries and parsing the
@ -63,7 +62,7 @@ public class SolrQueryUtils {
* the result, according to the fieldMap.
*/
public static List<Map<String, String>> parseResponse(
QueryResponse queryResponse, FieldMap fieldMap) {
SearchResponse queryResponse, FieldMap fieldMap) {
return new SolrResultsParser(queryResponse, fieldMap).parse();
}
@ -75,7 +74,7 @@ public class SolrQueryUtils {
* the result, according to the fieldMap.
*/
public static List<Map<String, String>> parseAndFilterResponse(
QueryResponse queryResponse, FieldMap fieldMap,
SearchResponse queryResponse, FieldMap fieldMap,
SolrResponseFilter filter, int maxNumberOfResults) {
return new SolrResultsParser(queryResponse, fieldMap)
.parseAndFilterResponse(filter, maxNumberOfResults);
@ -139,12 +138,12 @@ public class SolrQueryUtils {
}
//Get count of individuals without actually getting the results
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
SolrQuery query = new SolrQuery(makeMultiClassQuery(vclassUris));
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao) {
SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchQuery query = solr.createQuery(makeMultiClassQuery(vclassUris));
query.setRows(0);
try {
SolrServer solr = SolrSetup.getSolrServer(context);
QueryResponse response = null;
SearchResponse response = null;
response = solr.query(query);
return response.getResults().getNumFound();
} catch(Exception ex) {
@ -157,8 +156,9 @@ public class SolrQueryUtils {
* builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by
* alpha, and just the hits for the page for pageSize.
*/
public static SolrQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
public static SearchQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
String queryText = "";
SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try {
queryText = makeMultiClassQuery(vclassUris);
@ -168,31 +168,32 @@ public class SolrQueryUtils {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
}
SolrQuery query = new SolrQuery(queryText);
SearchQuery query = searchEngine.createQuery(queryText);
//page count starts at 1, row count starts at 0
int startRow = (page-1) * pageSize ;
query.setStart( startRow ).setRows( pageSize );
// Need a single-valued field for sorting
query.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
query.addSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
log.debug("Query is " + query.toString());
return query;
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return new SolrQuery();
return searchEngine.createQuery();
}
}
public static SolrQuery getRandomQuery(List<String> vclassUris, int page, int pageSize){
public static SearchQuery getRandomQuery(List<String> vclassUris, int page, int pageSize){
String queryText = "";
SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try {
try {
queryText = makeMultiClassQuery(vclassUris);
log.debug("queryText is " + queryText);
SolrQuery query = new SolrQuery(queryText);
SearchQuery query = searchEngine.createQuery(queryText);
//page count starts at 1, row count starts at 0
query.setStart( page ).setRows( pageSize );
@ -202,7 +203,7 @@ public class SolrQueryUtils {
} catch (Exception ex){
log.error("Could not make the Solr query",ex);
return new SolrQuery();
return searchEngine.createQuery();
}
}
@ -221,11 +222,10 @@ public class SolrQueryUtils {
}
public static IndividualListQueryResults buildAndExecuteVClassQuery(
List<String> vclassURIs, String alpha, int page, int pageSize,
ServletContext context, IndividualDao indDao)
throws SolrServerException {
SolrQuery query = SolrQueryUtils.getQuery(vclassURIs, alpha, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
List<String> vclassURIs, String alpha, int page, int pageSize, IndividualDao indDao)
throws SearchEngineException {
SearchQuery query = SolrQueryUtils.getQuery(vclassURIs, alpha, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao);
log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs);
@ -234,11 +234,10 @@ public class SolrQueryUtils {
}
public static IndividualListQueryResults buildAndExecuteRandomVClassQuery(
List<String> vclassURIs, int page, int pageSize,
ServletContext context, IndividualDao indDao)
throws SolrServerException {
SolrQuery query = SolrQueryUtils.getRandomQuery(vclassURIs, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
List<String> vclassURIs, int page, int pageSize, IndividualDao indDao)
throws SearchEngineException {
SearchQuery query = SolrQueryUtils.getRandomQuery(vclassURIs, page, pageSize);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao);
log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs);

41
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/solr/SolrResultsParser.java
View file

@ -9,9 +9,10 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
/**
* Parse this Solr response, creating a map of values for each document.
@ -22,10 +23,10 @@ import org.apache.solr.common.SolrDocumentList;
public class SolrResultsParser {
private static final Log log = LogFactory.getLog(SolrResultsParser.class);
private final QueryResponse queryResponse;
private final SearchResponse queryResponse;
private final Map<String, String> fieldNameMapping;
public SolrResultsParser(QueryResponse queryResponse, FieldMap fieldMap) {
public SolrResultsParser(SearchResponse queryResponse, FieldMap fieldMap) {
this.queryResponse = queryResponse;
this.fieldNameMapping = fieldMap.map();
}
@ -41,14 +42,14 @@ public class SolrResultsParser {
return maps;
}
SolrDocumentList docs = queryResponse.getResults();
SearchResultDocumentList docs = queryResponse.getResults();
if (docs == null) {
log.debug("Docs for a search was null");
return maps;
}
log.debug("Total number of hits = " + docs.getNumFound());
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
maps.add(parseSingleDocument(doc));
}
@ -69,14 +70,14 @@ public class SolrResultsParser {
return maps;
}
SolrDocumentList docs = queryResponse.getResults();
SearchResultDocumentList docs = queryResponse.getResults();
if (docs == null) {
log.debug("Docs for a search was null");
return maps;
}
log.debug("Total number of hits = " + docs.getNumFound());
for (SolrDocument doc : docs) {
for (SearchResultDocument doc : docs) {
Map<String, String> map = parseSingleDocument(doc);
if (filter.accept(map)) {
maps.add(map);
@ -92,7 +93,7 @@ public class SolrResultsParser {
/**
* Create a map from this document, applying translation on the field names.
*/
private Map<String, String> parseSingleDocument(SolrDocument doc) {
private Map<String, String> parseSingleDocument(SearchResultDocument doc) {
Map<String, String> result = new HashMap<String, String>();
for (String solrFieldName : fieldNameMapping.keySet()) {
String jsonFieldName = fieldNameMapping.get(solrFieldName);
@ -106,8 +107,8 @@ public class SolrResultsParser {
/**
* Find a single value in the document
*/
private String parseSingleValue(SolrDocument doc, String key) {
Object rawValue = getFirstValue(doc.get(key));
private String parseSingleValue(SearchResultDocument doc, String key) {
Object rawValue = doc.getFirstValue(key);
if (rawValue == null) {
return "";
@ -118,20 +119,4 @@ public class SolrResultsParser {
return String.valueOf(rawValue);
}
/**
* The result might be a list. If so, get the first element.
*/
private Object getFirstValue(Object rawValue) {
if (rawValue instanceof List<?>) {
List<?> list = (List<?>) rawValue;
if (list.isEmpty()) {
return null;
} else {
return list.get(0);
}
} else {
return rawValue;
}
}
}

27
webapp/test/edu/cornell/mannlib/vitro/webapp/controller/grefine/JSONReconcileServletTest.java
View file

@ -10,18 +10,19 @@ import java.util.regex.PatternSyntaxException;
import javax.servlet.ServletException;
import org.apache.solr.client.solrj.SolrQuery;
import org.json.JSONException;
import org.json.JSONObject;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletContextStub;
import stubs.javax.servlet.http.HttpServletRequestStub;
import stubs.javax.servlet.http.HttpServletResponseStub;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
/**
* Tests on various methods in the class.
@ -36,6 +37,8 @@ public class JSONReconcileServletTest extends AbstractTestClass {
@Before
public void setup() throws Exception {
ApplicationStub.setup(new ServletContextStub(), new SearchEngineStub());
request = new HttpServletRequestStub();
request.setRequestUrl(new URL("http://vivo.this.that/reconcile"));
request.setMethod("POST");
@ -78,23 +81,17 @@ public class JSONReconcileServletTest extends AbstractTestClass {
propertiesList.add(properties);
// test getQuery
SolrQuery solrQuery = reconcile.getQuery(nameStr, nameType, rowNum, propertiesList);
SearchQuery searchQuery = reconcile.getQuery(nameStr, nameType, rowNum, propertiesList);
String messagePrefix = "Query should contain the text: ";
testAssertTrue(messagePrefix + orgStr, orgStr, solrQuery.toString());
testAssertTrue(messagePrefix + nameStr, nameStr, solrQuery.toString());
testAssertTrue(messagePrefix + orgType, orgType, solrQuery.toString());
testAssertTrue(messagePrefix + nameType, orgType, solrQuery.toString());
testAssertTrue(messagePrefix + orgStr, orgStr, searchQuery.toString());
testAssertTrue(messagePrefix + nameStr, nameStr, searchQuery.toString());
testAssertTrue(messagePrefix + orgType, orgType, searchQuery.toString());
testAssertTrue(messagePrefix + nameType, orgType, searchQuery.toString());
}
private void testAssertTrue(String message, String inputStr, String resultStr) {
try {
String modStr = null;
if (inputStr.contains(":") && inputStr.contains("/")) {
modStr = inputStr.replaceAll(":", "%3A").replaceAll("/", "%2F");
} else {
modStr = inputStr;
}
Pattern regex = Pattern.compile(modStr, Pattern.CASE_INSENSITIVE);
Pattern regex = Pattern.compile(inputStr, Pattern.CASE_INSENSITIVE);
Matcher regexMatcher = regex.matcher(resultStr);
Assert.assertTrue(message, regexMatcher.find());
} catch (PatternSyntaxException e) {

10
webapp/test/edu/cornell/mannlib/vitro/webapp/controller/json/JsonServletTest.java
View file

@ -20,16 +20,16 @@ import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.dao.VClassDaoStub;
import stubs.edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactoryStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletConfigStub;
import stubs.javax.servlet.ServletContextStub;
import stubs.javax.servlet.http.HttpServletRequestStub;
import stubs.javax.servlet.http.HttpServletResponseStub;
import stubs.javax.servlet.http.HttpSessionStub;
import stubs.org.apache.solr.client.solrj.SolrServerStub;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/**
* TODO
@ -85,7 +85,7 @@ public class JsonServletTest extends AbstractTestClass {
private WebappDaoFactoryStub wadf;
private VClassDaoStub vcDao;
private SolrServerStub solr;
private SearchEngineStub solr;
@Before
public void setup() throws ServletException {
@ -112,8 +112,8 @@ public class JsonServletTest extends AbstractTestClass {
vcDao = new VClassDaoStub();
wadf.setVClassDao(vcDao);
solr = new SolrServerStub();
ctx.setAttribute(SolrSetup.SOLR_SERVER, solr);
solr = new SearchEngineStub();
ApplicationStub.setup(new ServletContextStub(), solr);
}
@Test

327
webapp/test/edu/cornell/mannlib/vitro/webapp/search/solr/SolrQueryTest.java
View file

@ -1,327 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.CoreContainer;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.xml.sax.SAXException;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ResIterator;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.util.FileManager;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactoryJena;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.model.RDFServiceModel;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
/**
* This tests what results will be returned for queries to the
* Solr service.
*
* The test will setup a temporary Solr service, setup a temporary RDF store, load the
* store with some individuals, and then index the individuals in the store.
*
* Once this is done a set of test queries will be run against that service.
*
* This code is based on a similar test from the CUL Discovery and Access
* integration layer.
*
* All RDF/XML files in webapp/test/testontologies/SolrQueryTestRDF will be
* loaded into the model.
*/
//ignored for now since the solr directory isn't yet found when running from vivo.
@Ignore
public class SolrQueryTest extends AbstractTestClass {
/** Key of system property for build directory. */
final static String buildDirSystemPropertyKey = "vitro.build.dir";
/** Solr index to run test queries on. */
static SolrServer solr = null;
/** Container for solr */
static CoreContainer coreContainer = null;
/** Folder to store the temporary Solr index. */
public static TemporaryFolder solrIndexFolder = null;
/**
* This test needs to load RDF data to use as a
* source of individuals to to build documents.
* This is relative to the build directory.
*/
final static String testRDFDir =
"/webapp/test/testontologies/SolrQueryTestRDF";
@Before
public void setup() throws Exception{
//solr makes a lot of output
suppressSysout();
suppressSyserr();
File buildDir = findBuildDir();
solrIndexFolder = new TemporaryFolder();
solrIndexFolder.create();
File tempSolrBase = solrIndexFolder.newFolder("tempSolrBase");
FileUtils.copyDirectory(getSolrTemplateDir(buildDir), tempSolrBase );
solr = setupSolr( tempSolrBase );
indexRdf( loadTestRDF( buildDir ) );
}
@After
public void takedown() throws Exception{
if( coreContainer != null )
coreContainer.shutdown();
restoreOutputStreams();
if( solrIndexFolder != null )
solrIndexFolder.delete();
}
/**
* This will return the directory to use as the Solr
* home template directory.
*
* Throws an exception if the directory is not found.
* @param buildDir - must not be null, must be the base of the build.
*/
private File getSolrTemplateDir(File buildDir) throws Exception {
if(buildDir == null || !buildDir.exists() )
throw new Exception("buildDir must not be null");
String solrTemplateDirName = buildDir.getAbsolutePath() + "/solr/homeDirectoryTemplate";
File solrTemplateDir = new File(solrTemplateDirName);
if( solrTemplateDir == null || ! solrTemplateDir.exists())
throw new Exception("Solr home directory template " +
"was not found at " + solrTemplateDirName);
else
return solrTemplateDir;
}
protected SolrServer setupSolr(File solrBase)
throws ParserConfigurationException, IOException, SAXException{
System.setProperty("solr.solr.home", solrBase.getAbsolutePath());
CoreContainer.Initializer initializer = new CoreContainer.Initializer();
coreContainer = initializer.initialize();
return new EmbeddedSolrServer(coreContainer, "");
}
private OntModel loadTestRDF(File buildDir) throws Exception {
String dirname = buildDir.getAbsolutePath() + testRDFDir;
File rdfDir = new File( dirname );
assertNotNull("could not find dir " + dirname , rdfDir);
assertTrue(dirname + " must be a directory." ,rdfDir.isDirectory());
//load all files in test dir.
File[] files = rdfDir.listFiles();
assertNotNull("no test RDF files found",files);
assertTrue("no test RDF files found", files.length > 0 );
OntModel model = ModelFactory.createOntologyModel();
for (File file : files ){
InputStream in = FileManager.get().open( file.getAbsolutePath() );
assertNotNull("Could not load file " + file.getAbsolutePath(), in );
try{
if( file.getName().endsWith(".rdf") ){
model.read(in,null);
}else if( file.getName().endsWith(".n3")){
model.read(in,null,"N3");
}else if( file.getName().endsWith(".nt")){
model.read(in,null,"N-TRIPLE");
}else if( file.getName().endsWith(".ttl")){
model.read(in,null,"TURTLE");
}else{
throw new Exception("Format unknown for file name " + file.getName());
}
}catch(Throwable th){
throw new Exception( "Could not load RDF file "
+ file.getAbsolutePath() , th);
}
}
return model ;
}
/**
* Find the base of the build directories.
* @return This method will return a non-null file to use
* or it will throw an exception if none can be found.
* @throws Exception
*/
private File findBuildDir() throws Exception {
//First try to find the base directory
//of the build in the system properties.
String buildDirName = System.getProperty(buildDirSystemPropertyKey);
if( buildDirName != null ){
File buildDir = new File(buildDirName);
if( buildDir.exists() && buildDir.isDirectory() ){
return buildDir;
}
}
//If there is no system property try to
//guess the location based on the working directory
File f = null;
String[] fallBackBases = {"","../","../../"};
List<String> attempted = new ArrayList<String>();
for( String base: fallBackBases){
String attemptedDir =base + "solr/homeDirectoryTemplate";
f = new File( attemptedDir );
attempted.add( System.getProperty("user.dir") + '/' + attemptedDir );
if( f != null && f.exists() && f.isDirectory() ){
f = new File( base );
break;
}else{
f = null;
}
}
if( f == null ){
throw new Exception(
"Could not find the base of the " +
"build directory for the project, " +
"checked the system property " + buildDirSystemPropertyKey
+ " and checked in these locations: \n" +
StringUtils.join(attempted,"\n "));
}else{
return f;
}
}
/** Query the RDF, build Solr Documents from results, load them to Solr. */
private void indexRdf(OntModel model) throws Exception {
RDFServiceModel rdfService = new RDFServiceModel(model);
IndividualToSolrDocument r2d =
SolrSetup.setupTransltion(
model,
model,
new RDFServiceFactorySingle(rdfService),
null, null);
WebappDaoFactory wdf = new WebappDaoFactoryJena(model);
for( String uri: getURISToIndex( model ) ){
SolrInputDocument doc;
try {
doc = r2d.translate( wdf.getIndividualDao().getIndividualByURI(uri));
} catch (Exception e) {
throw new Exception("Failed on building document for uri:" + uri, e);
}
try {
solr.add( doc );
} catch (Exception e) {
throw new Exception("Failed adding doc to solr for uri:" + uri, e);
}
}
solr.commit();
}
private List<String> getURISToIndex(Model model) {
//just return all the URIs in the subject position
List<String> uris = new LinkedList<String>();
ResIterator it = model.listSubjects();
while(it.hasNext() ){
Resource res = it.nextResource();
if( res != null && res.isURIResource() ){
uris.add( res.getURI() );
}
}
return uris;
}
/**
* Test that a document with the given URIs are in the results for the query.
* @throws SolrServerException */
void testQueryGetsDocs(String errmsg, SolrQuery query, String[] expectedUris) throws SolrServerException{
assertNotNull(errmsg + " but query was null", query);
assertNotNull(errmsg + " but expected URIs was null", expectedUris );
QueryResponse resp = solr.query(query);
if( resp == null )
fail( errmsg + " but Could not get a solr response");
Set<String> uris = new HashSet<String>(Arrays.asList( expectedUris ));
for( SolrDocument doc : resp.getResults()){
assertNotNull(errmsg + ": solr doc was null", doc);
String uri = (String) doc.getFirstValue( VitroSearchTermNames.URI );
assertNotNull(errmsg+": no URI field in solr doc" , uri);
uris.remove( uri );
}
if( uris.size() > 0){
String errorMsg =
"\nThe query '"+ query + "' was expected " +
"to return the following URIs but did not:";
for( String uri : uris){
errorMsg= errorMsg+"\n" + uri;
}
fail( errmsg + errorMsg);
}
}
@Test
public void testSolrWasStarted() throws SolrServerException, IOException {
assertNotNull( solr );
solr.ping();//this will throw an exception of the server is not reachable
}
@Test
public void testCorsonSearch() throws SolrServerException{
/* make sure that we have the document in the index before we do anything */
SolrQuery query = new SolrQuery().setQuery("corson");
testQueryGetsDocs("Expect to find a doc when searching for 'corson'",
query,new String[]{ "http://vivo.cornell.edu/individual/individual22972" } ) ;
}
@Test
public void testFormFeed() throws SolrServerException{
/* make sure that we have the document in the index before we do anything */
SolrQuery query = new SolrQuery().setQuery("vivo15");
testQueryGetsDocs("Expect to find a doc when searching for 'vivo15'",
query,new String[]{ "http://vivo.cornell.edu/individual/individualIssueVivo15" } ) ;
}
}

16
webapp/test/edu/cornell/mannlib/vitro/webapp/search/solr/ThumbnailImageURLTest.java
View file

@ -7,19 +7,24 @@ package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.io.InputStream;
import org.apache.log4j.Level;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletContextStub;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.model.RDFServiceModel;
@ -37,6 +42,7 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
@Before
public void setUp() throws Exception {
setLoggerLevel(RDFDefaultErrorHandler.class, Level.OFF);
ApplicationStub.setup(new ServletContextStub(), new SearchEngineStub());
Model model = ModelFactory.createDefaultModel();
InputStream in = ThumbnailImageURLTest.class.getResourceAsStream("testPerson.n3");
@ -52,7 +58,7 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
*/
@Test
public void testThumbnailFieldCreatedInSolrDoc() {
SolrInputDocument doc = new SolrInputDocument();
SearchInputDocument doc = ApplicationUtils.instance().getSearchEngine().createInputDocument();
ThumbnailImageURL testMe = new ThumbnailImageURL( testRDF );
Individual ind = new IndividualImpl();
ind.setURI(personsURI);
@ -66,11 +72,11 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
//make sure that a Solr document field got created for the thumbnail image
SolrInputField thumbnailField = doc.getField( VitroSearchTermNames.THUMBNAIL_URL );
SearchInputField thumbnailField = doc.getField( VitroSearchTermNames.THUMBNAIL_URL );
Assert.assertNotNull(thumbnailField);
Assert.assertNotNull( thumbnailField.getValues() );
Assert.assertEquals(1, thumbnailField.getValueCount());
Assert.assertEquals(1, thumbnailField.getValues().size());
Assert.assertEquals("http://vivo.cornell.edu/individual/n54945", thumbnailField.getFirstValue());
}

14
webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java
View file

@ -3,7 +3,6 @@
package stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@ -14,6 +13,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineExcepti
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchInputDocument;
/**
* TODO
@ -53,6 +53,11 @@ public class SearchEngineStub implements SearchEngine {
}
}
@Override
public SearchInputDocument createInputDocument() {
return new BaseSearchInputDocument();
}
// ----------------------------------------------------------------------
// Un-implemented methods
@ -80,13 +85,6 @@ public class SearchEngineStub implements SearchEngine {
throw new RuntimeException("SearchEngineStub.ping() not implemented.");
}
@Override
public SearchInputDocument createInputDocument() {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.createInputDocument() not implemented.");
}
@Override
public void add(SearchInputDocument... docs) throws SearchEngineException {
// TODO Auto-generated method stub

46
webapp/test/stubs/org/apache/solr/client/solrj/SolrServerStub.java
View file

@ -1,46 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package stubs.org.apache.solr.client.solrj;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.util.NamedList;
/**
* TODO
*/
public class SolrServerStub extends SolrServer {
private static final Log log = LogFactory.getLog(SolrServerStub.class);
// ----------------------------------------------------------------------
// Stub infrastructure
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
// Stub methods
// ----------------------------------------------------------------------
/*
* (non-Javadoc)
*
* @see
* org.apache.solr.client.solrj.SolrServer#request(org.apache.solr.client
* .solrj.SolrRequest)
*/
@Override
public NamedList<Object> request(SolrRequest request)
throws SolrServerException, IOException {
// TODO not really an implementation.
return new NamedList<Object>();
}
// ----------------------------------------------------------------------
// Un-implemented methods
// ----------------------------------------------------------------------
}