litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

VIVO-742 Change client code to use SearchEngine interface.

Browse source 
Removed SolrQueryTest until I can see how to update it. Everything compiles and tests run, but haven't tried running VIVO yet.
This commit is contained in:
Jim Blake 2014-04-22 13:49:27 -04:00
parent 823848123f
commit 90886c564e
38 changed files with 398 additions and 1109 deletions
Download patch file Download diff file Expand all files Collapse all files

36
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/IndividualListRdfController.java
View file

@ -8,11 +8,6 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -21,8 +16,13 @@ import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory; import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.vocabulary.RDF; import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
public class IndividualListRdfController extends VitroHttpServlet { public class IndividualListRdfController extends VitroHttpServlet {
@ -31,21 +31,22 @@ public class IndividualListRdfController extends VitroHttpServlet {
public static final int ENTITY_LIST_CONTROLLER_MAX_RESULTS = 30000; public static final int ENTITY_LIST_CONTROLLER_MAX_RESULTS = 30000;
public void doGet (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException { @Override
public void doGet (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException {
// Make the query // Make the query
String vclassUri = req.getParameter("vclass"); String vclassUri = req.getParameter("vclass");
String queryStr = VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\""; String queryStr = VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\"";
SolrQuery query = new SolrQuery(queryStr); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryStr);
query.setStart(0) query.setStart(0)
.setRows(ENTITY_LIST_CONTROLLER_MAX_RESULTS) .setRows(ENTITY_LIST_CONTROLLER_MAX_RESULTS)
.setFields(VitroSearchTermNames.URI); .addFields(VitroSearchTermNames.URI);
// For now, we're only displaying the url, so no need to sort. // For now, we're only displaying the url, so no need to sort.
//.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED); //.addSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED);
// Execute the query // Execute the query
SolrServer solr = SolrSetup.getSolrServer(getServletContext()); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
QueryResponse response = null; SearchResponse response = null;
try { try {
response = solr.query(query); response = solr.query(query);
@ -57,17 +58,17 @@ public class IndividualListRdfController extends VitroHttpServlet {
throw new ServletException("Could not run search in IndividualListRdfController"); throw new ServletException("Could not run search in IndividualListRdfController");
} }
SolrDocumentList docs = response.getResults(); SearchResultDocumentList docs = response.getResults();
if (docs == null) { if (docs == null) {
throw new ServletException("Could not run search in IndividualListRdfController"); throw new ServletException("Could not run search in IndividualListRdfController");
} }
Model model = ModelFactory.createDefaultModel(); Model model = ModelFactory.createDefaultModel();
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
Resource resource = ResourceFactory.createResource(uri); Resource resource = ResourceFactory.createResource(uri);
RDFNode node = (RDFNode) ResourceFactory.createResource(vclassUri); RDFNode node = ResourceFactory.createResource(vclassUri);
model.add(resource, RDF.type, node); model.add(resource, RDF.type, node);
} }
@ -75,7 +76,8 @@ public class IndividualListRdfController extends VitroHttpServlet {
model.write(res.getOutputStream(), "RDF/XML"); model.write(res.getOutputStream(), "RDF/XML");
} }
public void doPost (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException{ @Override
public void doPost (HttpServletRequest req, HttpServletResponse res) throws IOException, ServletException{
doGet(req,res); doGet(req,res);
} }

36
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/accounts/admin/ajax/ProfileAutoCompleter.java
View file

@ -16,17 +16,11 @@ import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.json.JSONException; import org.json.JSONException;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
@ -39,11 +33,16 @@ import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.query.Syntax; import com.hp.hpl.jena.query.Syntax;
import com.hp.hpl.jena.rdf.model.Literal; import com.hp.hpl.jena.rdf.model.Literal;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.SelfEditingConfiguration; import edu.cornell.mannlib.vitro.webapp.beans.SelfEditingConfiguration;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder; import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords; import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap; import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils; import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
@ -121,8 +120,8 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
} }
try { try {
SolrQuery query = buildSolrQuery(); SearchQuery query = buildSearchQuery();
QueryResponse queryResponse = executeSolrQuery(query); SearchResponse queryResponse = executeSearchQuery(query);
List<Map<String, String>> maps = SolrQueryUtils List<Map<String, String>> maps = SolrQueryUtils
.parseAndFilterResponse(queryResponse, RESPONSE_FIELDS, .parseAndFilterResponse(queryResponse, RESPONSE_FIELDS,
@ -133,17 +132,17 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
String response = assembleJsonResponse(maps); String response = assembleJsonResponse(maps);
log.debug(response); log.debug(response);
return response; return response;
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.error("Failed to get basic profile info", e); log.error("Failed to get basic profile info", e);
return EMPTY_RESPONSE; return EMPTY_RESPONSE;
} }
} }
private SolrQuery buildSolrQuery() { private SearchQuery buildSearchQuery() {
SolrQuery q = new SolrQuery(); SearchQuery q = ApplicationUtils.instance().getSearchEngine().createQuery();
q.setFields(NAME_RAW, URI); q.addFields(NAME_RAW, URI);
q.setSortField(NAME_LOWERCASE_SINGLE_VALUED, ORDER.asc); q.addSortField(NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
q.setFilterQueries(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE, q.addFilterQuery(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE,
profileTypes, OR)); profileTypes, OR));
q.setStart(0); q.setStart(0);
q.setRows(10000); q.setRows(10000);
@ -151,10 +150,9 @@ class ProfileAutoCompleter extends AbstractAjaxResponder implements
return q; return q;
} }
private QueryResponse executeSolrQuery(SolrQuery query) private SearchResponse executeSearchQuery(SearchQuery query)
throws SolrServerException { throws SearchEngineException {
ServletContext ctx = servlet.getServletContext(); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SolrServer solr = SolrSetup.getSolrServer(ctx);
return solr.query(query); return solr.query(query);
} }

34
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/accounts/manageproxies/ajax/BasicProfilesGetter.java
View file

@ -1,7 +1,6 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */ /* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.controller.accounts.manageproxies.ajax; package edu.cornell.mannlib.vitro.webapp.controller.accounts.manageproxies.ajax;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.AC_NAME_STEMMED; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.AC_NAME_STEMMED;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED;
import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_RAW; import static edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames.NAME_RAW;
@ -14,23 +13,22 @@ import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.ORDER;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.json.JSONException; import org.json.JSONException;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder; import edu.cornell.mannlib.vitro.webapp.controller.ajax.AbstractAjaxResponder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords; import edu.cornell.mannlib.vitro.webapp.utils.solr.AutoCompleteWords;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap; import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils; import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
@ -88,10 +86,9 @@ public class BasicProfilesGetter extends AbstractAjaxResponder {
} }
try { try {
ServletContext ctx = servlet.getServletContext(); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SolrServer solr = SolrSetup.getSolrServer(ctx); SearchQuery query = buildSearchQuery();
SolrQuery query = buildSolrQuery(); SearchResponse queryResponse = solr.query(query);
QueryResponse queryResponse = solr.query(query);
List<Map<String, String>> parsed = SolrQueryUtils List<Map<String, String>> parsed = SolrQueryUtils
.parseResponse(queryResponse, RESPONSE_FIELDS); .parseResponse(queryResponse, RESPONSE_FIELDS);
@ -99,18 +96,17 @@ public class BasicProfilesGetter extends AbstractAjaxResponder {
String response = assembleJsonResponse(parsed); String response = assembleJsonResponse(parsed);
log.debug(response); log.debug(response);
return response; return response;
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.error("Failed to get basic profile info", e); log.error("Failed to get basic profile info", e);
return EMPTY_RESPONSE; return EMPTY_RESPONSE;
} }
} }
private SolrQuery buildSolrQuery() { private SearchQuery buildSearchQuery() {
SolrQuery q = new SolrQuery(); SearchQuery q = ApplicationUtils.instance().getSearchEngine().createQuery();
q.setFields(NAME_RAW, URI); q.addFields(NAME_RAW, URI);
q.setSortField(NAME_LOWERCASE_SINGLE_VALUED, ORDER.asc); q.addSortField(NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
q.setFilterQueries(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE, q.addFilterQuery(SolrQueryUtils.assembleConjunctiveQuery(RDFTYPE, profileTypes, OR));
profileTypes, OR));
q.setStart(0); q.setStart(0);
q.setRows(30); q.setRows(30);
q.setQuery(searchWords.assembleQuery(NAME_UNSTEMMED, AC_NAME_STEMMED)); q.setQuery(searchWords.assembleQuery(NAME_UNSTEMMED, AC_NAME_STEMMED));

25
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java
View file

@ -8,12 +8,9 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrServerException;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -24,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Res
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.individuallist.IndividualListResults; import edu.cornell.mannlib.vitro.webapp.controller.individuallist.IndividualListResults;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils; import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.ListedIndividual;
@ -94,8 +92,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
vclass.getURI(), vclass.getURI(),
page, page,
alpha, alpha,
vreq.getWebappDaoFactory().getIndividualDao(), vreq.getWebappDaoFactory().getIndividualDao());
getServletContext());
body.putAll(vcResults.asFreemarkerMap()); body.putAll(vcResults.asFreemarkerMap());
List<Individual> inds = vcResults.getEntities(); List<Individual> inds = vcResults.getEntities();
@ -149,13 +146,13 @@ public class IndividualListController extends FreemarkerHttpServlet {
return SolrQueryUtils.getPageParameter(request); return SolrQueryUtils.getPageParameter(request);
} }
public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context) public static IndividualListResults getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao)
throws SearchException{ throws SearchException{
try{ try{
List<String> classUris = Collections.singletonList(vclassURI); List<String> classUris = Collections.singletonList(vclassURI);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, context, indDao); IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE, indDao);
return getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha); return getResultsForVClassQuery(results, page, INDIVIDUALS_PER_PAGE, alpha);
} catch (SolrServerException e) { } catch (SearchEngineException e) {
String msg = "An error occurred retrieving results for vclass query"; String msg = "An error occurred retrieving results for vclass query";
log.error(msg, e); log.error(msg, e);
// Throw this up to processRequest, so the template gets the error message. // Throw this up to processRequest, so the template gets the error message.
@ -166,9 +163,9 @@ public class IndividualListController extends FreemarkerHttpServlet {
} }
} }
public static IndividualListResults getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao, ServletContext context) { public static IndividualListResults getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao) {
try{ try{
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, context, indDao); IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteVClassQuery(vclassURIs, alpha, page, pageSize, indDao);
return getResultsForVClassQuery(results, page, pageSize, alpha); return getResultsForVClassQuery(results, page, pageSize, alpha);
} catch(Throwable th) { } catch(Throwable th) {
log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th); log.error("Error retrieving individuals corresponding to intersection multiple classes." + vclassURIs.toString(), th);
@ -176,10 +173,10 @@ public class IndividualListController extends FreemarkerHttpServlet {
} }
} }
public static IndividualListResults getRandomResultsForVClass(String vclassURI, int page, int pageSize, IndividualDao indDao, ServletContext context) { public static IndividualListResults getRandomResultsForVClass(String vclassURI, int page, int pageSize, IndividualDao indDao) {
try{ try{
List<String> classUris = Collections.singletonList(vclassURI); List<String> classUris = Collections.singletonList(vclassURI);
IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteRandomVClassQuery(classUris, page, pageSize, context, indDao); IndividualListQueryResults results = SolrQueryUtils.buildAndExecuteRandomVClassQuery(classUris, page, pageSize, indDao);
return getResultsForVClassQuery(results, page, pageSize, ""); return getResultsForVClassQuery(results, page, pageSize, "");
} catch(Throwable th) { } catch(Throwable th) {
log.error("An error occurred retrieving random results for vclass query", th); log.error("An error occurred retrieving random results for vclass query", th);
@ -189,8 +186,8 @@ public class IndividualListController extends FreemarkerHttpServlet {
//TODO: Get rid of this method and utilize SolrQueryUtils - currently appears to be referenced //TODO: Get rid of this method and utilize SolrQueryUtils - currently appears to be referenced
//only within DataGetterUtils //only within DataGetterUtils
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) { public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao) {
return SolrQueryUtils.getIndividualCount(vclassUris, indDao, context); return SolrQueryUtils.getIndividualCount(vclassUris, indDao);
} }
private static IndividualListResults getResultsForVClassQuery(IndividualListQueryResults results, int page, int pageSize, String alpha) { private static IndividualListResults getResultsForVClassQuery(IndividualListQueryResults results, int page, int pageSize, String alpha) {

32
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListQueryResults.java
View file

@ -5,21 +5,19 @@ package edu.cornell.mannlib.vitro.webapp.controller.freemarker;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/** /**
* Holds the Individuals that were found in a Solr search query. * Holds the Individuals that were found in a Solr search query.
@ -37,12 +35,12 @@ public class IndividualListQueryResults {
// Convenience method // Convenience method
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
public static IndividualListQueryResults runQuery(SolrQuery query, public static IndividualListQueryResults runQuery(SearchQuery query,
IndividualDao indDao, ServletContext context) IndividualDao indDao)
throws SolrServerException { throws SearchEngineException {
SolrServer solr = SolrSetup.getSolrServer(context); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
QueryResponse response = null; SearchResponse response = null;
response = solr.query(query); response = solr.query(query);
if (response == null) { if (response == null) {
@ -50,7 +48,7 @@ public class IndividualListQueryResults {
return EMPTY_RESULT; return EMPTY_RESULT;
} }
SolrDocumentList docs = response.getResults(); SearchResultDocumentList docs = response.getResults();
if (docs == null) { if (docs == null) {
log.debug("results from search query response was null"); log.debug("results from search query response was null");
return EMPTY_RESULT; return EMPTY_RESULT;
@ -61,8 +59,8 @@ public class IndividualListQueryResults {
log.debug("Number of search results: " + hitCount); log.debug("Number of search results: " + hitCount);
List<Individual> individuals = new ArrayList<Individual>(docs.size()); List<Individual> individuals = new ArrayList<Individual>(docs.size());
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual individual = indDao.getIndividualByURI(uri); Individual individual = indDao.getIndividualByURI(uri);
if (individual == null) { if (individual == null) {
log.debug("No individual for search document with uri = " + uri); log.debug("No individual for search document with uri = " + uri);

67
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/grefine/JSONReconcileServlet.java
View file

@ -18,23 +18,21 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.controller.VitroHttpServlet; import edu.cornell.mannlib.vitro.webapp.controller.VitroHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
/** /**
* This servlet is for servicing JSON requests from Google Refine's * This servlet is for servicing JSON requests from Google Refine's
@ -125,7 +123,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
try { try {
for (int i = 0; i < queries.size(); i++) { for (int i = 0; i < queries.size(); i++) {
String queryStr = (String) queries.get(i); String queryStr = queries.get(i);
JSONObject json = new JSONObject(queryStr); JSONObject json = new JSONObject(queryStr);
if (json.has("query")) { // single query if (json.has("query")) { // single query
@ -138,7 +136,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
} else { // multiple queries } else { // multiple queries
for (Iterator<String> iter = json.keys(); iter.hasNext();) { for (Iterator<String> iter = json.keys(); iter.hasNext();) {
ArrayList<JSONObject> jsonList = new ArrayList<JSONObject>(); ArrayList<JSONObject> jsonList = new ArrayList<JSONObject>();
String key = (String) iter.next(); String key = iter.next();
Object obj = json.get(key); Object obj = json.get(key);
JSONObject jsonLvl2 = (JSONObject) obj; JSONObject jsonLvl2 = (JSONObject) obj;
if (jsonLvl2.has("query")) { if (jsonLvl2.has("query")) {
@ -234,7 +232,7 @@ public class JSONReconcileServlet extends VitroHttpServlet {
for (Map.Entry<String, JSONObject> entry : currMap.entrySet()) { for (Map.Entry<String, JSONObject> entry : currMap.entrySet()) {
JSONObject resultAllJson = new JSONObject(); JSONObject resultAllJson = new JSONObject();
String key = entry.getKey(); String key = entry.getKey();
JSONObject json = (JSONObject) entry.getValue(); JSONObject json = entry.getValue();
String queryVal = json.getString("query"); String queryVal = json.getString("query");
// System.out.println("query: " + json.toString()); // System.out.println("query: " + json.toString());
@ -274,16 +272,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
JSONArray resultJsonArr = new JSONArray(); JSONArray resultJsonArr = new JSONArray();
// Solr // Solr
SolrQuery query = getQuery(queryVal, searchType, limit, propertiesList); SearchQuery query = getQuery(queryVal, searchType, limit, propertiesList);
QueryResponse queryResponse = null; SearchResponse queryResponse = null;
if (query != null) { if (query != null) {
SolrServer solr = SolrSetup.getSolrServer(getServletContext()); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
queryResponse = solr.query(query); queryResponse = solr.query(query);
} else { } else {
log.error("Query for a search was null"); log.error("Query for a search was null");
} }
SolrDocumentList docs = null; SearchResultDocumentList docs = null;
if (queryResponse != null) { if (queryResponse != null) {
docs = queryResponse.getResults(); docs = queryResponse.getResults();
} else { } else {
@ -293,29 +291,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
if (docs != null) { if (docs != null) {
List<SearchResult> results = new ArrayList<SearchResult>(); List<SearchResult> results = new ArrayList<SearchResult>();
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
try { try {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
// RY 7/1/2011 String name = doc.getStringValue(VitroSearchTermNames.NAME_RAW);
// Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() returns a list.
// Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() could return a list
// But in fact: I'm no longer seeing any lists returned for individuals with multiple labels. Not sure
// if this is new behavior or what. ???
Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
String name = null;
if (nameRaw instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> nameRawList = (List<String>) nameRaw;
name = nameRawList.get(0);
} else {
name = (String) nameRaw;
}
SearchResult result = new SearchResult(name, uri); SearchResult result = new SearchResult(name, uri);
// populate result for Google Refine // populate result for Google Refine
JSONObject resultJson = new JSONObject(); JSONObject resultJson = new JSONObject();
resultJson.put("score", doc.getFieldValue("score")); resultJson.put("score", doc.getFirstValue("score"));
String modUri = result.getUri().replace("#", "%23"); String modUri = result.getUri().replace("#", "%23");
resultJson.put("id", modUri); resultJson.put("id", modUri);
resultJson.put("name", result.getLabel()); resultJson.put("name", result.getLabel());
@ -361,16 +346,16 @@ public class JSONReconcileServlet extends VitroHttpServlet {
log.error("JSONException: " + ex); log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet JSONException: " throw new ServletException("JSONReconcileServlet JSONException: "
+ ex); + ex);
} catch (SolrServerException ex) { } catch (SearchEngineException ex) {
log.error("JSONException: " + ex); log.error("JSONException: " + ex);
throw new ServletException("JSONReconcileServlet SolrServerException: " throw new ServletException("JSONReconcileServlet SearchEngineException: "
+ ex); + ex);
} }
return qJson; return qJson;
} }
protected SolrQuery getQuery(String queryStr, String searchType, int limit, ArrayList<String[]> propertiesList) { protected SearchQuery getQuery(String queryStr, String searchType, int limit, ArrayList<String[]> propertiesList) {
if ( queryStr == null) { if ( queryStr == null) {
log.error("There was no parameter '"+ PARAM_QUERY log.error("There was no parameter '"+ PARAM_QUERY
@ -383,10 +368,10 @@ public class JSONReconcileServlet extends VitroHttpServlet {
} }
/// original /// original
///SolrQuery query = new SolrQuery(); ///SearchQuery query = new SearchQuery();
/// test /// test
SolrQuery query = new SolrQuery(queryStr.toLowerCase()); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryStr.toLowerCase());
// original code: // original code:
// query.setStart(0).setRows(DEFAULT_MAX_HIT_COUNT); // query.setStart(0).setRows(DEFAULT_MAX_HIT_COUNT);
@ -403,17 +388,17 @@ public class JSONReconcileServlet extends VitroHttpServlet {
} }
// Added score to original code: // Added score to original code:
query.setFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, "*", "score"); // fields to retrieve query.addFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, "*", "score"); // fields to retrieve
// if propertiesList has elements, add extra queries to query // if propertiesList has elements, add extra queries to query
Iterator<String[]> it = propertiesList.iterator(); Iterator<String[]> it = propertiesList.iterator();
while (it.hasNext()) { while (it.hasNext()) {
String[] pvPair = it.next(); String[] pvPair = it.next();
query.addFilterQuery(tokenizeNameQuery(pvPair[1]), VitroSearchTermNames.RDFTYPE + ":\"" + pvPair[0] + "\""); query.addFilterQueries(tokenizeNameQuery(pvPair[1]), VitroSearchTermNames.RDFTYPE + ":\"" + pvPair[0] + "\"");
} }
// Can't sort on multivalued field, so we sort the results in Java when we get them. // Can't sort on multivalued field, so we sort the results in Java when we get them.
// query.setSortField(VitroSearchTermNames.NAME_LOWERCASE, SolrQuery.ORDER.asc); // query.addSortField(VitroSearchTermNames.NAME_LOWERCASE, Order.ASC);
return query; return query;
} }

6
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/json/JsonServlet.java
View file

@ -113,8 +113,7 @@ public class JsonServlet extends VitroHttpServlet {
vclassURIs, vclassURIs,
page, INDIVIDUALS_PER_PAGE, page, INDIVIDUALS_PER_PAGE,
alpha, alpha,
vreq.getWebappDaoFactory().getIndividualDao(), vreq.getWebappDaoFactory().getIndividualDao());
context);
} catch(Exception ex) { } catch(Exception ex) {
log.error("Error in retrieval of search results for VClass " + vclassURIs.toString(), ex); log.error("Error in retrieval of search results for VClass " + vclassURIs.toString(), ex);
return IndividualListResults.EMPTY; return IndividualListResults.EMPTY;
@ -149,8 +148,7 @@ public class JsonServlet extends VitroHttpServlet {
vclassURI, vclassURI,
page, page,
pageSize, pageSize,
vreq.getWebappDaoFactory().getIndividualDao(), vreq.getWebappDaoFactory().getIndividualDao());
context);
} catch(Exception ex) { } catch(Exception ex) {
log.error("Error in retrieval of search results for VClass " + vclassURI, ex); log.error("Error in retrieval of search results for VClass " + vclassURI, ex);
return IndividualListResults.EMPTY; return IndividualListResults.EMPTY;

66
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/VClassGroupCache.java
View file

@ -14,12 +14,6 @@ import javax.servlet.http.HttpServletRequest;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.listeners.StatementListener; import com.hp.hpl.jena.rdf.listeners.StatementListener;
@ -30,6 +24,7 @@ import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF; import com.hp.hpl.jena.vocabulary.RDF;
import com.hp.hpl.jena.vocabulary.RDFS; import com.hp.hpl.jena.vocabulary.RDFS;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess; import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
@ -40,11 +35,17 @@ import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering; import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch; import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexingEventListener;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.searchindex.SearchIndexerSetup;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus; import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread; import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread;
@ -188,14 +189,14 @@ public class VClassGroupCache implements IndexingEventListener {
int attempts = 0; int attempts = 0;
int maxTries = 3; int maxTries = 3;
SolrServerException exception = null; SearchEngineException exception = null;
while( attempts < maxTries ){ while( attempts < maxTries ){
try { try {
attempts++; attempts++;
rebuildCacheUsingSolr(this); rebuildCacheUsingSolr(this);
break; break;
} catch (SolrServerException e) { } catch (SearchEngineException e) {
exception = e; exception = e;
try { Thread.sleep(250); } try { Thread.sleep(250); }
catch (InterruptedException e1) {/*ignore interrupt*/} catch (InterruptedException e1) {/*ignore interrupt*/}
@ -203,7 +204,7 @@ public class VClassGroupCache implements IndexingEventListener {
} }
if( exception != null ) if( exception != null )
log.error("Could not rebuild cache. " + exception.getRootCause().getMessage() ); log.error("Could not rebuild cache. " + exception.getCause().getMessage() );
} }
/** /**
@ -247,17 +248,13 @@ public class VClassGroupCache implements IndexingEventListener {
* *
* If ProhibitedFromSearch is not found in the context, that will be skipped. * If ProhibitedFromSearch is not found in the context, that will be skipped.
* *
* @throws SolrServerException if there are problems with the Solr server. * @throws SearchEngineException if there are problems with the Solr server.
*/ */
protected static void rebuildCacheUsingSolr( VClassGroupCache cache ) throws SolrServerException{ protected static void rebuildCacheUsingSolr( VClassGroupCache cache ) throws SearchEngineException{
long start = System.currentTimeMillis(); long start = System.currentTimeMillis();
WebappDaoFactory wdFactory = ModelAccess.on(cache.context).getWebappDaoFactory(); WebappDaoFactory wdFactory = ModelAccess.on(cache.context).getWebappDaoFactory();
SolrServer solrServer = (SolrServer)cache.context.getAttribute(SolrSetup.SOLR_SERVER); SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
if( solrServer == null){
log.error("Unable to rebuild cache: could not get solrServer from ServletContext");
return;
}
VitroFilters vFilters = VitroFilterUtils.getPublicFilter(cache.context); VitroFilters vFilters = VitroFilterUtils.getPublicFilter(cache.context);
VClassGroupDao vcgDao = new WebappDaoFactoryFiltering(wdFactory, vFilters).getVClassGroupDao(); VClassGroupDao vcgDao = new WebappDaoFactoryFiltering(wdFactory, vFilters).getVClassGroupDao();
@ -291,7 +288,7 @@ public class VClassGroupCache implements IndexingEventListener {
* Removes classes from groups that are prohibited from search. * Removes classes from groups that are prohibited from search.
*/ */
protected static void removeClassesHiddenFromSearch(List<VClassGroup> groups, ServletContext context2) { protected static void removeClassesHiddenFromSearch(List<VClassGroup> groups, ServletContext context2) {
ProhibitedFromSearch pfs = (ProhibitedFromSearch)context2.getAttribute(SolrSetup.PROHIBITED_FROM_SEARCH); ProhibitedFromSearch pfs = (ProhibitedFromSearch)context2.getAttribute(SearchIndexerSetup.PROHIBITED_FROM_SEARCH);
if(pfs==null){ if(pfs==null){
log.debug("Could not get ProhibitedFromSearch from ServletContext"); log.debug("Could not get ProhibitedFromSearch from ServletContext");
return; return;
@ -311,10 +308,10 @@ public class VClassGroupCache implements IndexingEventListener {
/** /**
* Add the Individual count to classes in groups. * Add the Individual count to classes in groups.
* @throws SolrServerException * @throws SearchEngineException
*/ */
protected static void addCountsUsingSolr(List<VClassGroup> groups, SolrServer solrServer) protected static void addCountsUsingSolr(List<VClassGroup> groups, SearchEngine solrServer)
throws SolrServerException { throws SearchEngineException {
if( groups == null || solrServer == null ) if( groups == null || solrServer == null )
return; return;
for( VClassGroup group : groups){ for( VClassGroup group : groups){
@ -322,23 +319,23 @@ public class VClassGroupCache implements IndexingEventListener {
} }
} }
protected static void addClassCountsToGroup(VClassGroup group, SolrServer solrServer) protected static void addClassCountsToGroup(VClassGroup group, SearchEngine solrServer)
throws SolrServerException { throws SearchEngineException {
if( group == null ) return; if( group == null ) return;
String groupUri = group.getURI(); String groupUri = group.getURI();
String facetOnField = VitroSearchTermNames.RDFTYPE; String facetOnField = VitroSearchTermNames.RDFTYPE;
SolrQuery query = new SolrQuery( ). SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery().
setRows(0). setRows(0).
setQuery(VitroSearchTermNames.CLASSGROUP_URI + ":" + groupUri ). setQuery(VitroSearchTermNames.CLASSGROUP_URI + ":" + groupUri ).
setFacet(true). //facet on type to get counts for classes in classgroup setFaceting(true). //facet on type to get counts for classes in classgroup
addFacetField( facetOnField ). addFacetFields( facetOnField ).
setFacetMinCount(0); setFacetMinCount(0);
log.debug("query: " + query); log.debug("query: " + query);
QueryResponse rsp = solrServer.query(query); SearchResponse rsp = solrServer.query(query);
//Get individual count //Get individual count
long individualCount = rsp.getResults().getNumFound(); long individualCount = rsp.getResults().getNumFound();
@ -346,7 +343,7 @@ public class VClassGroupCache implements IndexingEventListener {
group.setIndividualCount((int) individualCount); group.setIndividualCount((int) individualCount);
//get counts for classes //get counts for classes
FacetField ff = rsp.getFacetField( facetOnField ); SearchFacetField ff = rsp.getFacetField( facetOnField );
if( ff != null ){ if( ff != null ){
List<Count> counts = ff.getValues(); List<Count> counts = ff.getValues();
if( counts != null ){ if( counts != null ){
@ -407,7 +404,8 @@ public class VClassGroupCache implements IndexingEventListener {
this.cache = cache; this.cache = cache;
} }
public void run() { @Override
public void run() {
while (!die) { while (!die) {
int delay; int delay;
@ -425,12 +423,12 @@ public class VClassGroupCache implements IndexingEventListener {
log.debug("rebuildGroupCacheThread.run() -- rebuilt cache "); log.debug("rebuildGroupCacheThread.run() -- rebuilt cache ");
failedAttempts = 0; failedAttempts = 0;
delay = 100; delay = 100;
} catch (SolrServerException e) { } catch (SearchEngineException e) {
failedAttempts++; failedAttempts++;
if( failedAttempts >= maxFailedAttempts ){ if( failedAttempts >= maxFailedAttempts ){
log.error("Could not build VClassGroupCache. " + log.error("Could not build VClassGroupCache. " +
"Could not connect with Solr after " + "Could not connect with Solr after " +
failedAttempts + " attempts.", e.getRootCause()); failedAttempts + " attempts.", e.getCause());
rebuildRequested = false; rebuildRequested = false;
failedAttempts = 0; failedAttempts = 0;
delay = 1000; delay = 1000;
@ -479,11 +477,13 @@ public class VClassGroupCache implements IndexingEventListener {
* Listen for changes to what class group classes are in and their display rank. * Listen for changes to what class group classes are in and their display rank.
*/ */
protected class VClassGroupCacheChangeListener extends StatementListener { protected class VClassGroupCacheChangeListener extends StatementListener {
public void addedStatement(Statement stmt) { @Override
public void addedStatement(Statement stmt) {
checkAndDoUpdate(stmt); checkAndDoUpdate(stmt);
} }
public void removedStatement(Statement stmt) { @Override
public void removedStatement(Statement stmt) {
checkAndDoUpdate(stmt); checkAndDoUpdate(stmt);
} }

32
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/VTwo/fields/IndividualsViaSolrQueryOptions.java
View file

@ -8,23 +8,21 @@ import java.util.List;
import java.util.ListIterator; import java.util.ListIterator;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement; import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary; import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory; import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.fields.FieldUtils; import edu.cornell.mannlib.vitro.webapp.utils.fields.FieldUtils;
/* /*
@ -34,13 +32,11 @@ import edu.cornell.mannlib.vitro.webapp.utils.fields.FieldUtils;
public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions implements FieldOptions { public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions implements FieldOptions {
private Log log = LogFactory.getLog(IndividualsViaSolrQueryOptions.class); private Log log = LogFactory.getLog(IndividualsViaSolrQueryOptions.class);
private ServletContext servletContext;
private String subjectUri; private String subjectUri;
private String predicateUri; private String predicateUri;
private String objectUri; private String objectUri;
public IndividualsViaSolrQueryOptions(ServletContext context, String inputSubjectUri, String inputPredicateUri, String inputObjectUri, String ... vclassURIs) throws Exception { public IndividualsViaSolrQueryOptions(String inputSubjectUri, String inputPredicateUri, String inputObjectUri, String ... vclassURIs) throws Exception {
super(vclassURIs); super(vclassURIs);
this.servletContext = context;
this.subjectUri = inputSubjectUri; this.subjectUri = inputSubjectUri;
this.predicateUri = inputPredicateUri; this.predicateUri = inputPredicateUri;
this.objectUri = inputObjectUri; this.objectUri = inputObjectUri;
@ -50,10 +46,10 @@ public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions
protected Map<String,Individual> getIndividualsForClass(String vclassURI, WebappDaoFactory wDaoFact ){ protected Map<String,Individual> getIndividualsForClass(String vclassURI, WebappDaoFactory wDaoFact ){
Map<String, Individual> individualMap = new HashMap<String, Individual>(); Map<String, Individual> individualMap = new HashMap<String, Individual>();
try { try {
SolrServer solrServer = SolrSetup.getSolrServer(servletContext); SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
//solr query for type count. //solr query for type count.
SolrQuery query = new SolrQuery(); SearchQuery query = solrServer.createQuery();
if( VitroVocabulary.OWL_THING.equals( vclassURI )){ if( VitroVocabulary.OWL_THING.equals( vclassURI )){
query.setQuery( "*:*" ); query.setQuery( "*:*" );
}else{ }else{
@ -61,15 +57,15 @@ public class IndividualsViaSolrQueryOptions extends IndividualsViaVClassOptions
} }
query.setStart(0) query.setStart(0)
.setRows(1000); .setRows(1000);
query.setFields(VitroSearchTermNames.URI); // fields to retrieve query.addFields(VitroSearchTermNames.URI); // fields to retrieve
QueryResponse rsp = solrServer.query(query); SearchResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults(); SearchResultDocumentList docs = rsp.getResults();
long found = docs.getNumFound(); long found = docs.getNumFound();
if(found > 0) { if(found > 0) {
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
try { try {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual individual = wDaoFact.getIndividualDao().getIndividualByURI(uri); Individual individual = wDaoFact.getIndividualDao().getIndividualByURI(uri);
if (individual == null) { if (individual == null) {
log.debug("No individual for search document with uri = " + uri); log.debug("No individual for search document with uri = " + uri);

36
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/configuration/generators/DefaultObjectPropertyFormGenerator.java
View file

@ -5,24 +5,18 @@ package edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.generators
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.http.HttpSession; import javax.servlet.http.HttpSession;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Literal; import com.hp.hpl.jena.rdf.model.Literal;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty; import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -37,9 +31,13 @@ import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.FieldVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropetyOptions; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropetyOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.validators.AntiXssValidation; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.validators.AntiXssValidation;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n; import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch; import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils; import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils.EditMode; import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils.EditMode;
@ -175,9 +173,9 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
return types; return types;
} }
private boolean tooManyRangeOptions(VitroRequest vreq, HttpSession session ) throws SolrServerException { private boolean tooManyRangeOptions(VitroRequest vreq, HttpSession session ) throws SearchEngineException {
List<VClass> rangeTypes = getRangeTypes(vreq); List<VClass> rangeTypes = getRangeTypes(vreq);
SolrServer solrServer = SolrSetup.getSolrServer(session.getServletContext()); SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
List<String> types = new ArrayList<String>(); List<String> types = new ArrayList<String>();
for (VClass vclass : rangeTypes) { for (VClass vclass : rangeTypes) {
@ -194,15 +192,15 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
long count = 0; long count = 0;
for( String type:types){ for( String type:types){
//solr query for type count. //solr query for type count.
SolrQuery query = new SolrQuery(); SearchQuery query = solrServer.createQuery();
if( VitroVocabulary.OWL_THING.equals( type )){ if( VitroVocabulary.OWL_THING.equals( type )){
query.setQuery( "*:*" ); query.setQuery( "*:*" );
}else{ }else{
query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type); query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type);
} }
query.setRows(0); query.setRows(0);
QueryResponse rsp = solrServer.query(query); SearchResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults(); SearchResultDocumentList docs = rsp.getResults();
long found = docs.getNumFound(); long found = docs.getNumFound();
count = count + found; count = count + found;
if( count > maxNonACRangeIndividualCount ) if( count > maxNonACRangeIndividualCount )
@ -524,7 +522,7 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
editConfiguration.setFormSpecificData(formSpecificData); editConfiguration.setFormSpecificData(formSpecificData);
} }
public void addFormSpecificDataForAC(EditConfigurationVTwo editConfiguration, VitroRequest vreq, HttpSession session) throws SolrServerException { public void addFormSpecificDataForAC(EditConfigurationVTwo editConfiguration, VitroRequest vreq, HttpSession session) throws SearchEngineException {
HashMap<String, Object> formSpecificData = new HashMap<String, Object>(); HashMap<String, Object> formSpecificData = new HashMap<String, Object>();
//Get the edit mode //Get the edit mode
formSpecificData.put("editMode", getEditMode(vreq).toString().toLowerCase()); formSpecificData.put("editMode", getEditMode(vreq).toString().toLowerCase());
@ -564,19 +562,19 @@ public class DefaultObjectPropertyFormGenerator implements EditConfigurationGene
editConfiguration.setFormSpecificData(formSpecificData); editConfiguration.setFormSpecificData(formSpecificData);
} }
private Object rangeIndividualsExist(HttpSession session, List<VClass> types) throws SolrServerException { private Object rangeIndividualsExist(HttpSession session, List<VClass> types) throws SearchEngineException {
SolrServer solrServer = SolrSetup.getSolrServer(session.getServletContext()); SearchEngine solrServer = ApplicationUtils.instance().getSearchEngine();
boolean rangeIndividualsFound = false; boolean rangeIndividualsFound = false;
for( VClass type:types){ for( VClass type:types){
//solr for type count. //solr for type count.
SolrQuery query = new SolrQuery(); SearchQuery query =ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type.getURI()); query.setQuery( VitroSearchTermNames.RDFTYPE + ":" + type.getURI());
query.setRows(0); query.setRows(0);
QueryResponse rsp = solrServer.query(query); SearchResponse rsp = solrServer.query(query);
SolrDocumentList docs = rsp.getResults(); SearchResultDocumentList docs = rsp.getResults();
if( docs.getNumFound() > 0 ){ if( docs.getNumFound() > 0 ){
rangeIndividualsFound = true; rangeIndividualsFound = true;
break; break;

29
webapp/src/edu/cornell/mannlib/vitro/webapp/edit/n3editing/configuration/generators/IndividualsByRankFormGenerator.java
View file

@ -2,47 +2,18 @@
package edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.generators; package edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.generators;
import static edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary.*;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.http.HttpSession;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Literal;
import com.hp.hpl.jena.rdf.model.Model;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationUtils;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.EditConfigurationVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.FieldVTwo; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.FieldVTwo;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropertyByRankOptions; import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropertyByRankOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.VTwo.fields.IndividualsViaObjectPropetyOptions;
import edu.cornell.mannlib.vitro.webapp.edit.n3editing.configuration.validators.AntiXssValidation;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils;
import edu.cornell.mannlib.vitro.webapp.utils.FrontEndEditingUtils.EditMode;
/** /**
* Generates the edit configuration for a default property form. * Generates the edit configuration for a default property form.

21
webapp/src/edu/cornell/mannlib/vitro/webapp/filters/CachingResponseFilter.java
View file

@ -21,15 +21,15 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.collections.EnumerationUtils; import org.apache.commons.collections.EnumerationUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vedit.beans.LoginStatusBean; import edu.cornell.mannlib.vedit.beans.LoginStatusBean;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.UserAccount; import edu.cornell.mannlib.vitro.webapp.beans.UserAccount;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties; import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap; import edu.cornell.mannlib.vitro.webapp.utils.solr.FieldMap;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils; import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrQueryUtils;
import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrResultsParser; import edu.cornell.mannlib.vitro.webapp.utils.solr.SolrResultsParser;
@ -244,13 +244,12 @@ public class CachingResponseFilter implements Filter {
* Ask Solr whether it has an ETAG for this URI. * Ask Solr whether it has an ETAG for this URI.
*/ */
private String findEtagForIndividual(String individualUri) { private String findEtagForIndividual(String individualUri) {
SolrQuery query = new SolrQuery("URI:" + individualUri) SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
.setFields(ETAG_FIELD); SearchQuery query = solr.createQuery("URI:" + individualUri)
.addFields(ETAG_FIELD);
SolrServer solr = SolrSetup.getSolrServer(ctx);
try { try {
QueryResponse response = solr.query(query); SearchResponse response = solr.query(query);
List<Map<String, String>> maps = new SolrResultsParser(response, List<Map<String, String>> maps = new SolrResultsParser(response,
parserFieldMap).parse(); parserFieldMap).parse();
log.debug("Solr response for '" + query.getQuery() + "' was " log.debug("Solr response for '" + query.getQuery() + "' was "
@ -261,7 +260,7 @@ public class CachingResponseFilter implements Filter {
} else { } else {
return maps.get(0).get(ETAG_FIELD); return maps.get(0).get(ETAG_FIELD);
} }
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.warn( log.warn(
"Solr query '" + query.getQuery() + "' threw an exception", "Solr query '" + query.getQuery() + "' threw an exception",
e); e);

72
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/AutocompleteController.java
View file

@ -16,20 +16,20 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController; import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/** /**
* AutocompleteController generates autocomplete content * AutocompleteController generates autocomplete content
@ -66,7 +66,7 @@ public class AutocompleteController extends VitroAjaxController {
String qtxt = vreq.getParameter(PARAM_QUERY); String qtxt = vreq.getParameter(PARAM_QUERY);
SolrQuery query = getQuery(qtxt, vreq); SearchQuery query = getQuery(qtxt, vreq);
if (query == null ) { if (query == null ) {
log.debug("query for '" + qtxt +"' is null."); log.debug("query for '" + qtxt +"' is null.");
doNoQuery(response); doNoQuery(response);
@ -74,8 +74,8 @@ public class AutocompleteController extends VitroAjaxController {
} }
log.debug("query for '" + qtxt +"' is " + query.toString()); log.debug("query for '" + qtxt +"' is " + query.toString());
SolrServer solr = SolrSetup.getSolrServer(getServletContext()); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
QueryResponse queryResponse = solr.query(query); SearchResponse queryResponse = solr.query(query);
if ( queryResponse == null) { if ( queryResponse == null) {
log.error("Query response for a search was null"); log.error("Query response for a search was null");
@ -83,7 +83,7 @@ public class AutocompleteController extends VitroAjaxController {
return; return;
} }
SolrDocumentList docs = queryResponse.getResults(); SearchResultDocumentList docs = queryResponse.getResults();
if ( docs == null) { if ( docs == null) {
log.error("Docs for a search was null"); log.error("Docs for a search was null");
@ -99,33 +99,11 @@ public class AutocompleteController extends VitroAjaxController {
} }
List<SearchResult> results = new ArrayList<SearchResult>(); List<SearchResult> results = new ArrayList<SearchResult>();
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
try { try {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
// RY 7/1/2011 String name = doc.getStringValue(VitroSearchTermNames.NAME_RAW);
// Comment was: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() returns a list. String mst = doc.getStringValue(VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS);
// Changed to: VitroSearchTermNames.NAME_RAW is a multivalued field, so doc.get() could return a list
// But in fact: I'm no longer seeing any lists returned for individuals with multiple labels. Not sure
// if this is new behavior or what. ???
Object nameRaw = doc.get(VitroSearchTermNames.NAME_RAW);
String name = null;
if (nameRaw instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> nameRawList = (List<String>) nameRaw;
name = nameRawList.get(0);
} else {
name = (String) nameRaw;
}
Object mostSpecificType = doc.get(VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS);
String mst = null;
if (mostSpecificType instanceof List<?>) {
@SuppressWarnings("unchecked")
List<String> mstList = (List<String>) mostSpecificType;
mst = mstList.get(0);
} else {
mst = (String) mostSpecificType;
}
SearchResult result = new SearchResult(name, uri, mst); SearchResult result = new SearchResult(name, uri, mst);
results.add(result); results.add(result);
@ -150,7 +128,7 @@ public class AutocompleteController extends VitroAjaxController {
} }
} }
private SolrQuery getQuery(String queryStr, VitroRequest vreq) { private SearchQuery getQuery(String queryStr, VitroRequest vreq) {
if ( queryStr == null) { if ( queryStr == null) {
log.error("There was no parameter '"+ PARAM_QUERY log.error("There was no parameter '"+ PARAM_QUERY
@ -162,26 +140,26 @@ public class AutocompleteController extends VitroAjaxController {
return null; return null;
} }
SolrQuery query = new SolrQuery(); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setStart(0) query.setStart(0)
.setRows(DEFAULT_MAX_HIT_COUNT); .setRows(DEFAULT_MAX_HIT_COUNT);
setNameQuery(query, queryStr, vreq); setNameQuery(query, queryStr, vreq);
// Filter by type // Filter by type
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE); String typeParam = vreq.getParameter(PARAM_RDFTYPE);
String multipleTypesParam = (String) vreq.getParameter(PARAM_MULTIPLE_RDFTYPE); String multipleTypesParam = vreq.getParameter(PARAM_MULTIPLE_RDFTYPE);
if (typeParam != null) { if (typeParam != null) {
addFilterQuery(query, typeParam, multipleTypesParam); addFilterQuery(query, typeParam, multipleTypesParam);
} }
query.setFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve query.addFields(VitroSearchTermNames.NAME_RAW, VitroSearchTermNames.URI, VitroSearchTermNames.MOST_SPECIFIC_TYPE_URIS); // fields to retrieve
// Can't sort on multivalued field, so we sort the results in Java when we get them. // Can't sort on multivalued field, so we sort the results in Java when we get them.
// query.setSortField(VitroSearchTermNames.NAME_LOWERCASE, SolrQuery.ORDER.asc); // query.addSortField(VitroSearchTermNames.NAME_LOWERCASE, Order.ASC);
return query; return query;
} }
private void addFilterQuery(SolrQuery query, String typeParam, String multipleTypesParam) { private void addFilterQuery(SearchQuery query, String typeParam, String multipleTypesParam) {
if(multipleTypesParam == null || multipleTypesParam.equals("null") || multipleTypesParam.isEmpty()) { if(multipleTypesParam == null || multipleTypesParam.equals("null") || multipleTypesParam.isEmpty()) {
//Single type parameter, process as usual //Single type parameter, process as usual
query.addFilterQuery(VitroSearchTermNames.RDFTYPE + ":\"" + typeParam + "\""); query.addFilterQuery(VitroSearchTermNames.RDFTYPE + ":\"" + typeParam + "\"");
@ -200,12 +178,12 @@ public class AutocompleteController extends VitroAjaxController {
} }
} }
private void setNameQuery(SolrQuery query, String queryStr, HttpServletRequest request) { private void setNameQuery(SearchQuery query, String queryStr, HttpServletRequest request) {
if (StringUtils.isBlank(queryStr)) { if (StringUtils.isBlank(queryStr)) {
log.error("No query string"); log.error("No query string");
} }
String tokenizeParam = (String) request.getParameter("tokenize"); String tokenizeParam = request.getParameter("tokenize");
boolean tokenize = "true".equals(tokenizeParam); boolean tokenize = "true".equals(tokenizeParam);
// Note: Stemming is only relevant if we are tokenizing: an untokenized name // Note: Stemming is only relevant if we are tokenizing: an untokenized name
@ -218,7 +196,7 @@ public class AutocompleteController extends VitroAjaxController {
} }
} }
private void setTokenizedNameQuery(SolrQuery query, String queryStr, HttpServletRequest request) { private void setTokenizedNameQuery(SearchQuery query, String queryStr, HttpServletRequest request) {
/* We currently have no use case for a tokenized, unstemmed autocomplete search field, so the option /* We currently have no use case for a tokenized, unstemmed autocomplete search field, so the option
* has been disabled. If needed in the future, will need to add a new field and field type which * has been disabled. If needed in the future, will need to add a new field and field type which
@ -262,7 +240,7 @@ public class AutocompleteController extends VitroAjaxController {
} }
private void setUntokenizedNameQuery(SolrQuery query, String queryStr) { private void setUntokenizedNameQuery(SearchQuery query, String queryStr) {
queryStr = queryStr.trim(); queryStr = queryStr.trim();
queryStr = makeTermQuery(VitroSearchTermNames.AC_NAME_UNTOKENIZED, queryStr, true); queryStr = makeTermQuery(VitroSearchTermNames.AC_NAME_UNTOKENIZED, queryStr, true);
query.setQuery(queryStr); query.setQuery(queryStr);

17
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/DataAutocompleteController.java
View file

@ -5,28 +5,15 @@ package edu.cornell.mannlib.vitro.webapp.search.controller;
import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
import java.io.IOException; import java.io.IOException;
import java.net.URLEncoder;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject;
import com.hp.hpl.jena.query.Dataset; import com.hp.hpl.jena.query.Dataset;
import com.hp.hpl.jena.query.DatasetFactory; import com.hp.hpl.jena.query.DatasetFactory;
@ -42,10 +29,8 @@ import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils; import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils.AjaxControllerException; import edu.cornell.mannlib.vitro.webapp.controller.ajax.SparqlUtils.AjaxControllerException;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.controller.ajax.VitroAjaxController;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/** /**
* DataAutocompleteController generates autocomplete content * DataAutocompleteController generates autocomplete content

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/IndexController.java
View file

@ -15,9 +15,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission; import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions; import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@ -28,7 +27,6 @@ import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.Red
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder; import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel; import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevel;
import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp; import edu.cornell.mannlib.vitro.webapp.utils.threads.VitroBackgroundThread.WorkLevelStamp;
@ -160,7 +158,7 @@ public class IndexController extends FreemarkerHttpServlet {
private Boolean testIndexConnection() { private Boolean testIndexConnection() {
try { try {
SolrSetup.getSolrServer(getServletContext()).ping(); ApplicationUtils.instance().getSearchEngine().ping();
return Boolean.TRUE; return Boolean.TRUE;
} catch (Exception e) { } catch (Exception e) {
log.error("Can't connect to the Solr server.", e); log.error("Can't connect to the Solr server.", e);

70
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java
View file

@ -21,14 +21,8 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean; import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
@ -47,9 +41,15 @@ import edu.cornell.mannlib.vitro.webapp.dao.VClassGroupsForRequest;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary; import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache; import edu.cornell.mannlib.vitro.webapp.dao.jena.VClassGroupCache;
import edu.cornell.mannlib.vitro.webapp.i18n.I18n; import edu.cornell.mannlib.vitro.webapp.i18n.I18n;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchFacetField.Count;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.IndexConstants; import edu.cornell.mannlib.vitro.webapp.search.IndexConstants;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
import edu.ucsf.vitro.opensocial.OpenSocialManager; import edu.ucsf.vitro.opensocial.OpenSocialManager;
@ -169,9 +169,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return doFailedSearch(badQueryMsg, queryText, format, vreq); return doFailedSearch(badQueryMsg, queryText, format, vreq);
} }
SolrQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq); SearchQuery query = getQuery(queryText, hitsPerPage, startIndex, vreq);
SolrServer solr = SolrSetup.getSolrServer(getServletContext()); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
QueryResponse response = null; SearchResponse response = null;
try { try {
response = solr.query(query); response = solr.query(query);
@ -186,7 +186,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText, format, vreq); return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText, format, vreq);
} }
SolrDocumentList docs = response.getResults(); SearchResultDocumentList docs = response.getResults();
if (docs == null) { if (docs == null) {
log.error("Document list for a search was null"); log.error("Document list for a search was null");
return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText,format, vreq); return doFailedSearch(I18n.text(vreq, "error_in_search_request"), queryText,format, vreq);
@ -199,11 +199,11 @@ public class PagedSearchController extends FreemarkerHttpServlet {
} }
List<Individual> individuals = new ArrayList<Individual>(docs.size()); List<Individual> individuals = new ArrayList<Individual>(docs.size());
Iterator<SolrDocument> docIter = docs.iterator(); Iterator<SearchResultDocument> docIter = docs.iterator();
while( docIter.hasNext() ){ while( docIter.hasNext() ){
try { try {
SolrDocument doc = docIter.next(); SearchResultDocument doc = docIter.next();
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.getStringValue(VitroSearchTermNames.URI);
Individual ind = iDao.getIndividualByURI(uri); Individual ind = iDao.getIndividualByURI(uri);
if(ind != null) { if(ind != null) {
ind.setSearchSnippet( getSnippet(doc, response) ); ind.setSearchSnippet( getSnippet(doc, response) );
@ -353,12 +353,12 @@ public class PagedSearchController extends FreemarkerHttpServlet {
* Get the class groups represented for the individuals in the documents. * Get the class groups represented for the individuals in the documents.
* @param qtxt * @param qtxt
*/ */
private List<VClassGroupSearchLink> getClassGroupsLinks(VitroRequest vreq, VClassGroupDao grpDao, SolrDocumentList docs, QueryResponse rsp, String qtxt) { private List<VClassGroupSearchLink> getClassGroupsLinks(VitroRequest vreq, VClassGroupDao grpDao, SearchResultDocumentList docs, SearchResponse rsp, String qtxt) {
Map<String,Long> cgURItoCount = new HashMap<String,Long>(); Map<String,Long> cgURItoCount = new HashMap<String,Long>();
List<VClassGroup> classgroups = new ArrayList<VClassGroup>( ); List<VClassGroup> classgroups = new ArrayList<VClassGroup>( );
List<FacetField> ffs = rsp.getFacetFields(); List<SearchFacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){ for(SearchFacetField ff : ffs){
if(VitroSearchTermNames.CLASSGROUP_URI.equals(ff.getName())){ if(VitroSearchTermNames.CLASSGROUP_URI.equals(ff.getName())){
List<Count> counts = ff.getValues(); List<Count> counts = ff.getValues();
for( Count ct: counts){ for( Count ct: counts){
@ -388,13 +388,13 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return classGroupLinks; return classGroupLinks;
} }
private List<VClassSearchLink> getVClassLinks(VClassDao vclassDao, SolrDocumentList docs, QueryResponse rsp, String qtxt){ private List<VClassSearchLink> getVClassLinks(VClassDao vclassDao, SearchResultDocumentList docs, SearchResponse rsp, String qtxt){
HashSet<String> typesInHits = getVClassUrisForHits(docs); HashSet<String> typesInHits = getVClassUrisForHits(docs);
List<VClass> classes = new ArrayList<VClass>(typesInHits.size()); List<VClass> classes = new ArrayList<VClass>(typesInHits.size());
Map<String,Long> typeURItoCount = new HashMap<String,Long>(); Map<String,Long> typeURItoCount = new HashMap<String,Long>();
List<FacetField> ffs = rsp.getFacetFields(); List<SearchFacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){ for(SearchFacetField ff : ffs){
if(VitroSearchTermNames.RDFTYPE.equals(ff.getName())){ if(VitroSearchTermNames.RDFTYPE.equals(ff.getName())){
List<Count> counts = ff.getValues(); List<Count> counts = ff.getValues();
for( Count ct: counts){ for( Count ct: counts){
@ -435,9 +435,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return vClassLinks; return vClassLinks;
} }
private HashSet<String> getVClassUrisForHits(SolrDocumentList docs){ private HashSet<String> getVClassUrisForHits(SearchResultDocumentList docs){
HashSet<String> typesInHits = new HashSet<String>(); HashSet<String> typesInHits = new HashSet<String>();
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
try { try {
Collection<Object> types = doc.getFieldValues(VitroSearchTermNames.RDFTYPE); Collection<Object> types = doc.getFieldValues(VitroSearchTermNames.RDFTYPE);
if (types != null) { if (types != null) {
@ -453,8 +453,8 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return typesInHits; return typesInHits;
} }
private String getSnippet(SolrDocument doc, QueryResponse response) { private String getSnippet(SearchResultDocument doc, SearchResponse response) {
String docId = doc.get(VitroSearchTermNames.DOCID).toString(); String docId = doc.getStringValue(VitroSearchTermNames.DOCID);
StringBuffer text = new StringBuffer(); StringBuffer text = new StringBuffer();
Map<String, Map<String, List<String>>> highlights = response.getHighlighting(); Map<String, Map<String, List<String>>> highlights = response.getHighlighting();
if (highlights != null && highlights.get(docId) != null) { if (highlights != null && highlights.get(docId) != null) {
@ -466,19 +466,19 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return text.toString(); return text.toString();
} }
private SolrQuery getQuery(String queryText, int hitsPerPage, int startIndex, VitroRequest vreq) { private SearchQuery getQuery(String queryText, int hitsPerPage, int startIndex, VitroRequest vreq) {
// Lowercase the search term to support wildcard searches: Solr applies no text // Lowercase the search term to support wildcard searches: Solr applies no text
// processing to a wildcard search term. // processing to a wildcard search term.
SolrQuery query = new SolrQuery( queryText ); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery(queryText);
query.setStart( startIndex ) query.setStart( startIndex )
.setRows(hitsPerPage); .setRows(hitsPerPage);
// ClassGroup filtering param // ClassGroup filtering param
String classgroupParam = (String) vreq.getParameter(PARAM_CLASSGROUP); String classgroupParam = vreq.getParameter(PARAM_CLASSGROUP);
// rdf:type filtering param // rdf:type filtering param
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE); String typeParam = vreq.getParameter(PARAM_RDFTYPE);
if ( ! StringUtils.isBlank(classgroupParam) ) { if ( ! StringUtils.isBlank(classgroupParam) ) {
// ClassGroup filtering // ClassGroup filtering
@ -487,9 +487,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
query.addFilterQuery(VitroSearchTermNames.CLASSGROUP_URI + ":\"" + classgroupParam + "\""); query.addFilterQuery(VitroSearchTermNames.CLASSGROUP_URI + ":\"" + classgroupParam + "\"");
//with ClassGroup filtering we want type facets //with ClassGroup filtering we want type facets
query.add("facet","true"); query.addParameter("facet","true");
query.add("facet.limit","-1"); query.addParameter("facet.limit","-1");
query.add("facet.field",VitroSearchTermNames.RDFTYPE); query.addParameter("facet.field",VitroSearchTermNames.RDFTYPE);
}else if ( ! StringUtils.isBlank(typeParam) ) { }else if ( ! StringUtils.isBlank(typeParam) ) {
// rdf:type filtering // rdf:type filtering
@ -500,9 +500,9 @@ public class PagedSearchController extends FreemarkerHttpServlet {
//with type filtering we don't have facets. //with type filtering we don't have facets.
}else{ }else{
//When no filtering is set, we want ClassGroup facets //When no filtering is set, we want ClassGroup facets
query.add("facet","true"); query.addParameter("facet","true");
query.add("facet.limit","-1"); query.addParameter("facet.limit","-1");
query.add("facet.field",VitroSearchTermNames.CLASSGROUP_URI); query.addParameter("facet.field",VitroSearchTermNames.CLASSGROUP_URI);
} }
log.debug("Query = " + query.toString()); log.debug("Query = " + query.toString());

64
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrIndexer.java
View file

@ -7,15 +7,16 @@ import java.util.HashSet;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException; import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface; import edu.cornell.mannlib.vitro.webapp.search.beans.IndexerIface;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument; import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
@ -24,7 +25,7 @@ import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualT
public class SolrIndexer implements IndexerIface { public class SolrIndexer implements IndexerIface {
private final static Log log = LogFactory.getLog(SolrIndexer.class); private final static Log log = LogFactory.getLog(SolrIndexer.class);
protected SolrServer server; protected SearchEngine server;
protected boolean indexing; protected boolean indexing;
protected HashSet<String> urisIndexed; protected HashSet<String> urisIndexed;
protected IndividualToSolrDocument individualToSolrDoc; protected IndividualToSolrDocument individualToSolrDoc;
@ -48,7 +49,7 @@ public class SolrIndexer implements IndexerIface {
*/ */
protected boolean doingFullIndexRebuild = false; protected boolean doingFullIndexRebuild = false;
public SolrIndexer( SolrServer server, IndividualToSolrDocument indToDoc){ public SolrIndexer( SearchEngine server, IndividualToSolrDocument indToDoc){
this.server = server; this.server = server;
this.individualToSolrDoc = indToDoc; this.individualToSolrDoc = indToDoc;
} }
@ -59,15 +60,17 @@ public class SolrIndexer implements IndexerIface {
throw new IndexingException("SolrIndexer: must call " + throw new IndexingException("SolrIndexer: must call " +
"startIndexing() before index()."); "startIndexing() before index().");
if( ind == null ) if( ind == null ) {
log.debug("Individual to index was null, ignoring."); log.debug("Individual to index was null, ignoring.");
return;
}
try{ try{
if( urisIndexed.contains(ind.getURI()) ){ if( urisIndexed.contains(ind.getURI()) ){
log.debug("already indexed " + ind.getURI() ); log.debug("already indexed " + ind.getURI() );
return; return;
}else{ }else{
SolrInputDocument solrDoc = null; SearchInputDocument solrDoc = null;
synchronized(this){ synchronized(this){
urisIndexed.add(ind.getURI()); urisIndexed.add(ind.getURI());
} }
@ -80,16 +83,14 @@ public class SolrIndexer implements IndexerIface {
log.debug( solrDoc.toString() ); log.debug( solrDoc.toString() );
} }
UpdateResponse res = server.add( solrDoc ); server.add( solrDoc );
log.debug("response after adding docs to server: "+ res); log.debug("Added docs to server.");
}else{ }else{
log.debug("removing from index " + ind.getURI()); log.debug("removing from index " + ind.getURI());
removeFromIndex(ind.getURI()); removeFromIndex(ind.getURI());
} }
} }
} catch (IOException ex) { } catch (SearchEngineException ex) {
throw new IndexingException(ex.getMessage());
} catch (SolrServerException ex) {
throw new IndexingException(ex.getMessage()); throw new IndexingException(ex.getMessage());
} }
} }
@ -111,9 +112,7 @@ public class SolrIndexer implements IndexerIface {
try { try {
server.deleteById(individualToSolrDoc.getIdForUri(uri)); server.deleteById(individualToSolrDoc.getIdForUri(uri));
log.debug("deleted " + " " + uri); log.debug("deleted " + " " + uri);
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.error( "could not delete individual " + uri, e);
} catch (IOException e) {
log.error( "could not delete individual " + uri, e); log.error( "could not delete individual " + uri, e);
} }
} }
@ -172,10 +171,7 @@ public class SolrIndexer implements IndexerIface {
try { try {
server.deleteByQuery("indexedTime:[ * TO " + reindexStart + " ]"); server.deleteByQuery("indexedTime:[ * TO " + reindexStart + " ]");
server.commit(); server.commit();
} catch (SolrServerException e) { } catch (SearchEngineException e) {
if( ! shutdownRequested )
log.error("could not delete documents from before rebuild.",e);
} catch (IOException e) {
if( ! shutdownRequested ) if( ! shutdownRequested )
log.error("could not delete documents from before rebuild.",e); log.error("could not delete documents from before rebuild.",e);
} }
@ -186,17 +182,17 @@ public class SolrIndexer implements IndexerIface {
public long getModified() { public long getModified() {
long modified = 0; long modified = 0;
SolrQuery query = new SolrQuery(); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery("*:*"); query.setQuery("*:*");
query.addSortField("indexedTime", SolrQuery.ORDER.desc); query.addSortField("indexedTime", Order.DESC);
try { try {
QueryResponse rsp = server.query(query); SearchResponse rsp = server.query(query);
SolrDocumentList docs = rsp.getResults(); SearchResultDocumentList docs = rsp.getResults();
if(docs!=null){ if(docs!=null){
modified = (Long)docs.get(0).getFieldValue("indexedTime"); modified = (Long)docs.get(0).getFirstValue("indexedTime");
} }
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.error(e,e); log.error(e,e);
} }
@ -208,16 +204,16 @@ public class SolrIndexer implements IndexerIface {
* and returns false on failure to connect to server. * and returns false on failure to connect to server.
*/ */
public boolean isIndexEmpty() { public boolean isIndexEmpty() {
SolrQuery query = new SolrQuery(); SearchQuery query = ApplicationUtils.instance().getSearchEngine().createQuery();
query.setQuery("*:*"); query.setQuery("*:*");
try { try {
QueryResponse rsp = server.query(query); SearchResponse rsp = server.query(query);
SolrDocumentList docs = rsp.getResults(); SearchResultDocumentList docs = rsp.getResults();
if(docs==null || docs.size()==0){ if(docs==null || docs.size()==0){
return true; return true;
} }
} catch (SolrServerException e) { } catch (SearchEngineException e) {
log.error("Could not connect to solr server" ,e.getRootCause()); log.error("Could not connect to solr server" ,e.getCause());
} }
return false; return false;
} }

209
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java
View file

@ -1,209 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.ArrayList;
import java.util.List;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
import edu.cornell.mannlib.vitro.webapp.search.beans.StatementToURIsToUpdate;
import edu.cornell.mannlib.vitro.webapp.search.indexing.AdditionalUriFinders;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
import edu.cornell.mannlib.vitro.webapp.search.indexing.SearchReindexingListener;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.DocumentModifier;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnNamespace;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnType;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeBasedOnTypeNamespace;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ExcludeNonFlagVitro;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.NameBoost;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.NameFields;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.SearchIndexExcluder;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.SyncingExcludeBasedOnType;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.ThumbnailImageURL;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
public class SolrSetup implements javax.servlet.ServletContextListener{
public static final String SOLR_SERVER = "vitro.local.solr.server";
public static final String PROHIBITED_FROM_SEARCH = "edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch";
/** Exclude from the search index Individuals with types from these namespaces */
private static final String[] TYPE_NS_EXCLUDES = {
VitroVocabulary.PUBLIC
//if you do OWL.NS here you will exclude all of owl:Thing.
};
/** Exclude from the search index individuals who's URIs start with these namespaces. */
private static final String[] INDIVIDUAL_NS_EXCLUDES={
VitroVocabulary.vitroURI,
VitroVocabulary.VITRO_PUBLIC,
VitroVocabulary.PSEUDO_BNODE_NS,
OWL.NS
};
/** Individuals of these types will be excluded from the search index */
private static final String[] OWL_TYPES_EXCLUDES = {
OWL.ObjectProperty.getURI(),
OWL.DatatypeProperty.getURI(),
OWL.AnnotationProperty.getURI()
};
@Override
public void contextInitialized(ServletContextEvent sce) {
ServletContext context = sce.getServletContext();
StartupStatus ss = StartupStatus.getBean(context);
/* setup the http connection with the solr server */
String solrServerUrlString = ConfigurationProperties.getBean(sce).getProperty("vitro.local.solr.url");
if( solrServerUrlString == null ){
ss.fatal(this, "Could not find vitro.local.solr.url in runtime.properties. "+
"Vitro application needs a URL of a solr server that it can use to index its data. " +
"It should be something like http://localhost:${port}" + context.getContextPath() + "solr"
);
return;
}
try {
HttpSolrServer server;
boolean useMultiPartPost = true;
server = new HttpSolrServer( solrServerUrlString );
server.setSoTimeout(10000); // socket read timeout
server.setConnectionTimeout(10000);
server.setDefaultMaxConnectionsPerHost(100);
server.setMaxTotalConnections(100);
server.setMaxRetries(1);
context.setAttribute(SOLR_SERVER, server);
/* set up the individual to solr doc translation */
OntModel jenaOntModel = ModelAccess.on(context).getJenaOntModel();
OntModel displayModel = ModelAccess.on(context).getDisplayModel();
/* try to get context attribute DocumentModifiers
* and use that as the start of the list of DocumentModifier
* objects. This allows other ContextListeners to add to
* the basic set of DocumentModifiers. */
@SuppressWarnings("unchecked")
List<DocumentModifier> modifiersFromContext =
(List<DocumentModifier>)context.getAttribute("DocumentModifiers");
/* try to get context attribute SearchIndexExcludes
* and use that as the start of the list of exclude
* objects. This allows other ContextListeners to add to
* the basic set of SearchIndexExcludes . */
@SuppressWarnings("unchecked")
List<SearchIndexExcluder> searchIndexExcludesFromContext =
(List<SearchIndexExcluder>)context.getAttribute("SearchIndexExcludes");
IndividualToSolrDocument indToSolrDoc =
setupTransltion(jenaOntModel, displayModel,
RDFServiceUtils.getRDFServiceFactory(context),
modifiersFromContext,
searchIndexExcludesFromContext);
/* setup solr indexer */
SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc);
// This is where the builder gets the list of places to try to
// get objects to index. It is filtered so that non-public text
// does not get into the search index.
WebappDaoFactory wadf = ModelAccess.on(context).getWebappDaoFactory();
VitroFilters vf = VitroFilterUtils.getPublicFilter(context);
wadf = new WebappDaoFactoryFiltering(wadf, vf);
// make objects that will find additional URIs for context nodes etc
RDFService rdfService = RDFServiceUtils.getRDFServiceFactory(context).getRDFService();
List<StatementToURIsToUpdate> uriFinders = AdditionalUriFinders.getList(rdfService,wadf.getIndividualDao());
// Make the IndexBuilder
IndexBuilder builder = new IndexBuilder( solrIndexer, wadf, uriFinders );
// Save it to the servlet context so we can access it later in the webapp.
context.setAttribute(IndexBuilder.class.getName(), builder);
// set up listeners so search index builder is notified of changes to model
ServletContext ctx = sce.getServletContext();
SearchReindexingListener srl = new SearchReindexingListener( builder );
ModelContext.registerListenerForChanges(ctx, srl);
ss.info(this, "Setup of Solr index completed.");
} catch (Throwable e) {
ss.fatal(this, "could not setup local solr server",e);
}
}
@Override
public void contextDestroyed(ServletContextEvent sce) {
IndexBuilder builder = (IndexBuilder)sce.getServletContext().getAttribute(IndexBuilder.class.getName());
if( builder != null )
builder.stopIndexingThread();
}
public static SolrServer getSolrServer(ServletContext ctx){
return (SolrServer) ctx.getAttribute(SOLR_SERVER);
}
public static IndividualToSolrDocument setupTransltion(
OntModel jenaOntModel,
Model displayModel,
RDFServiceFactory rdfServiceFactory,
List<DocumentModifier> modifiersFromContext,
List<SearchIndexExcluder> searchIndexExcludesFromContext
) {
/* try to get context attribute DocumentModifiers
* and use that as the start of the list of DocumentModifier
* objects. This allows other ContextListeners to add to
* the basic set of DocumentModifiers. */
List<DocumentModifier> modifiers = new ArrayList<DocumentModifier>();
if( modifiersFromContext != null ){
modifiers.addAll( modifiersFromContext);
}
modifiers.add( new NameFields( rdfServiceFactory ));
modifiers.add( new NameBoost( 1.2f ));
modifiers.add( new ThumbnailImageURL( rdfServiceFactory ));
/* try to get context attribute SearchIndexExcludes
* and use that as the start of the list of exclude
* objects. This allows other ContextListeners to add to
* the basic set of SearchIndexExcludes . */
List<SearchIndexExcluder> excludes =
new ArrayList<SearchIndexExcluder>();
if( searchIndexExcludesFromContext != null ){
excludes.addAll(searchIndexExcludesFromContext);
}
excludes.add(new ExcludeBasedOnNamespace( INDIVIDUAL_NS_EXCLUDES ));
excludes.add(new ExcludeBasedOnTypeNamespace( TYPE_NS_EXCLUDES ) );
excludes.add(new ExcludeBasedOnType( OWL_TYPES_EXCLUDES) );
excludes.add(new ExcludeNonFlagVitro() );
excludes.add( new SyncingExcludeBasedOnType( displayModel ) );
return new IndividualToSolrDocument(excludes, modifiers);
}
}

12
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ContextNodeFields.java
View file

@ -9,16 +9,14 @@ import java.util.List;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import com.hp.hpl.jena.query.QuerySolution; import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.QuerySolutionMap;
import com.hp.hpl.jena.query.ResultSet; import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -53,7 +51,7 @@ public class ContextNodeFields implements DocumentModifier{
} }
@Override @Override
public void modifyDocument(Individual individual, SolrInputDocument doc, StringBuffer addUri) { public void modifyDocument(Individual individual, SearchInputDocument doc, StringBuffer addUri) {
if( individual == null ) if( individual == null )
return; return;
@ -65,11 +63,11 @@ public class ContextNodeFields implements DocumentModifier{
/* get text from the context nodes and add the to ALLTEXT */ /* get text from the context nodes and add the to ALLTEXT */
StringBuffer values = executeQueryForValues(individual, queries); StringBuffer values = executeQueryForValues(individual, queries);
SolrInputField field = doc.getField(VitroSearchTermNames.ALLTEXT); SearchInputField field = doc.getField(VitroSearchTermNames.ALLTEXT);
if( field == null ){ if( field == null ){
doc.addField(VitroSearchTermNames.ALLTEXT, values); doc.addField(VitroSearchTermNames.ALLTEXT, values);
}else{ }else{
field.addValue(values, field.getBoost()); field.addValues(values, field.getBoost());
} }
} }

7
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/DocumentModifier.java
View file

@ -2,14 +2,13 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding; package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
/** /**
* This interface represents an object that can add to a SolrInputDocument. * This interface represents an object that can add to a SearchInputDocument.
*/ */
public interface DocumentModifier { public interface DocumentModifier {
public void modifyDocument(Individual individual, SolrInputDocument doc, StringBuffer addUri) throws SkipIndividualException; public void modifyDocument(Individual individual, SearchInputDocument doc, StringBuffer addUri) throws SkipIndividualException;
//called to inform the DocumentModifier that the system is shutting down //called to inform the DocumentModifier that the system is shutting down
public void shutdown(); public void shutdown();

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ExcludeBasedOnTypeNamespace.java
View file

@ -7,8 +7,6 @@ import java.util.List;
import java.util.regex.Matcher; import java.util.regex.Matcher;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;

27
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/IndividualToSolrDocument.java
View file

@ -12,19 +12,20 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.joda.time.DateTime; import org.joda.time.DateTime;
import org.jsoup.Jsoup; import org.jsoup.Jsoup;
import com.hp.hpl.jena.shared.JenaException; import com.hp.hpl.jena.shared.JenaException;
import com.hp.hpl.jena.vocabulary.OWL; import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement; import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl; import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement; import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException; import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
@ -46,7 +47,7 @@ public class IndividualToSolrDocument {
} }
@SuppressWarnings("static-access") @SuppressWarnings("static-access")
public SolrInputDocument translate(Individual ind) throws IndexingException{ public SearchInputDocument translate(Individual ind) throws IndexingException{
try{ try{
String excludeMsg = checkExcludes( ind ); String excludeMsg = checkExcludes( ind );
if( excludeMsg != DONT_EXCLUDE){ if( excludeMsg != DONT_EXCLUDE){
@ -54,7 +55,7 @@ public class IndividualToSolrDocument {
return null; return null;
} }
SolrInputDocument doc = new SolrInputDocument(); SearchInputDocument doc = ApplicationUtils.instance().getSearchEngine().createInputDocument();
//DocID //DocID
doc.addField(term.DOCID, getIdForUri( ind.getURI() ) ); doc.addField(term.DOCID, getIdForUri( ind.getURI() ) );
@ -79,7 +80,7 @@ public class IndividualToSolrDocument {
addObjectPropertyText(ind, doc, objectNames, addUri); addObjectPropertyText(ind, doc, objectNames, addUri);
//time of index in msec past epoch //time of index in msec past epoch
doc.addField(term.INDEXEDTIME, new Long( (new DateTime()).getMillis() ) ); doc.addField(term.INDEXEDTIME, (Object) new DateTime().getMillis() );
addAllText( ind, doc, classPublicNames, objectNames ); addAllText( ind, doc, classPublicNames, objectNames );
@ -125,7 +126,7 @@ public class IndividualToSolrDocument {
protected Map<String,Long> docModClassToTime = new HashMap<String,Long>(); protected Map<String,Long> docModClassToTime = new HashMap<String,Long>();
protected long docModCount =0; protected long docModCount =0;
protected void runAdditionalDocModifers( Individual ind, SolrInputDocument doc, StringBuffer addUri ) protected void runAdditionalDocModifers( Individual ind, SearchInputDocument doc, StringBuffer addUri )
throws SkipIndividualException{ throws SkipIndividualException{
//run the document modifiers //run the document modifiers
if( documentModifiers != null && !documentModifiers.isEmpty()){ if( documentModifiers != null && !documentModifiers.isEmpty()){
@ -158,7 +159,7 @@ public class IndividualToSolrDocument {
} }
} }
protected void addAllText(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames, StringBuffer objectNames) { protected void addAllText(Individual ind, SearchInputDocument doc, StringBuffer classPublicNames, StringBuffer objectNames) {
String t=null; String t=null;
//ALLTEXT, all of the 'full text' //ALLTEXT, all of the 'full text'
StringBuffer allTextValue = new StringBuffer(); StringBuffer allTextValue = new StringBuffer();
@ -209,7 +210,7 @@ public class IndividualToSolrDocument {
* Get the rdfs:labes for objects of statements and put in objectNames. * Get the rdfs:labes for objects of statements and put in objectNames.
* Get the URIs for objects of statements and put in addUri. * Get the URIs for objects of statements and put in addUri.
*/ */
protected void addObjectPropertyText(Individual ind, SolrInputDocument doc, protected void addObjectPropertyText(Individual ind, SearchInputDocument doc,
StringBuffer objectNames, StringBuffer addUri) { StringBuffer objectNames, StringBuffer addUri) {
try{ try{
@ -245,7 +246,7 @@ public class IndividualToSolrDocument {
* @returns true if prohibited from search * @returns true if prohibited from search
* @throws SkipIndividualException * @throws SkipIndividualException
*/ */
protected void addClasses(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{ protected void addClasses(Individual ind, SearchInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{
ArrayList<String> superClassNames = null; ArrayList<String> superClassNames = null;
List<VClass> vclasses = ind.getVClasses(false); List<VClass> vclasses = ind.getVClasses(false);
@ -279,7 +280,7 @@ public class IndividualToSolrDocument {
} }
} }
protected void addMostSpecificTypeUris(Individual ind, SolrInputDocument doc){ protected void addMostSpecificTypeUris(Individual ind, SearchInputDocument doc){
List<String> mstURIs = ind.getMostSpecificTypeURIs(); List<String> mstURIs = ind.getMostSpecificTypeURIs();
if( mstURIs != null ){ if( mstURIs != null ){
for( String typeURI : mstURIs ){ for( String typeURI : mstURIs ){
@ -289,7 +290,7 @@ public class IndividualToSolrDocument {
} }
} }
protected void addLabel(Individual ind, SolrInputDocument doc) { protected void addLabel(Individual ind, SearchInputDocument doc) {
String value = ""; String value = "";
String label = ind.getRdfsLabel(); String label = ind.getRdfsLabel();
if (label != null) { if (label != null) {
@ -324,8 +325,8 @@ public class IndividualToSolrDocument {
public Individual unTranslate(Object result) { public Individual unTranslate(Object result) {
Individual ent = null; Individual ent = null;
if( result != null && result instanceof SolrDocument){ if( result instanceof SearchResultDocument){
SolrDocument hit = (SolrDocument) result; SearchResultDocument hit = (SearchResultDocument) result;
String uri= (String) hit.getFirstValue(term.URI); String uri= (String) hit.getFirstValue(term.URI);
ent = new IndividualImpl(); ent = new IndividualImpl();

9
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/NameBoost.java
View file

@ -2,10 +2,9 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding; package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
public class NameBoost implements DocumentModifier { public class NameBoost implements DocumentModifier {
@ -27,11 +26,11 @@ public class NameBoost implements DocumentModifier {
} }
@Override @Override
public void modifyDocument(Individual individual, SolrInputDocument doc, public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) { StringBuffer addUri) {
for( String fieldName : fieldsToBoost){ for( String fieldName : fieldsToBoost){
SolrInputField field = doc.getField(fieldName); SearchInputField field = doc.getField(fieldName);
if( field != null ){ if( field != null ){
field.setBoost(field.getBoost() + boost); field.setBoost(field.getBoost() + boost);
} }

4
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/NameFields.java
View file

@ -8,9 +8,9 @@ import java.io.InputStreamReader;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService.ResultFormat;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException;
@ -31,7 +31,7 @@ public class NameFields implements DocumentModifier {
} }
@Override @Override
public void modifyDocument(Individual ind, SolrInputDocument doc, public void modifyDocument(Individual ind, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException { StringBuffer addUri) throws SkipIndividualException {
if( ind == null || ind.getURI() == null ){ if( ind == null || ind.getURI() == null ){
return; return;

5
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/SourceInstitution.java
View file

@ -2,9 +2,8 @@
package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding; package edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
public class SourceInstitution implements DocumentModifier { public class SourceInstitution implements DocumentModifier {
@ -22,7 +21,7 @@ public class SourceInstitution implements DocumentModifier {
} }
@Override @Override
public void modifyDocument(Individual individual, SolrInputDocument doc, public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException { StringBuffer addUri) throws SkipIndividualException {
doc.addField(VitroSearchTermNames.SITE_URL, siteURL); doc.addField(VitroSearchTermNames.SITE_URL, siteURL);

6
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/documentBuilding/ThumbnailImageURL.java
View file

@ -9,13 +9,13 @@ import java.util.Iterator;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrInputDocument;
import com.hp.hpl.jena.query.QuerySolution; import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.ResultSet; import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.RDFNode; import com.hp.hpl.jena.rdf.model.RDFNode;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -45,7 +45,7 @@ public class ThumbnailImageURL implements DocumentModifier {
} }
@Override @Override
public void modifyDocument(Individual individual, SolrInputDocument doc, public void modifyDocument(Individual individual, SearchInputDocument doc,
StringBuffer addUri) throws SkipIndividualException { StringBuffer addUri) throws SkipIndividualException {
//add a field for storing the location of thumbnail for the individual. //add a field for storing the location of thumbnail for the individual.
@ -56,7 +56,7 @@ public class ThumbnailImageURL implements DocumentModifier {
/** /**
* Adds if the individual has a thumbnail image or not. * Adds if the individual has a thumbnail image or not.
*/ */
protected void addThumbnailExistence(Individual ind, SolrInputDocument doc) { protected void addThumbnailExistence(Individual ind, SearchInputDocument doc) {
try{ try{
if(ind.hasThumb()) if(ind.hasThumb())
doc.addField(THUMBNAIL, "1"); doc.addField(THUMBNAIL, "1");

7
webapp/src/edu/cornell/mannlib/vitro/webapp/searchindex/SearchIndexerSetup.java
View file

@ -9,8 +9,6 @@ import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener; import javax.servlet.ServletContextListener;
import org.apache.solr.client.solrj.impl.HttpSolrServer;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.vocabulary.OWL; import com.hp.hpl.jena.vocabulary.OWL;
@ -23,6 +21,7 @@ import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters; import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext; import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils;
@ -75,7 +74,7 @@ public class SearchIndexerSetup implements ServletContextListener {
public void contextInitialized(ServletContextEvent sce) { public void contextInitialized(ServletContextEvent sce) {
ServletContext context = sce.getServletContext(); ServletContext context = sce.getServletContext();
StartupStatus ss = StartupStatus.getBean(context); StartupStatus ss = StartupStatus.getBean(context);
HttpSolrServer server = (HttpSolrServer) ApplicationUtils.instance().getSearchEngine(); SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try { try {
/* set up the individual to solr doc translation */ /* set up the individual to solr doc translation */
@ -107,7 +106,7 @@ public class SearchIndexerSetup implements ServletContextListener {
modifiersFromContext, searchIndexExcludesFromContext); modifiersFromContext, searchIndexExcludesFromContext);
/* setup solr indexer */ /* setup solr indexer */
SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc); SolrIndexer solrIndexer = new SolrIndexer(searchEngine, indToSolrDoc);
// This is where the builder gets the list of places to try to // This is where the builder gets the list of places to try to
// get objects to index. It is filtered so that non-public text // get objects to index. It is filtered so that non-public text

17
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/DataGetterUtils.java
View file

@ -8,7 +8,6 @@ import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
@ -56,7 +55,7 @@ public class DataGetterUtils {
* exception if a page has PageDataGetters. * exception if a page has PageDataGetters.
*/ */
public static List<DataGetter> getDataGettersForPage(VitroRequest vreq, Model displayModel, String pageURI) public static List<DataGetter> getDataGettersForPage(VitroRequest vreq, Model displayModel, String pageURI)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException { throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
if( vreq.getAttribute(DATA_GETTERS_FOR_PAGE) != null){ if( vreq.getAttribute(DATA_GETTERS_FOR_PAGE) != null){
return (List<DataGetter>) vreq.getAttribute(DATA_GETTERS_FOR_PAGE); return (List<DataGetter>) vreq.getAttribute(DATA_GETTERS_FOR_PAGE);
@ -74,7 +73,7 @@ public class DataGetterUtils {
* This allows the individual profile for an individual of a specific class to be returned . * This allows the individual profile for an individual of a specific class to be returned .
*/ */
public static List<DataGetter> getDataGettersForClass( VitroRequest vreq, Model displayModel, String classURI) public static List<DataGetter> getDataGettersForClass( VitroRequest vreq, Model displayModel, String classURI)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException{ throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, classURI); List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, classURI);
List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris); List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris);
log.debug("getDataGettersForClass: " + dgList); log.debug("getDataGettersForClass: " + dgList);
@ -86,7 +85,7 @@ public class DataGetterUtils {
* @param templateName a filename like "index.ftl", which will be used as a URI like "freemarker:index.ftl". * @param templateName a filename like "index.ftl", which will be used as a URI like "freemarker:index.ftl".
*/ */
public static List<DataGetter> getDataGettersForTemplate( VitroRequest vreq, Model displayModel, String templateName) public static List<DataGetter> getDataGettersForTemplate( VitroRequest vreq, Model displayModel, String templateName)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException, NoSuchMethodException{ throws InstantiationException, IllegalAccessException, ClassNotFoundException, IllegalArgumentException, SecurityException, InvocationTargetException {
String templateUri = "freemarker:" + templateName; String templateUri = "freemarker:" + templateName;
List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, templateUri); List<String> dgUris = getDataGetterURIsForAssociatedURI( displayModel, templateUri);
List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris); List<DataGetter> dgList = dataGettersForURIs(vreq, displayModel, dgUris);
@ -197,7 +196,7 @@ public class DataGetterUtils {
}finally{ displayModel.leaveCriticalSection(); } }finally{ displayModel.leaveCriticalSection(); }
return chooseType( types, displayModel, dataGetterURI); return chooseType( types, dataGetterURI);
} }
@ -229,7 +228,7 @@ public class DataGetterUtils {
return dgURIs; return dgURIs;
} }
private static String chooseType(List<String> types, Model displayModel, String dataGetterURI) throws IllegalAccessException { private static String chooseType(List<String> types, String dataGetterURI) throws IllegalAccessException {
//currently just get the first one that is not owl:Thing //currently just get the first one that is not owl:Thing
for(String type : types){ for(String type : types){
if( ! StringUtils.isEmpty( type ) && !type.equals( OWL.Thing.getURI() )) if( ! StringUtils.isEmpty( type ) && !type.equals( OWL.Thing.getURI() ))
@ -270,8 +269,8 @@ public class DataGetterUtils {
/** /**
* Get Individual count for Solr query for intersection of multiple classes * Get Individual count for Solr query for intersection of multiple classes
*/ */
public static long getIndividualCountForIntersection(VitroRequest vreq, ServletContext context, List<String> classUris) { public static long getIndividualCountForIntersection(VitroRequest vreq, List<String> classUris) {
return IndividualListController.getIndividualCount(classUris, vreq.getWebappDaoFactory().getIndividualDao(), context); return IndividualListController.getIndividualCount(classUris, vreq.getWebappDaoFactory().getIndividualDao());
} }
//Return data getter type to be employed in display model //Return data getter type to be employed in display model
@ -410,7 +409,7 @@ public class DataGetterUtils {
/* /*
* Copied from JSONServlet as expect this to be related to VitroClassGroup * Copied from JSONServlet as expect this to be related to VitroClassGroup
*/ */
public static JSONObject processVClassGroupJSON(VitroRequest vreq, ServletContext context, VClassGroup vcg) { public static JSONObject processVClassGroupJSON(VClassGroup vcg) {
JSONObject map = new JSONObject(); JSONObject map = new JSONObject();
try { try {
ArrayList<JSONObject> classes = new ArrayList<JSONObject>(vcg.size()); ArrayList<JSONObject> classes = new ArrayList<JSONObject>(vcg.size());

8
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/IndividualsForClassesDataGetter.java
View file

@ -163,7 +163,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
protected void processClassesAndRestrictions(VitroRequest vreq, ServletContext context, protected void processClassesAndRestrictions(VitroRequest vreq, ServletContext context,
HashMap<String, Object> data, List<String> classes, List<String> restrictClasses ) { HashMap<String, Object> data, List<String> classes, List<String> restrictClasses ) {
processClassesForDisplay(vreq, context, data, classes); processClassesForDisplay(vreq, context, data, classes);
processRestrictionClasses(vreq, context, data, restrictClasses); processRestrictionClasses(vreq, data, restrictClasses);
processIntersections(vreq, context, data); processIntersections(vreq, context, data);
} }
@ -217,7 +217,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
for(VClass r: restrictClasses) { for(VClass r: restrictClasses) {
classUris.add(r.getURI()); classUris.add(r.getURI());
} }
long count = DataGetterUtils.getIndividualCountForIntersection(vreq, context, classUris); long count = DataGetterUtils.getIndividualCountForIntersection(vreq, classUris);
return new Long(count).intValue(); return new Long(count).intValue();
} }
@ -254,7 +254,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
data.put("vClassGroup", classesGroup); data.put("vClassGroup", classesGroup);
} }
private void processRestrictionClasses(VitroRequest vreq, ServletContext context, private void processRestrictionClasses(VitroRequest vreq,
HashMap<String, Object> data, List<String> restrictClasses) { HashMap<String, Object> data, List<String> restrictClasses) {
try { try {
VClassGroup restrictClassesGroup = new VClassGroup(); VClassGroup restrictClassesGroup = new VClassGroup();
@ -305,7 +305,7 @@ public class IndividualsForClassesDataGetter extends DataGetterBase implements D
} }
} }
public static VClassGroupTemplateModel getClassGroup(String classGroupUri, ServletContext context, VitroRequest vreq){ public static VClassGroupTemplateModel getClassGroup(String classGroupUri, VitroRequest vreq){
VClassGroupsForRequest vcgc = VClassGroupCache.getVClassGroups(vreq); VClassGroupsForRequest vcgc = VClassGroupCache.getVClassGroups(vreq);
List<VClassGroup> vcgList = vcgc.getGroups(); List<VClassGroup> vcgList = vcgc.getGroups();

3
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/dataGetter/SolrIndividualsDataGetter.java
View file

@ -176,8 +176,7 @@ public class SolrIndividualsDataGetter extends DataGetterBase implements DataGet
vclass.getURI(), vclass.getURI(),
page, page,
alpha, alpha,
vreq.getWebappDaoFactory().getIndividualDao(), vreq.getWebappDaoFactory().getIndividualDao());
vreq.getSession().getServletContext());
body.putAll(vcResults.asFreemarkerMap()); body.putAll(vcResults.asFreemarkerMap());
List<Individual> inds = vcResults.getEntities(); List<Individual> inds = vcResults.getEntities();

61
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/solr/SolrQueryUtils.java
View file

@ -7,21 +7,20 @@ import java.util.Collection;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import javax.servlet.ServletContext;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListQueryResults; import edu.cornell.mannlib.vitro.webapp.controller.freemarker.IndividualListQueryResults;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao; import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngine;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineException;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery.Order;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames; import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/** /**
* Some static method to help in constructing Solr queries and parsing the * Some static method to help in constructing Solr queries and parsing the
@ -63,7 +62,7 @@ public class SolrQueryUtils {
* the result, according to the fieldMap. * the result, according to the fieldMap.
*/ */
public static List<Map<String, String>> parseResponse( public static List<Map<String, String>> parseResponse(
QueryResponse queryResponse, FieldMap fieldMap) { SearchResponse queryResponse, FieldMap fieldMap) {
return new SolrResultsParser(queryResponse, fieldMap).parse(); return new SolrResultsParser(queryResponse, fieldMap).parse();
} }
@ -75,7 +74,7 @@ public class SolrQueryUtils {
* the result, according to the fieldMap. * the result, according to the fieldMap.
*/ */
public static List<Map<String, String>> parseAndFilterResponse( public static List<Map<String, String>> parseAndFilterResponse(
QueryResponse queryResponse, FieldMap fieldMap, SearchResponse queryResponse, FieldMap fieldMap,
SolrResponseFilter filter, int maxNumberOfResults) { SolrResponseFilter filter, int maxNumberOfResults) {
return new SolrResultsParser(queryResponse, fieldMap) return new SolrResultsParser(queryResponse, fieldMap)
.parseAndFilterResponse(filter, maxNumberOfResults); .parseAndFilterResponse(filter, maxNumberOfResults);
@ -139,12 +138,12 @@ public class SolrQueryUtils {
} }
//Get count of individuals without actually getting the results //Get count of individuals without actually getting the results
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) { public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao) {
SolrQuery query = new SolrQuery(makeMultiClassQuery(vclassUris)); SearchEngine solr = ApplicationUtils.instance().getSearchEngine();
SearchQuery query = solr.createQuery(makeMultiClassQuery(vclassUris));
query.setRows(0); query.setRows(0);
try { try {
SolrServer solr = SolrSetup.getSolrServer(context); SearchResponse response = null;
QueryResponse response = null;
response = solr.query(query); response = solr.query(query);
return response.getResults().getNumFound(); return response.getResults().getNumFound();
} catch(Exception ex) { } catch(Exception ex) {
@ -157,8 +156,9 @@ public class SolrQueryUtils {
* builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by * builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by
* alpha, and just the hits for the page for pageSize. * alpha, and just the hits for the page for pageSize.
*/ */
public static SolrQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){ public static SearchQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
String queryText = ""; String queryText = "";
SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try { try {
queryText = makeMultiClassQuery(vclassUris); queryText = makeMultiClassQuery(vclassUris);
@ -168,31 +168,32 @@ public class SolrQueryUtils {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
} }
SolrQuery query = new SolrQuery(queryText); SearchQuery query = searchEngine.createQuery(queryText);
//page count starts at 1, row count starts at 0 //page count starts at 1, row count starts at 0
int startRow = (page-1) * pageSize ; int startRow = (page-1) * pageSize ;
query.setStart( startRow ).setRows( pageSize ); query.setStart( startRow ).setRows( pageSize );
// Need a single-valued field for sorting // Need a single-valued field for sorting
query.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc); query.addSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, Order.ASC);
log.debug("Query is " + query.toString()); log.debug("Query is " + query.toString());
return query; return query;
} catch (Exception ex){ } catch (Exception ex){
log.error("Could not make Solr query",ex); log.error("Could not make Solr query",ex);
return new SolrQuery(); return searchEngine.createQuery();
} }
} }
public static SolrQuery getRandomQuery(List<String> vclassUris, int page, int pageSize){ public static SearchQuery getRandomQuery(List<String> vclassUris, int page, int pageSize){
String queryText = ""; String queryText = "";
SearchEngine searchEngine = ApplicationUtils.instance().getSearchEngine();
try { try {
queryText = makeMultiClassQuery(vclassUris); queryText = makeMultiClassQuery(vclassUris);
log.debug("queryText is " + queryText); log.debug("queryText is " + queryText);
SolrQuery query = new SolrQuery(queryText); SearchQuery query = searchEngine.createQuery(queryText);
//page count starts at 1, row count starts at 0 //page count starts at 1, row count starts at 0
query.setStart( page ).setRows( pageSize ); query.setStart( page ).setRows( pageSize );
@ -202,7 +203,7 @@ public class SolrQueryUtils {
} catch (Exception ex){ } catch (Exception ex){
log.error("Could not make the Solr query",ex); log.error("Could not make the Solr query",ex);
return new SolrQuery(); return searchEngine.createQuery();
} }
} }
@ -221,11 +222,10 @@ public class SolrQueryUtils {
} }
public static IndividualListQueryResults buildAndExecuteVClassQuery( public static IndividualListQueryResults buildAndExecuteVClassQuery(
List<String> vclassURIs, String alpha, int page, int pageSize, List<String> vclassURIs, String alpha, int page, int pageSize, IndividualDao indDao)
ServletContext context, IndividualDao indDao) throws SearchEngineException {
throws SolrServerException { SearchQuery query = SolrQueryUtils.getQuery(vclassURIs, alpha, page, pageSize);
SolrQuery query = SolrQueryUtils.getQuery(vclassURIs, alpha, page, pageSize); IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
log.debug("Executed solr query for " + vclassURIs); log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) { if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs); log.debug("entities list is null for vclass " + vclassURIs);
@ -234,11 +234,10 @@ public class SolrQueryUtils {
} }
public static IndividualListQueryResults buildAndExecuteRandomVClassQuery( public static IndividualListQueryResults buildAndExecuteRandomVClassQuery(
List<String> vclassURIs, int page, int pageSize, List<String> vclassURIs, int page, int pageSize, IndividualDao indDao)
ServletContext context, IndividualDao indDao) throws SearchEngineException {
throws SolrServerException { SearchQuery query = SolrQueryUtils.getRandomQuery(vclassURIs, page, pageSize);
SolrQuery query = SolrQueryUtils.getRandomQuery(vclassURIs, page, pageSize); IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao);
IndividualListQueryResults results = IndividualListQueryResults.runQuery(query, indDao, context);
log.debug("Executed solr query for " + vclassURIs); log.debug("Executed solr query for " + vclassURIs);
if (results.getIndividuals().isEmpty()) { if (results.getIndividuals().isEmpty()) {
log.debug("entities list is null for vclass " + vclassURIs); log.debug("entities list is null for vclass " + vclassURIs);

41
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/solr/SolrResultsParser.java
View file

@ -9,9 +9,10 @@ import java.util.Map;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import org.apache.solr.common.SolrDocumentList; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResultDocumentList;
/** /**
* Parse this Solr response, creating a map of values for each document. * Parse this Solr response, creating a map of values for each document.
@ -22,10 +23,10 @@ import org.apache.solr.common.SolrDocumentList;
public class SolrResultsParser { public class SolrResultsParser {
private static final Log log = LogFactory.getLog(SolrResultsParser.class); private static final Log log = LogFactory.getLog(SolrResultsParser.class);
private final QueryResponse queryResponse; private final SearchResponse queryResponse;
private final Map<String, String> fieldNameMapping; private final Map<String, String> fieldNameMapping;
public SolrResultsParser(QueryResponse queryResponse, FieldMap fieldMap) { public SolrResultsParser(SearchResponse queryResponse, FieldMap fieldMap) {
this.queryResponse = queryResponse; this.queryResponse = queryResponse;
this.fieldNameMapping = fieldMap.map(); this.fieldNameMapping = fieldMap.map();
} }
@ -41,14 +42,14 @@ public class SolrResultsParser {
return maps; return maps;
} }
SolrDocumentList docs = queryResponse.getResults(); SearchResultDocumentList docs = queryResponse.getResults();
if (docs == null) { if (docs == null) {
log.debug("Docs for a search was null"); log.debug("Docs for a search was null");
return maps; return maps;
} }
log.debug("Total number of hits = " + docs.getNumFound()); log.debug("Total number of hits = " + docs.getNumFound());
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
maps.add(parseSingleDocument(doc)); maps.add(parseSingleDocument(doc));
} }
@ -69,14 +70,14 @@ public class SolrResultsParser {
return maps; return maps;
} }
SolrDocumentList docs = queryResponse.getResults(); SearchResultDocumentList docs = queryResponse.getResults();
if (docs == null) { if (docs == null) {
log.debug("Docs for a search was null"); log.debug("Docs for a search was null");
return maps; return maps;
} }
log.debug("Total number of hits = " + docs.getNumFound()); log.debug("Total number of hits = " + docs.getNumFound());
for (SolrDocument doc : docs) { for (SearchResultDocument doc : docs) {
Map<String, String> map = parseSingleDocument(doc); Map<String, String> map = parseSingleDocument(doc);
if (filter.accept(map)) { if (filter.accept(map)) {
maps.add(map); maps.add(map);
@ -92,7 +93,7 @@ public class SolrResultsParser {
/** /**
* Create a map from this document, applying translation on the field names. * Create a map from this document, applying translation on the field names.
*/ */
private Map<String, String> parseSingleDocument(SolrDocument doc) { private Map<String, String> parseSingleDocument(SearchResultDocument doc) {
Map<String, String> result = new HashMap<String, String>(); Map<String, String> result = new HashMap<String, String>();
for (String solrFieldName : fieldNameMapping.keySet()) { for (String solrFieldName : fieldNameMapping.keySet()) {
String jsonFieldName = fieldNameMapping.get(solrFieldName); String jsonFieldName = fieldNameMapping.get(solrFieldName);
@ -106,8 +107,8 @@ public class SolrResultsParser {
/** /**
* Find a single value in the document * Find a single value in the document
*/ */
private String parseSingleValue(SolrDocument doc, String key) { private String parseSingleValue(SearchResultDocument doc, String key) {
Object rawValue = getFirstValue(doc.get(key)); Object rawValue = doc.getFirstValue(key);
if (rawValue == null) { if (rawValue == null) {
return ""; return "";
@ -118,20 +119,4 @@ public class SolrResultsParser {
return String.valueOf(rawValue); return String.valueOf(rawValue);
} }
/**
* The result might be a list. If so, get the first element.
*/
private Object getFirstValue(Object rawValue) {
if (rawValue instanceof List<?>) {
List<?> list = (List<?>) rawValue;
if (list.isEmpty()) {
return null;
} else {
return list.get(0);
}
} else {
return rawValue;
}
}
} }

27
webapp/test/edu/cornell/mannlib/vitro/webapp/controller/grefine/JSONReconcileServletTest.java
View file

@ -10,18 +10,19 @@ import java.util.regex.PatternSyntaxException;
import javax.servlet.ServletException; import javax.servlet.ServletException;
import org.apache.solr.client.solrj.SolrQuery;
import org.json.JSONException; import org.json.JSONException;
import org.json.JSONObject; import org.json.JSONObject;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletContextStub;
import stubs.javax.servlet.http.HttpServletRequestStub; import stubs.javax.servlet.http.HttpServletRequestStub;
import stubs.javax.servlet.http.HttpServletResponseStub; import stubs.javax.servlet.http.HttpServletResponseStub;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass; import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
/** /**
* Tests on various methods in the class. * Tests on various methods in the class.
@ -36,6 +37,8 @@ public class JSONReconcileServletTest extends AbstractTestClass {
@Before @Before
public void setup() throws Exception { public void setup() throws Exception {
ApplicationStub.setup(new ServletContextStub(), new SearchEngineStub());
request = new HttpServletRequestStub(); request = new HttpServletRequestStub();
request.setRequestUrl(new URL("http://vivo.this.that/reconcile")); request.setRequestUrl(new URL("http://vivo.this.that/reconcile"));
request.setMethod("POST"); request.setMethod("POST");
@ -78,23 +81,17 @@ public class JSONReconcileServletTest extends AbstractTestClass {
propertiesList.add(properties); propertiesList.add(properties);
// test getQuery // test getQuery
SolrQuery solrQuery = reconcile.getQuery(nameStr, nameType, rowNum, propertiesList); SearchQuery searchQuery = reconcile.getQuery(nameStr, nameType, rowNum, propertiesList);
String messagePrefix = "Query should contain the text: "; String messagePrefix = "Query should contain the text: ";
testAssertTrue(messagePrefix + orgStr, orgStr, solrQuery.toString()); testAssertTrue(messagePrefix + orgStr, orgStr, searchQuery.toString());
testAssertTrue(messagePrefix + nameStr, nameStr, solrQuery.toString()); testAssertTrue(messagePrefix + nameStr, nameStr, searchQuery.toString());
testAssertTrue(messagePrefix + orgType, orgType, solrQuery.toString()); testAssertTrue(messagePrefix + orgType, orgType, searchQuery.toString());
testAssertTrue(messagePrefix + nameType, orgType, solrQuery.toString()); testAssertTrue(messagePrefix + nameType, orgType, searchQuery.toString());
} }
private void testAssertTrue(String message, String inputStr, String resultStr) { private void testAssertTrue(String message, String inputStr, String resultStr) {
try { try {
String modStr = null; Pattern regex = Pattern.compile(inputStr, Pattern.CASE_INSENSITIVE);
if (inputStr.contains(":") && inputStr.contains("/")) {
modStr = inputStr.replaceAll(":", "%3A").replaceAll("/", "%2F");
} else {
modStr = inputStr;
}
Pattern regex = Pattern.compile(modStr, Pattern.CASE_INSENSITIVE);
Matcher regexMatcher = regex.matcher(resultStr); Matcher regexMatcher = regex.matcher(resultStr);
Assert.assertTrue(message, regexMatcher.find()); Assert.assertTrue(message, regexMatcher.find());
} catch (PatternSyntaxException e) { } catch (PatternSyntaxException e) {

10
webapp/test/edu/cornell/mannlib/vitro/webapp/controller/json/JsonServletTest.java
View file

@ -20,16 +20,16 @@ import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.dao.VClassDaoStub; import stubs.edu.cornell.mannlib.vitro.webapp.dao.VClassDaoStub;
import stubs.edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactoryStub; import stubs.edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactoryStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletConfigStub; import stubs.javax.servlet.ServletConfigStub;
import stubs.javax.servlet.ServletContextStub; import stubs.javax.servlet.ServletContextStub;
import stubs.javax.servlet.http.HttpServletRequestStub; import stubs.javax.servlet.http.HttpServletRequestStub;
import stubs.javax.servlet.http.HttpServletResponseStub; import stubs.javax.servlet.http.HttpServletResponseStub;
import stubs.javax.servlet.http.HttpSessionStub; import stubs.javax.servlet.http.HttpSessionStub;
import stubs.org.apache.solr.client.solrj.SolrServerStub;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass; import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess; import edu.cornell.mannlib.vitro.webapp.dao.ModelAccess;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
/** /**
* TODO * TODO
@ -85,7 +85,7 @@ public class JsonServletTest extends AbstractTestClass {
private WebappDaoFactoryStub wadf; private WebappDaoFactoryStub wadf;
private VClassDaoStub vcDao; private VClassDaoStub vcDao;
private SolrServerStub solr; private SearchEngineStub solr;
@Before @Before
public void setup() throws ServletException { public void setup() throws ServletException {
@ -112,8 +112,8 @@ public class JsonServletTest extends AbstractTestClass {
vcDao = new VClassDaoStub(); vcDao = new VClassDaoStub();
wadf.setVClassDao(vcDao); wadf.setVClassDao(vcDao);
solr = new SolrServerStub(); solr = new SearchEngineStub();
ctx.setAttribute(SolrSetup.SOLR_SERVER, solr); ApplicationStub.setup(new ServletContextStub(), solr);
} }
@Test @Test

327
webapp/test/edu/cornell/mannlib/vitro/webapp/search/solr/SolrQueryTest.java
View file

@ -1,327 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.CoreContainer;
import org.junit.After;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.xml.sax.SAXException;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.ResIterator;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.util.FileManager;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactoryJena;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.model.RDFServiceModel;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.solr.documentBuilding.IndividualToSolrDocument;
/**
* This tests what results will be returned for queries to the
* Solr service.
*
* The test will setup a temporary Solr service, setup a temporary RDF store, load the
* store with some individuals, and then index the individuals in the store.
*
* Once this is done a set of test queries will be run against that service.
*
* This code is based on a similar test from the CUL Discovery and Access
* integration layer.
*
* All RDF/XML files in webapp/test/testontologies/SolrQueryTestRDF will be
* loaded into the model.
*/
//ignored for now since the solr directory isn't yet found when running from vivo.
@Ignore
public class SolrQueryTest extends AbstractTestClass {
/** Key of system property for build directory. */
final static String buildDirSystemPropertyKey = "vitro.build.dir";
/** Solr index to run test queries on. */
static SolrServer solr = null;
/** Container for solr */
static CoreContainer coreContainer = null;
/** Folder to store the temporary Solr index. */
public static TemporaryFolder solrIndexFolder = null;
/**
* This test needs to load RDF data to use as a
* source of individuals to to build documents.
* This is relative to the build directory.
*/
final static String testRDFDir =
"/webapp/test/testontologies/SolrQueryTestRDF";
@Before
public void setup() throws Exception{
//solr makes a lot of output
suppressSysout();
suppressSyserr();
File buildDir = findBuildDir();
solrIndexFolder = new TemporaryFolder();
solrIndexFolder.create();
File tempSolrBase = solrIndexFolder.newFolder("tempSolrBase");
FileUtils.copyDirectory(getSolrTemplateDir(buildDir), tempSolrBase );
solr = setupSolr( tempSolrBase );
indexRdf( loadTestRDF( buildDir ) );
}
@After
public void takedown() throws Exception{
if( coreContainer != null )
coreContainer.shutdown();
restoreOutputStreams();
if( solrIndexFolder != null )
solrIndexFolder.delete();
}
/**
* This will return the directory to use as the Solr
* home template directory.
*
* Throws an exception if the directory is not found.
* @param buildDir - must not be null, must be the base of the build.
*/
private File getSolrTemplateDir(File buildDir) throws Exception {
if(buildDir == null || !buildDir.exists() )
throw new Exception("buildDir must not be null");
String solrTemplateDirName = buildDir.getAbsolutePath() + "/solr/homeDirectoryTemplate";
File solrTemplateDir = new File(solrTemplateDirName);
if( solrTemplateDir == null || ! solrTemplateDir.exists())
throw new Exception("Solr home directory template " +
"was not found at " + solrTemplateDirName);
else
return solrTemplateDir;
}
protected SolrServer setupSolr(File solrBase)
throws ParserConfigurationException, IOException, SAXException{
System.setProperty("solr.solr.home", solrBase.getAbsolutePath());
CoreContainer.Initializer initializer = new CoreContainer.Initializer();
coreContainer = initializer.initialize();
return new EmbeddedSolrServer(coreContainer, "");
}
private OntModel loadTestRDF(File buildDir) throws Exception {
String dirname = buildDir.getAbsolutePath() + testRDFDir;
File rdfDir = new File( dirname );
assertNotNull("could not find dir " + dirname , rdfDir);
assertTrue(dirname + " must be a directory." ,rdfDir.isDirectory());
//load all files in test dir.
File[] files = rdfDir.listFiles();
assertNotNull("no test RDF files found",files);
assertTrue("no test RDF files found", files.length > 0 );
OntModel model = ModelFactory.createOntologyModel();
for (File file : files ){
InputStream in = FileManager.get().open( file.getAbsolutePath() );
assertNotNull("Could not load file " + file.getAbsolutePath(), in );
try{
if( file.getName().endsWith(".rdf") ){
model.read(in,null);
}else if( file.getName().endsWith(".n3")){
model.read(in,null,"N3");
}else if( file.getName().endsWith(".nt")){
model.read(in,null,"N-TRIPLE");
}else if( file.getName().endsWith(".ttl")){
model.read(in,null,"TURTLE");
}else{
throw new Exception("Format unknown for file name " + file.getName());
}
}catch(Throwable th){
throw new Exception( "Could not load RDF file "
+ file.getAbsolutePath() , th);
}
}
return model ;
}
/**
* Find the base of the build directories.
* @return This method will return a non-null file to use
* or it will throw an exception if none can be found.
* @throws Exception
*/
private File findBuildDir() throws Exception {
//First try to find the base directory
//of the build in the system properties.
String buildDirName = System.getProperty(buildDirSystemPropertyKey);
if( buildDirName != null ){
File buildDir = new File(buildDirName);
if( buildDir.exists() && buildDir.isDirectory() ){
return buildDir;
}
}
//If there is no system property try to
//guess the location based on the working directory
File f = null;
String[] fallBackBases = {"","../","../../"};
List<String> attempted = new ArrayList<String>();
for( String base: fallBackBases){
String attemptedDir =base + "solr/homeDirectoryTemplate";
f = new File( attemptedDir );
attempted.add( System.getProperty("user.dir") + '/' + attemptedDir );
if( f != null && f.exists() && f.isDirectory() ){
f = new File( base );
break;
}else{
f = null;
}
}
if( f == null ){
throw new Exception(
"Could not find the base of the " +
"build directory for the project, " +
"checked the system property " + buildDirSystemPropertyKey
+ " and checked in these locations: \n" +
StringUtils.join(attempted,"\n "));
}else{
return f;
}
}
/** Query the RDF, build Solr Documents from results, load them to Solr. */
private void indexRdf(OntModel model) throws Exception {
RDFServiceModel rdfService = new RDFServiceModel(model);
IndividualToSolrDocument r2d =
SolrSetup.setupTransltion(
model,
model,
new RDFServiceFactorySingle(rdfService),
null, null);
WebappDaoFactory wdf = new WebappDaoFactoryJena(model);
for( String uri: getURISToIndex( model ) ){
SolrInputDocument doc;
try {
doc = r2d.translate( wdf.getIndividualDao().getIndividualByURI(uri));
} catch (Exception e) {
throw new Exception("Failed on building document for uri:" + uri, e);
}
try {
solr.add( doc );
} catch (Exception e) {
throw new Exception("Failed adding doc to solr for uri:" + uri, e);
}
}
solr.commit();
}
private List<String> getURISToIndex(Model model) {
//just return all the URIs in the subject position
List<String> uris = new LinkedList<String>();
ResIterator it = model.listSubjects();
while(it.hasNext() ){
Resource res = it.nextResource();
if( res != null && res.isURIResource() ){
uris.add( res.getURI() );
}
}
return uris;
}
/**
* Test that a document with the given URIs are in the results for the query.
* @throws SolrServerException */
void testQueryGetsDocs(String errmsg, SolrQuery query, String[] expectedUris) throws SolrServerException{
assertNotNull(errmsg + " but query was null", query);
assertNotNull(errmsg + " but expected URIs was null", expectedUris );
QueryResponse resp = solr.query(query);
if( resp == null )
fail( errmsg + " but Could not get a solr response");
Set<String> uris = new HashSet<String>(Arrays.asList( expectedUris ));
for( SolrDocument doc : resp.getResults()){
assertNotNull(errmsg + ": solr doc was null", doc);
String uri = (String) doc.getFirstValue( VitroSearchTermNames.URI );
assertNotNull(errmsg+": no URI field in solr doc" , uri);
uris.remove( uri );
}
if( uris.size() > 0){
String errorMsg =
"\nThe query '"+ query + "' was expected " +
"to return the following URIs but did not:";
for( String uri : uris){
errorMsg= errorMsg+"\n" + uri;
}
fail( errmsg + errorMsg);
}
}
@Test
public void testSolrWasStarted() throws SolrServerException, IOException {
assertNotNull( solr );
solr.ping();//this will throw an exception of the server is not reachable
}
@Test
public void testCorsonSearch() throws SolrServerException{
/* make sure that we have the document in the index before we do anything */
SolrQuery query = new SolrQuery().setQuery("corson");
testQueryGetsDocs("Expect to find a doc when searching for 'corson'",
query,new String[]{ "http://vivo.cornell.edu/individual/individual22972" } ) ;
}
@Test
public void testFormFeed() throws SolrServerException{
/* make sure that we have the document in the index before we do anything */
SolrQuery query = new SolrQuery().setQuery("vivo15");
testQueryGetsDocs("Expect to find a doc when searching for 'vivo15'",
query,new String[]{ "http://vivo.cornell.edu/individual/individualIssueVivo15" } ) ;
}
}

16
webapp/test/edu/cornell/mannlib/vitro/webapp/search/solr/ThumbnailImageURLTest.java
View file

@ -7,19 +7,24 @@ package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.io.InputStream; import java.io.InputStream;
import org.apache.log4j.Level; import org.apache.log4j.Level;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.ApplicationStub;
import stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineStub;
import stubs.javax.servlet.ServletContextStub;
import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler; import com.hp.hpl.jena.rdf.model.impl.RDFDefaultErrorHandler;
import edu.cornell.mannlib.vitro.testing.AbstractTestClass; import edu.cornell.mannlib.vitro.testing.AbstractTestClass;
import edu.cornell.mannlib.vitro.webapp.application.ApplicationUtils;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl; import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputField;
import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceFactory;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceFactorySingle;
import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.model.RDFServiceModel; import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.jena.model.RDFServiceModel;
@ -37,6 +42,7 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
setLoggerLevel(RDFDefaultErrorHandler.class, Level.OFF); setLoggerLevel(RDFDefaultErrorHandler.class, Level.OFF);
ApplicationStub.setup(new ServletContextStub(), new SearchEngineStub());
Model model = ModelFactory.createDefaultModel(); Model model = ModelFactory.createDefaultModel();
InputStream in = ThumbnailImageURLTest.class.getResourceAsStream("testPerson.n3"); InputStream in = ThumbnailImageURLTest.class.getResourceAsStream("testPerson.n3");
@ -52,7 +58,7 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
*/ */
@Test @Test
public void testThumbnailFieldCreatedInSolrDoc() { public void testThumbnailFieldCreatedInSolrDoc() {
SolrInputDocument doc = new SolrInputDocument(); SearchInputDocument doc = ApplicationUtils.instance().getSearchEngine().createInputDocument();
ThumbnailImageURL testMe = new ThumbnailImageURL( testRDF ); ThumbnailImageURL testMe = new ThumbnailImageURL( testRDF );
Individual ind = new IndividualImpl(); Individual ind = new IndividualImpl();
ind.setURI(personsURI); ind.setURI(personsURI);
@ -66,11 +72,11 @@ public class ThumbnailImageURLTest extends AbstractTestClass{
//make sure that a Solr document field got created for the thumbnail image //make sure that a Solr document field got created for the thumbnail image
SolrInputField thumbnailField = doc.getField( VitroSearchTermNames.THUMBNAIL_URL ); SearchInputField thumbnailField = doc.getField( VitroSearchTermNames.THUMBNAIL_URL );
Assert.assertNotNull(thumbnailField); Assert.assertNotNull(thumbnailField);
Assert.assertNotNull( thumbnailField.getValues() ); Assert.assertNotNull( thumbnailField.getValues() );
Assert.assertEquals(1, thumbnailField.getValueCount()); Assert.assertEquals(1, thumbnailField.getValues().size());
Assert.assertEquals("http://vivo.cornell.edu/individual/n54945", thumbnailField.getFirstValue()); Assert.assertEquals("http://vivo.cornell.edu/individual/n54945", thumbnailField.getFirstValue());
} }

14
webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/modules/searchEngine/SearchEngineStub.java
View file

@ -3,7 +3,6 @@
package stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine; package stubs.edu.cornell.mannlib.vitro.webapp.modules.searchEngine;
import java.util.Collection; import java.util.Collection;
import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -14,6 +13,7 @@ import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchEngineExcepti
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchInputDocument;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchQuery;
import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse; import edu.cornell.mannlib.vitro.webapp.modules.searchEngine.SearchResponse;
import edu.cornell.mannlib.vitro.webapp.searchengine.base.BaseSearchInputDocument;
/** /**
* TODO * TODO
@ -53,6 +53,11 @@ public class SearchEngineStub implements SearchEngine {
} }
} }
@Override
public SearchInputDocument createInputDocument() {
return new BaseSearchInputDocument();
}
// ---------------------------------------------------------------------- // ----------------------------------------------------------------------
// Un-implemented methods // Un-implemented methods
@ -80,13 +85,6 @@ public class SearchEngineStub implements SearchEngine {
throw new RuntimeException("SearchEngineStub.ping() not implemented."); throw new RuntimeException("SearchEngineStub.ping() not implemented.");
} }
@Override
public SearchInputDocument createInputDocument() {
// TODO Auto-generated method stub
throw new RuntimeException(
"SearchEngineStub.createInputDocument() not implemented.");
}
@Override @Override
public void add(SearchInputDocument... docs) throws SearchEngineException { public void add(SearchInputDocument... docs) throws SearchEngineException {
// TODO Auto-generated method stub // TODO Auto-generated method stub

46
webapp/test/stubs/org/apache/solr/client/solrj/SolrServerStub.java
View file

@ -1,46 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package stubs.org.apache.solr.client.solrj;
import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.util.NamedList;
/**
* TODO
*/
public class SolrServerStub extends SolrServer {
private static final Log log = LogFactory.getLog(SolrServerStub.class);
// ----------------------------------------------------------------------
// Stub infrastructure
// ----------------------------------------------------------------------
// ----------------------------------------------------------------------
// Stub methods
// ----------------------------------------------------------------------
/*
* (non-Javadoc)
*
* @see
* org.apache.solr.client.solrj.SolrServer#request(org.apache.solr.client
* .solrj.SolrRequest)
*/
@Override
public NamedList<Object> request(SolrRequest request)
throws SolrServerException, IOException {
// TODO not really an implementation.
return new NamedList<Object>();
}
// ----------------------------------------------------------------------
// Un-implemented methods
// ----------------------------------------------------------------------
}