litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

merging 8813 8810 8803 8802 8741 8731

Browse source
This commit is contained in:
briancaruso 2011-07-22 23:20:13 +00:00
parent ee42cad576
commit 199a5f6c2f
12 changed files with 585 additions and 344 deletions
Download patch file Download diff file Expand all files Collapse all files

13
doc/list_view_configuration_guidelines.txt
View file

@ -5,13 +5,16 @@ List view configuration guidelines
REGISTERING THE LIST VIEW REGISTERING THE LIST VIEW
------------------------- -------------------------
A custom list view is associated with an object property in the file /vivo/productMods/WEB-INF/ontologies/app/listViewConfig.owl. A custom list view is associated with an object property in the RDF files in the directory /vivo/productMods/WEB-INF/ontologies/app/loadedAtStartup
Example: To register a list view, create a new .rdf or .n3 file in that directory. The file must be well formed RDF/XML or N3.
Example of registering a new association in a file named newListViews.n3:
<rdf:Description rdf:about="http://vivoweb.org/ontology/core#authorInAuthorship"> <http://vivoweb.org/ontology/core#authorInAuthorship>
<display:listViewConfigFile rdf:datatype="http://www.w3.org/2001/XMLSchema#string">listViewConfig-authorInAuthorship.xml</display:listViewConfigFile> <http://vitro.mannlib.cornell.edu/ontologies/display/1.1#listViewConfigFile>
</rdf:Description> "listViewConfig-authorInAuthorship.xml" .
Place this file in /vivo/productMods/WEB-INF/ontologies/app/loadedAtStartup, redeploy and restart tomcat to put the new custom list view in effect.
----------------- -----------------
REQUIRED ELEMENTS REQUIRED ELEMENTS

23
webapp/ontologies/app/loadedAtStartup/displayModelListViews.rdf Normal file
View file

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE rdf:RDF [
<!ENTITY display "http://vitro.mannlib.cornell.edu/ontologies/display/1.1#">
<!ENTITY owl "http://www.w3.org/2002/07/owl#">
<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<!ENTITY rdfs "http://www.w3.org/2000/01/rdf-schema#">
<!ENTITY xsd "http://www.w3.org/2001/XMLSchema#">
<!ENTITY vitro "http://vitro.mannlib.cornell.edu/ns/vitro/0.7#">
]>
<rdf:RDF xml:base="http://vitro.mannlib.cornell.edu/ontologies/display/1.1/"
xmlns:display="&display;"
xmlns:owl="&owl;"
xmlns:rdf="&rdf;"
xmlns:rdfs="&rdfs;"
xmlns:vitro="&vitro;">
<rdf:Description rdf:about="http://vitro.mannlib.cornell.edu/ontologies/display/1.1#hasElement">
<display:listViewConfigFile rdf:datatype="http://www.w3.org/2001/XMLSchema#string">listViewConfig-hasElement.xml</display:listViewConfigFile>
</rdf:Description>
</rdf:RDF>

22
webapp/ontologies/search/vitroSearchProhibited.n3
View file

@ -1,23 +1,27 @@
# $This file is distributed under the terms of the license in /doc/license.txt$ # $This file is distributed under the terms of the license in /doc/license.txt$
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix vitroDisplay: <http://vitro.mannlib.cornell.edu/ontologies/display/1.1#> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix core: <http://vivoweb.org/ontology/core#> .
@prefix example: <http://example/ns/> .
# All instances of a class can be excluded from the search index # All instances of a class can be excluded from the search index
# by adding a vitroDisplay:excludeClass property between # by adding a vitroDisplay:excludeClass property between
# vitroDisplay:SearchIndex and the URI of the class # vitroDisplay:SearchIndex and the URI of the class
# that you would like to exclude. # that you would like to exclude.
# All .n3 files in this directory will be used to configure # All .n3 or .rdf files in this directory will be used to configure
# the search exclusions. Only .n3 files will be loaded. # the search exclusions. Each file must be a valid file in the format
# # specified by its extension. Each file will be loaded individually and
# must be a complete stand alone example of its format. Each file must contain all
# the necessary prefixes, namespaces and preambles required by the format
# specified by its extension.
# If you would like to add classes to the # If you would like to add classes to the
# exclusions, add a file to this directory ending in .n3 with # exclusions, add a file to this directory ending in .n3 with
# N3 statements similar to this example. # N3 statements similar to this example.
#
# @prefix owl: <http://www.w3.org/2002/07/owl#> .
# @prefix vitroDisplay: <http://vitro.mannlib.cornell.edu/ontologies/display/1.1#> .
# @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
# @prefix example: <http://example/ns/> .
#
# vitroDisplay:SearchIndex # vitroDisplay:SearchIndex
# rdf:type owl:Thing ; # rdf:type owl:Thing ;
# vitroDisplay:excludeClass example:classToExclude ; # vitroDisplay:excludeClass example:classToExclude ;

17
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/JsonServlet.java
View file

@ -54,6 +54,7 @@ public class JsonServlet extends VitroHttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(JsonServlet.class); private static final Log log = LogFactory.getLog(JsonServlet.class);
private static final int REPLY_SIZE = 256; private static final int REPLY_SIZE = 256;
private static final int INDIVIDUALS_PER_PAGE = 30;
@Override @Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
@ -267,7 +268,7 @@ public class JsonServlet extends VitroHttpServlet {
try { try {
map = IndividualListController.getResultsForVClassIntersections( map = IndividualListController.getResultsForVClassIntersections(
vclassURIs, vclassURIs,
page, page, INDIVIDUALS_PER_PAGE,
alpha, alpha,
vreq.getWebappDaoFactory().getIndividualDao(), vreq.getWebappDaoFactory().getIndividualDao(),
context); context);
@ -292,17 +293,11 @@ public class JsonServlet extends VitroHttpServlet {
} }
public static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){ public static String getDataPropertyValue(Individual ind, DataProperty dp, WebappDaoFactory wdf){
List<Literal> values = wdf.getDataPropertyStatementDao() String value = ind.getDataValue(dp.getURI());
.getDataPropertyValuesForIndividualByProperty(ind, dp); if( value == null || value.isEmpty() )
if( values == null || values.isEmpty() )
return ""; return "";
else{ else
if( values.get(0) != null ) return value;
return values.get(0).getLexicalForm();
else
return "";
}
} }
/** /**

155
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/freemarker/IndividualListController.java
View file

@ -6,6 +6,7 @@ import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.Iterator;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -40,8 +41,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(IndividualListController.class.getName()); private static final Log log = LogFactory.getLog(IndividualListController.class.getName());
private static final int INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS = 30000;
private static final int INDIVIDUALS_PER_PAGE = 30; private static final int INDIVIDUALS_PER_PAGE = 30;
private static final int MAX_PAGES = 40; // must be even private static final int MAX_PAGES = 40; // must be even
@ -165,22 +165,23 @@ public class IndividualListController extends FreemarkerHttpServlet {
} }
} }
//Pulling out common code that is used for both single (regular) vclass query and multiple (intersection) query // //Pulling out common code that is used for both single (regular) vclass query and multiple (intersection) query
public static Map<String,Object> getResultsForVClasses(List<String> vclassURIs, int page, String alpha, IndividualDao indDao, ServletContext context) // public static Map<String,Object> getResultsForVClasses(List<String> vclassURIs, int page, String alpha, IndividualDao indDao, ServletContext context)
throws IOException, ServletException{ // throws IOException, ServletException{
Map<String,Object> rvMap = new HashMap<String,Object>(); // Map<String,Object> rvMap = new HashMap<String,Object>();
try{ // try{
SolrQuery query = getQuery(vclassURIs, alpha); // SolrQuery query = getQuery(vclassURIs, alpha, page, INDIVIDUALS_PER_PAGE);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); // rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); // List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) // if (individuals == null)
log.debug("entities list is null for vclasses " + vclassURIs.toString() ); // log.debug("entities list is null for vclasses " + vclassURIs.toString() );
} catch(Throwable th) { // } catch(Throwable th) {
log.error("An error occurred retrieving results for vclass query", th); // log.error("An error occurred retrieving results for vclass query", th);
} // }
return rvMap; // return rvMap;
} // }
@SuppressWarnings("unchecked")
public static Map<String,Object> getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context) public static Map<String,Object> getResultsForVClass(String vclassURI, int page, String alpha, IndividualDao indDao, ServletContext context)
throws IOException, SearchException{ throws IOException, SearchException{
Map<String,Object> rvMap = new HashMap<String,Object>(); Map<String,Object> rvMap = new HashMap<String,Object>();
@ -188,8 +189,8 @@ public class IndividualListController extends FreemarkerHttpServlet {
//make query for this rdf:type //make query for this rdf:type
List<String> classUris = new ArrayList<String>(); List<String> classUris = new ArrayList<String>();
classUris.add(vclassURI); classUris.add(vclassURI);
SolrQuery query = getQuery(classUris, alpha); SolrQuery query = getQuery(classUris, alpha, page, INDIVIDUALS_PER_PAGE);
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); rvMap = getResultsForVClassQuery(query, page, INDIVIDUALS_PER_PAGE, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) if (individuals == null)
log.debug("entities list is null for vclass " + vclassURI ); log.debug("entities list is null for vclass " + vclassURI );
@ -204,14 +205,15 @@ public class IndividualListController extends FreemarkerHttpServlet {
return rvMap; return rvMap;
} }
public static Map<String,Object> getResultsForVClassIntersections(List<String> vclassURIs, int page, String alpha, IndividualDao indDao, ServletContext context) @SuppressWarnings("unchecked")
public static Map<String,Object> getResultsForVClassIntersections(List<String> vclassURIs, int page, int pageSize, String alpha, IndividualDao indDao, ServletContext context)
throws IOException, ServletException{ throws IOException, ServletException{
Map<String,Object> rvMap = new HashMap<String,Object>(); Map<String,Object> rvMap = new HashMap<String,Object>();
try{ try{
// make query for multiple rdf types // make query for multiple rdf types
SolrQuery query = getQuery(vclassURIs, alpha); SolrQuery query = getQuery(vclassURIs, alpha, page, pageSize);
log.debug("Executed solr query for " + vclassURIs.toString()); log.debug("Executed solr query for " + vclassURIs.toString());
rvMap = getResultsForVClassQuery(query, page, alpha, indDao, context); rvMap = getResultsForVClassQuery(query, page, pageSize, alpha, indDao, context);
List<Individual> individuals = (List<Individual>) rvMap.get("entities"); List<Individual> individuals = (List<Individual>) rvMap.get("entities");
if (individuals == null) if (individuals == null)
log.debug("entities list is null for vclass " + vclassURIs.toString() ); log.debug("entities list is null for vclass " + vclassURIs.toString() );
@ -225,7 +227,7 @@ public class IndividualListController extends FreemarkerHttpServlet {
* This method is now called in a couple of places. It should be refactored * This method is now called in a couple of places. It should be refactored
* into a DAO or similar object. * into a DAO or similar object.
*/ */
public static Map<String,Object> getResultsForVClassQuery(SolrQuery query, int page, String alpha, IndividualDao indDao, ServletContext context) protected static Map<String,Object> getResultsForVClassQuery(SolrQuery query, int page, int pageSize, String alpha, IndividualDao indDao, ServletContext context)
throws IOException, ServletException { throws IOException, ServletException {
Map<String,Object> rvMap = new HashMap<String,Object>(); Map<String,Object> rvMap = new HashMap<String,Object>();
SolrServer solr = SolrSetup.getSolrServer(context); SolrServer solr = SolrSetup.getSolrServer(context);
@ -252,31 +254,26 @@ public class IndividualListController extends FreemarkerHttpServlet {
long hitCount = docs.getNumFound(); long hitCount = docs.getNumFound();
log.debug("Number of search results: " + hitCount); log.debug("Number of search results: " + hitCount);
List<Individual> individuals = new ArrayList<Individual>(INDIVIDUALS_PER_PAGE); Iterator<SolrDocument> docIter = docs.iterator();
int individualsAdded = 0; List<Individual> individuals = new ArrayList<Individual>(docs.size());
int index = (page-1) * INDIVIDUALS_PER_PAGE;
if(docs.size() > 0) { while ( docIter.hasNext() ){
while (individualsAdded < INDIVIDUALS_PER_PAGE && index < hitCount) { SolrDocument doc = docIter.next();
SolrDocument doc = docs.get(index);
if (doc != null) { if (doc != null) {
String uri = doc.get(VitroSearchTermNames.URI).toString(); String uri = doc.get(VitroSearchTermNames.URI).toString();
Individual individual = indDao.getIndividualByURI( uri ); Individual individual = indDao.getIndividualByURI( uri );
if (individual != null) { if (individual != null) {
individualsAdded++; individuals.add( individual );
individuals.add(individual);
log.debug("Adding individual " + uri + " to individual list display"); log.debug("Adding individual " + uri + " to individual list display");
} else { } else {
log.debug("No existing individual for search document with uri = " + uri); log.debug("No existing individual for search document with uri = " + uri);
} }
} }
index++; }
}
} else { if ( hitCount > pageSize ){
log.debug("Docs size is 0");
}
if ( hitCount > INDIVIDUALS_PER_PAGE ){
rvMap.put("showPages", Boolean.TRUE); rvMap.put("showPages", Boolean.TRUE);
List<PageRecord> pageRecords = makePagesList(hitCount, INDIVIDUALS_PER_PAGE, page); List<PageRecord> pageRecords = makePagesList(hitCount, pageSize, page);
rvMap.put("pages", pageRecords); rvMap.put("pages", pageRecords);
}else{ }else{
rvMap.put("showPages", Boolean.FALSE); rvMap.put("showPages", Boolean.FALSE);
@ -292,63 +289,71 @@ public class IndividualListController extends FreemarkerHttpServlet {
} }
//Get count of individuals without actually getting the results //Get count of individuals without actually getting the results
public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) { public static long getIndividualCount(List<String> vclassUris, IndividualDao indDao, ServletContext context) {
SolrQuery query = getQuery(vclassUris, null, 0); SolrQuery query = new SolrQuery(makeMultiClassQuery(vclassUris));
try { query.setRows(0);
Map<String,Object> rvMap = getResultsForVClassQuery(query, 1, null, indDao, context); try {
Long count = (Long) rvMap.get("totalCount"); SolrServer solr = SolrSetup.getSolrServer(context);
return count.longValue(); QueryResponse response = null;
response = solr.query(query);
return response.getResults().getNumFound();
} catch(Exception ex) { } catch(Exception ex) {
log.error("An error occured in retrieving individual count", ex); log.error("An error occured in retrieving individual count", ex);
} }
return 0; return 0;
} }
private static SolrQuery getQuery(List<String> vclassUris, String alpha) { /**
return getQuery(vclassUris, alpha, INDIVIDUAL_LIST_CONTROLLER_MAX_RESULTS); * builds a query with a type clause for each type in vclassUris, NAME_LOWERCASE filetred by
} * alpha, and just the hits for the page for pageSize.
*/
private static SolrQuery getQuery(List<String> vclassUris, String alpha, int numberRows){ private static SolrQuery getQuery(List<String> vclassUris, String alpha, int page, int pageSize){
String queryText = ""; String queryText = "";
List<String> queryTypes = new ArrayList<String>(); try {
try { queryText = makeMultiClassQuery(vclassUris);
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
}
if (queryTypes.size() > 0) {
queryText = StringUtils.join(queryTypes, " AND ");
}
// Add alpha filter if applicable // Add alpha filter if applicable
if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) { if ( alpha != null && !"".equals(alpha) && alpha.length() == 1) {
queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*"; queryText += VitroSearchTermNames.NAME_LOWERCASE + ":" + alpha.toLowerCase() + "*";
} }
SolrQuery query = new SolrQuery(queryText); SolrQuery query = new SolrQuery(queryText);
log.debug("Query text is " + queryText);
// Get all available results from index rather than just those for the current page.
// Otherwise, if there are a large number of non-existent individuals in the search
// index, the current page of results might not retrieve any existing individuals,
// and nothing gets returned.
query.setStart(0)
.setRows(numberRows)
// Need a single-valued field for sorting
.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
query.setStart( page > 0 ? pageSize * page + 1 : 0 )
.setRows( pageSize );
// Need a single-valued field for sorting
query.setSortField(VitroSearchTermNames.NAME_LOWERCASE_SINGLE_VALUED, SolrQuery.ORDER.asc);
log.debug("Query text is " + queryText);
return query; return query;
} catch (Exception ex){ } catch (Exception ex){
log.error(ex,ex); log.error("Could not make Solr query",ex);
return new SolrQuery(); return new SolrQuery();
} }
} }
private static String makeMultiClassQuery( List<String> vclassUris){
String queryText = "";
List<String> queryTypes = new ArrayList<String>();
try {
// query term for rdf:type - multiple types possible
for(String vclassUri: vclassUris) {
queryTypes.add(VitroSearchTermNames.RDFTYPE + ":\"" + vclassUri + "\" ");
}
if (queryTypes.size() > 0) {
queryText = StringUtils.join(queryTypes, " AND ");
}
return queryText;
} catch (Exception ex){
log.error("Could not make Solr query",ex);
return "";
}
}
public static List<PageRecord> makePagesList( long size, int pageSize, int selectedPage ) { public static List<PageRecord> makePagesList( long size, int pageSize, int selectedPage ) {
List<PageRecord> records = new ArrayList<PageRecord>( MAX_PAGES + 1 ); List<PageRecord> records = new ArrayList<PageRecord>( MAX_PAGES + 1 );

36
webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/FileBasedProhibitedFromSearch.java
View file

@ -8,6 +8,8 @@ import java.io.FileInputStream;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
import edu.cornell.mannlib.vitro.webapp.servlet.setup.JenaDataSourceSetupBase;
public class FileBasedProhibitedFromSearch extends ProhibitedFromSearch { public class FileBasedProhibitedFromSearch extends ProhibitedFromSearch {
/** /**
@ -17,42 +19,12 @@ public class FileBasedProhibitedFromSearch extends ProhibitedFromSearch {
* @param dir to find N3 files in. * @param dir to find N3 files in.
*/ */
public FileBasedProhibitedFromSearch(String uri, File dir){ public FileBasedProhibitedFromSearch(String uri, File dir){
super( uri, getModelFromDir(dir)); super( uri, JenaDataSourceSetupBase.getModelFromDir(dir));
} }
public FileBasedProhibitedFromSearch(String URI, OntModel model) { public FileBasedProhibitedFromSearch(String URI, OntModel model) {
super(URI, model); super(URI, model);
} }
protected static OntModel getModelFromDir( File dir){
if( dir == null )
throw new IllegalStateException("Must pass a File to FileBasedProhibitedFromSearch");
if( !dir.isDirectory() )
throw new IllegalStateException("Parameter dir to FileBasedProhibitedFromSearch " +
"must be a File object for a directory");
if( !dir.canRead() )
throw new IllegalStateException("Parameter dir to FileBasedProhibitedFromSearch must " +
"be a directory that is readable, check premissions on " + dir.getAbsolutePath());
OntModel modelOnlyForPFS = ModelFactory.createOntologyModel();
for( File file : dir.listFiles()){
if( file.isFile()
&& file.canRead()
&& file.getName() != null
&& file.getName().endsWith(".n3")){
try{
modelOnlyForPFS.read( new FileInputStream(file), null, "N3");
}catch( Throwable th){
log.warn("could not load file " +
file.getAbsolutePath() + file.separator + file.getName(), th);
}
}
}
if( modelOnlyForPFS.size() == 0 ){
log.warn("No class exclusion statements found.");
}
return modelOnlyForPFS;
}
} }

381
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchController.java
View file

@ -11,7 +11,6 @@ import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.Iterator; import java.util.Iterator;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@ -24,13 +23,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrDocumentList;
import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean; import edu.cornell.mannlib.vitro.webapp.beans.ApplicationBean;
import edu.cornell.mannlib.vitro.webapp.beans.Individual; import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.IndividualImpl;
import edu.cornell.mannlib.vitro.webapp.beans.VClass; import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup; import edu.cornell.mannlib.vitro.webapp.beans.VClassGroup;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest; import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
@ -51,7 +51,6 @@ import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQuery;
import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryFactory; import edu.cornell.mannlib.vitro.webapp.search.beans.VitroQueryFactory;
import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup; import edu.cornell.mannlib.vitro.webapp.search.solr.SolrSetup;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.LinkTemplateModel;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.individuallist.BaseListedIndividual;
import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult; import edu.cornell.mannlib.vitro.webapp.web.templatemodels.searchresult.IndividualSearchResult;
import freemarker.template.Configuration; import freemarker.template.Configuration;
@ -67,8 +66,8 @@ public class PagedSearchController extends FreemarkerHttpServlet {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Log log = LogFactory.getLog(PagedSearchController.class); private static final Log log = LogFactory.getLog(PagedSearchController.class);
private static final int DEFAULT_HITS_PER_PAGE = 25; protected static final int DEFAULT_HITS_PER_PAGE = 25;
private static final int DEFAULT_MAX_HIT_COUNT = 1000; protected static final int DEFAULT_MAX_HIT_COUNT = 1000;
private static final String PARAM_XML_REQUEST = "xml"; private static final String PARAM_XML_REQUEST = "xml";
private static final String PARAM_START_INDEX = "startIndex"; private static final String PARAM_START_INDEX = "startIndex";
@ -146,26 +145,8 @@ public class PagedSearchController extends FreemarkerHttpServlet {
log.debug("IndividualDao is " + iDao.toString() + " Public classes in the classgroup are " + grpDao.getPublicGroupsWithVClasses().toString()); log.debug("IndividualDao is " + iDao.toString() + " Public classes in the classgroup are " + grpDao.getPublicGroupsWithVClasses().toString());
log.debug("VClassDao is "+ vclassDao.toString() ); log.debug("VClassDao is "+ vclassDao.toString() );
int startIndex = 0; int startIndex = getStartIndex(vreq);
try{ int hitsPerPage = getHitsPerPage( vreq );
startIndex = Integer.parseInt(vreq.getParameter(PARAM_START_INDEX));
}catch (Throwable e) {
startIndex = 0;
}
log.debug("startIndex is " + startIndex);
int hitsPerPage = DEFAULT_HITS_PER_PAGE;
try{
hitsPerPage = Integer.parseInt(vreq.getParameter(PARAM_HITS_PER_PAGE));
} catch (Throwable e) {
hitsPerPage = DEFAULT_HITS_PER_PAGE;
}
log.debug("hitsPerPage is " + hitsPerPage);
int maxHitCount = DEFAULT_MAX_HIT_COUNT ;
if( startIndex >= DEFAULT_MAX_HIT_COUNT - hitsPerPage )
maxHitCount = startIndex + DEFAULT_MAX_HIT_COUNT ;
log.debug("maxHitCount is " + maxHitCount);
String qtxt = vreq.getParameter(VitroQuery.QUERY_PARAMETER_NAME); String qtxt = vreq.getParameter(VitroQuery.QUERY_PARAMETER_NAME);
log.debug("Query text is \""+ qtxt + "\""); log.debug("Query text is \""+ qtxt + "\"");
@ -175,42 +156,27 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return doFailedSearch(badQueryMsg, qtxt, format); return doFailedSearch(badQueryMsg, qtxt, format);
} }
SolrQuery query = getQuery(qtxt, maxHitCount, vreq); SolrQuery query = getQuery(qtxt, hitsPerPage, startIndex, vreq);
SolrServer solr = SolrSetup.getSolrServer(getServletContext()); SolrServer solr = SolrSetup.getSolrServer(getServletContext());
QueryResponse response = null; QueryResponse response = null;
try { try {
response = solr.query(query); response = solr.query(query);
} catch (Exception ex) {
} catch (Throwable t) { String msg = makeBadSearchMessage(qtxt, ex.getMessage());
log.error("in first pass at search: " + t); log.error("could not run Solr query",ex);
// this is a hack to deal with odd cases where search and index threads interact return doFailedSearch(msg, qtxt, format);
try{
wait(150);
response = solr.query(query);
} catch (Exception ex) {
log.error(ex);
String msg = makeBadSearchMessage(qtxt, ex.getMessage());
if (msg == null) {
msg = "The search request contained errors.";
}
return doFailedSearch(msg, qtxt, format);
}
} }
if (response == null) { if (response == null) {
log.error("Search response was null"); log.error("Search response was null");
String msg = "The search request contained errors."; return doFailedSearch("The search request contained errors.", qtxt, format);
return doFailedSearch(msg, qtxt, format);
} }
SolrDocumentList docs = response.getResults(); SolrDocumentList docs = response.getResults();
if (docs == null) { if (docs == null) {
log.error("Document list for a search was null"); log.error("Document list for a search was null");
String msg = "The search request contained errors."; return doFailedSearch("The search request contained errors.", qtxt,format);
return doFailedSearch(msg, qtxt,format);
} }
long hitCount = docs.getNumFound(); long hitCount = docs.getNumFound();
@ -218,30 +184,20 @@ public class PagedSearchController extends FreemarkerHttpServlet {
if ( hitCount < 1 ) { if ( hitCount < 1 ) {
return doNoHits(qtxt,format); return doNoHits(qtxt,format);
} }
long lastHitToShow = 0;
if ((startIndex + hitsPerPage) > hitCount ) {
lastHitToShow = hitCount;
} else {
lastHitToShow = startIndex + hitsPerPage;
}
List<Individual> individuals = new LinkedList<Individual>(); List<Individual> individuals = new ArrayList<Individual>(docs.size());
for(int i = startIndex; i < lastHitToShow; i++){ Iterator<SolrDocument> docIter = docs.iterator();
try { while( docIter.hasNext() ){
SolrDocument doc = docs.get(i); try {
String uri = doc.get(VitroSearchTermNames.URI).toString(); SolrDocument doc = docIter.next();
log.debug("Retrieving individual with uri "+ uri); String uri = doc.get(VitroSearchTermNames.URI).toString();
Individual ent = new IndividualImpl(); Individual ind = iDao.getIndividualByURI(uri);
ent.setURI(uri); if(ind != null) {
ent = iDao.getIndividualByURI(uri); ind.setSearchSnippet( getSnippet(doc, response) );
if(ent!=null) { individuals.add(ind);
ent.setSearchSnippet(getSnippet(doc, response));
individuals.add(ent);
} }
} catch(Exception e) { } catch(Exception e) {
log.error("Problem getting usable individuals from search hits. " + log.error("Problem getting usable individuals from search hits. ",e);
e.getMessage());
} }
} }
@ -275,31 +231,19 @@ public class PagedSearchController extends FreemarkerHttpServlet {
body.put("typeName", type.getName()); body.put("typeName", type.getName());
} }
/* Add classgroup and type refinement links to body */ /* Add ClassGroup and type refinement links to body */
if( wasHtmlRequested ){ if( wasHtmlRequested ){
// Search request includes no classgroup and no type, so add classgroup search refinement links. if ( !classGroupFilterRequested && !typeFilterRequested ) {
if ( !classGroupFilterRequested && !typeFilterRequested ) { // Search request includes no ClassGroup and no type, so add ClassGroup search refinement links.
List<VClassGroup> classgroups = getClassGroups(grpDao, docs, maxHitCount); body.put("classGroupLinks", getClassGroupsLinks(grpDao, docs, response, qtxt));
List<VClassGroupSearchLink> classGroupLinks = new ArrayList<VClassGroupSearchLink>(classgroups.size()); } else if ( classGroupFilterRequested && !typeFilterRequested ) {
for (VClassGroup vcg : classgroups) { // Search request is for a ClassGroup, so add rdf:type search refinement links
if (vcg.getPublicName() != null) { // but try to filter out classes that are subclasses
classGroupLinks.add(new VClassGroupSearchLink(qtxt, vcg)); body.put("classLinks", getVClassLinks(vclassDao, docs, response, qtxt));
}
}
body.put("classGroupLinks", classGroupLinks);
// Search request is for a classgroup, so add rdf:type search refinement links
// but try to filter out classes that are subclasses
} else if ( classGroupFilterRequested && !typeFilterRequested ) {
List<VClass> vClasses = getVClasses(vclassDao, docs);
List<VClassSearchLink> vClassLinks = new ArrayList<VClassSearchLink>(vClasses.size());
for (VClass vc : vClasses) {
vClassLinks.add(new VClassSearchLink(qtxt, vc));
}
body.put("classLinks", vClassLinks);
pagingLinkParams.put(PARAM_CLASSGROUP, classGroupParam); pagingLinkParams.put(PARAM_CLASSGROUP, classGroupParam);
} else { } else {
//search request is for a class so there are no more refinements
pagingLinkParams.put(PARAM_RDFTYPE, typeParam); pagingLinkParams.put(PARAM_RDFTYPE, typeParam);
} }
} }
@ -314,9 +258,10 @@ public class PagedSearchController extends FreemarkerHttpServlet {
body.put("hitCount", hitCount); body.put("hitCount", hitCount);
body.put("startIndex", startIndex); body.put("startIndex", startIndex);
body.put("pagingLinks", getPagingLinks(startIndex, hitsPerPage, body.put("pagingLinks",
hitCount, maxHitCount, vreq.getServletPath(), getPagingLinks(startIndex, hitsPerPage, hitCount,
pagingLinkParams)); vreq.getServletPath(),
pagingLinkParams));
if (startIndex != 0) { if (startIndex != 0) {
body.put("prevPage", getPreviousPageLink(startIndex, body.put("prevPage", getPreviousPageLink(startIndex,
@ -336,6 +281,28 @@ public class PagedSearchController extends FreemarkerHttpServlet {
} }
private int getHitsPerPage(VitroRequest vreq) {
int hitsPerPage = DEFAULT_HITS_PER_PAGE;
try{
hitsPerPage = Integer.parseInt(vreq.getParameter(PARAM_HITS_PER_PAGE));
} catch (Throwable e) {
hitsPerPage = DEFAULT_HITS_PER_PAGE;
}
log.debug("hitsPerPage is " + hitsPerPage);
return hitsPerPage;
}
private int getStartIndex(VitroRequest vreq) {
int startIndex = 0;
try{
startIndex = Integer.parseInt(vreq.getParameter(PARAM_START_INDEX));
}catch (Throwable e) {
startIndex = 0;
}
log.debug("startIndex is " + startIndex);
return startIndex;
}
private String badQueryText(String qtxt) { private String badQueryText(String qtxt) {
if( qtxt == null || "".equals( qtxt.trim() ) ) if( qtxt == null || "".equals( qtxt.trim() ) )
return "Please enter a search term."; return "Please enter a search term.";
@ -348,80 +315,103 @@ public class PagedSearchController extends FreemarkerHttpServlet {
/** /**
* Get the class groups represented for the individuals in the documents. * Get the class groups represented for the individuals in the documents.
* @param qtxt
*/ */
private List<VClassGroup> getClassGroups(VClassGroupDao grpDao, SolrDocumentList docs, int maxHitCount) { private List<VClassGroupSearchLink> getClassGroupsLinks(VClassGroupDao grpDao, SolrDocumentList docs, QueryResponse rsp, String qtxt) {
LinkedHashMap<String,VClassGroup> grpMap = grpDao.getClassGroupMap(); Map<String,Long> cgURItoCount = new HashMap<String,Long>();
int n = grpMap.size();
HashSet<String> classGroupsInHits = new HashSet<String>(n); List<VClassGroup> classgroups = new ArrayList<VClassGroup>( );
int grpsFound = 0; List<FacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){
if(VitroSearchTermNames.CLASSGROUP_URI.equals(ff.getName())){
List<Count> counts = ff.getValues();
for( Count ct: counts){
VClassGroup vcg = grpDao.getGroupByURI( ct.getName() );
if( vcg == null ){
log.debug("could not get classgroup for URI " + ct.getName());
}else{
classgroups.add(vcg);
cgURItoCount.put(vcg.getURI(), ct.getCount());
}
}
}
}
long maxHits = Math.min(docs.getNumFound(), maxHitCount);
for(int i = 0; i < maxHits && n > grpsFound ;i++){
try{
SolrDocument doc = docs.get(i);
Collection<Object> grps = doc.getFieldValues(VitroSearchTermNames.CLASSGROUP_URI);
if (grps != null) {
for (Object o : grps) {
String groupUri = o.toString();
if( groupUri != null && !classGroupsInHits.contains(groupUri)){
classGroupsInHits.add(groupUri);
grpsFound++;
if( grpsFound >= n )
break;
}
}
}
} catch(Exception e) {
log.error("problem getting VClassGroups from search hits "
+ e.getMessage() );
e.printStackTrace();
}
}
List<String> classgroupURIs= Collections.list(Collections.enumeration(classGroupsInHits));
List<VClassGroup> classgroups = new ArrayList<VClassGroup>( classgroupURIs.size() );
for(String cgUri: classgroupURIs){
if( cgUri != null && ! "".equals(cgUri) ){
VClassGroup vcg = grpDao.getGroupByURI( cgUri );
if( vcg == null ){
log.debug("could not get classgroup for URI " + cgUri);
}else{
classgroups.add(vcg);
}
}
}
grpDao.sortGroupList(classgroups); grpDao.sortGroupList(classgroups);
return classgroups; List<VClassGroupSearchLink> classGroupLinks = new ArrayList<VClassGroupSearchLink>(classgroups.size());
for (VClassGroup vcg : classgroups) {
long count = cgURItoCount.get( vcg.getURI() );
if (vcg.getPublicName() != null && count > 0 ) {
classGroupLinks.add(new VClassGroupSearchLink(qtxt, vcg, count));
}
}
return classGroupLinks;
} }
private List<VClass> getVClasses(VClassDao vclassDao, SolrDocumentList docs){ private List<VClassSearchLink> getVClassLinks(VClassDao vclassDao, SolrDocumentList docs, QueryResponse rsp, String qtxt){
HashSet<String> typesInHits = getVClassUrisForHits(docs); HashSet<String> typesInHits = getVClassUrisForHits(docs);
List<VClass> classes = new ArrayList<VClass>(typesInHits.size()); List<VClass> classes = new ArrayList<VClass>(typesInHits.size());
Map<String,Long> typeURItoCount = new HashMap<String,Long>();
Iterator<String> it = typesInHits.iterator(); // Iterator<String> it = typesInHits.iterator();
while(it.hasNext()){ // while(it.hasNext()){
String typeUri = it.next(); // String typeUri = it.next();
try{ // try{
if( VitroVocabulary.OWL_THING.equals(typeUri)) // if( VitroVocabulary.OWL_THING.equals(typeUri))
continue; // continue;
VClass type = vclassDao.getVClassByURI(typeUri); // VClass type = vclassDao.getVClassByURI(typeUri);
if( type != null && // if( type != null &&
! type.isAnonymous() && // ! type.isAnonymous() &&
type.getName() != null && !"".equals(type.getName()) && // type.getName() != null && !"".equals(type.getName()) &&
type.getGroupURI() != null ) //don't display classes that aren't in classgroups // type.getGroupURI() != null ) //don't display classes that aren't in classgroups
classes.add(type); // classes.add(type);
}catch(Exception ex){ // }catch(Exception ex){
if( log.isDebugEnabled() ) // if( log.isDebugEnabled() )
log.debug("could not add type " + typeUri, ex); // log.debug("could not add type " + typeUri, ex);
} // }
// }
List<FacetField> ffs = rsp.getFacetFields();
for(FacetField ff : ffs){
if(VitroSearchTermNames.RDFTYPE.equals(ff.getName())){
List<Count> counts = ff.getValues();
for( Count ct: counts){
String typeUri = ct.getName();
long count = ct.getCount();
try{
if( VitroVocabulary.OWL_THING.equals(typeUri) ||
count == 0 )
continue;
VClass type = vclassDao.getVClassByURI(typeUri);
if( type != null &&
! type.isAnonymous() &&
type.getName() != null && !"".equals(type.getName()) &&
type.getGroupURI() != null ){ //don't display classes that aren't in classgroups
typeURItoCount.put(typeUri,count);
classes.add(type);
}
}catch(Exception ex){
if( log.isDebugEnabled() )
log.debug("could not add type " + typeUri, ex);
}
}
}
} }
Collections.sort(classes, new Comparator<VClass>(){ Collections.sort(classes, new Comparator<VClass>(){
public int compare(VClass o1, VClass o2) { public int compare(VClass o1, VClass o2) {
return o1.compareTo(o2); return o1.compareTo(o2);
}}); }});
return classes;
List<VClassSearchLink> vClassLinks = new ArrayList<VClassSearchLink>(classes.size());
for (VClass vc : classes) {
long count = typeURItoCount.get(vc.getURI());
vClassLinks.add(new VClassSearchLink(qtxt, vc, count ));
}
return vClassLinks;
} }
private HashSet<String> getVClassUrisForHits(SolrDocumentList docs){ private HashSet<String> getVClassUrisForHits(SolrDocumentList docs){
@ -455,51 +445,68 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return text.toString(); return text.toString();
} }
private SolrQuery getQuery(String queryText, int maxHitCount, VitroRequest vreq) { private SolrQuery getQuery(String queryText, int hitsPerPage, int startIndex, VitroRequest vreq) {
SolrQuery query = new SolrQuery(queryText); SolrQuery query = new SolrQuery(queryText);
// Solr requires these values, but we don't want them to be the real values for this page query.setStart( startIndex )
// of results, else the refinement links won't work correctly: each page of results needs to .setRows(hitsPerPage);
// show refinement links generated for all results, not just for the results on the current page.
query.setStart(0)
.setRows(maxHitCount);
// Classgroup filtering // ClassGroup filtering param
String classgroupParam = (String) vreq.getParameter(PARAM_CLASSGROUP); String classgroupParam = (String) vreq.getParameter(PARAM_CLASSGROUP);
if ( ! StringUtils.isBlank(classgroupParam) ) {
// rdf:type filtering param
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE);
if ( ! StringUtils.isBlank(classgroupParam) ) {
// ClassGroup filtering
log.debug("Firing classgroup query "); log.debug("Firing classgroup query ");
log.debug("request.getParameter(classgroup) is "+ classgroupParam); log.debug("request.getParameter(classgroup) is "+ classgroupParam);
query.addFilterQuery(VitroSearchTermNames.CLASSGROUP_URI + ":\"" + classgroupParam + "\""); query.addFilterQuery(VitroSearchTermNames.CLASSGROUP_URI + ":\"" + classgroupParam + "\"");
}
//with ClassGroup filtering we want type facets
// rdf:type filtering query.add("facet","true");
String typeParam = (String) vreq.getParameter(PARAM_RDFTYPE); query.add("facet.limit","-1");
if ( ! StringUtils.isBlank(typeParam) ) { query.add("facet.field",VitroSearchTermNames.RDFTYPE);
}else if ( ! StringUtils.isBlank(typeParam) ) {
// rdf:type filtering
log.debug("Firing type query "); log.debug("Firing type query ");
log.debug("request.getParameter(type) is "+ typeParam); log.debug("request.getParameter(type) is "+ typeParam);
query.addFilterQuery(VitroSearchTermNames.RDFTYPE + ":\"" + typeParam + "\""); query.addFilterQuery(VitroSearchTermNames.RDFTYPE + ":\"" + typeParam + "\"");
}
//with type filtering we don't have facets.
//query.setQuery(queryText); }else{
//When no filtering is set, we want ClassGroup facets
query.add("facet","true");
query.add("facet.limit","-1");
query.add("facet.field",VitroSearchTermNames.CLASSGROUP_URI);
}
log.debug("Query = " + query.toString()); log.debug("Query = " + query.toString());
return query; return query;
} }
private class VClassGroupSearchLink extends LinkTemplateModel { protected class VClassGroupSearchLink extends LinkTemplateModel {
long count = 0;
VClassGroupSearchLink(String querytext, VClassGroup classgroup) { VClassGroupSearchLink(String querytext, VClassGroup classgroup, long count) {
super(classgroup.getPublicName(), "/search", PARAM_QUERY_TEXT, querytext, PARAM_CLASSGROUP, classgroup.getURI()); super(classgroup.getPublicName(), "/search", PARAM_QUERY_TEXT, querytext, PARAM_CLASSGROUP, classgroup.getURI());
this.count = count;
} }
}
private class VClassSearchLink extends LinkTemplateModel {
VClassSearchLink(String querytext, VClass type) { public String getCount() { return Long.toString(count); }
super(type.getName(), "/search", PARAM_QUERY_TEXT, querytext, PARAM_RDFTYPE, type.getURI());
}
} }
private List<PagingLink> getPagingLinks(int startIndex, int hitsPerPage, long hitCount, int maxHitCount, String baseUrl, ParamMap params) { protected class VClassSearchLink extends LinkTemplateModel {
long count = 0;
VClassSearchLink(String querytext, VClass type, long count) {
super(type.getName(), "/search", PARAM_QUERY_TEXT, querytext, PARAM_RDFTYPE, type.getURI());
this.count = count;
}
public String getCount() { return Long.toString(count); }
}
protected static List<PagingLink> getPagingLinks(int startIndex, int hitsPerPage, long hitCount, String baseUrl, ParamMap params) {
List<PagingLink> pagingLinks = new ArrayList<PagingLink>(); List<PagingLink> pagingLinks = new ArrayList<PagingLink>();
@ -508,17 +515,23 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return pagingLinks; return pagingLinks;
} }
int maxHitCount = DEFAULT_MAX_HIT_COUNT ;
if( startIndex >= DEFAULT_MAX_HIT_COUNT - hitsPerPage )
maxHitCount = startIndex + DEFAULT_MAX_HIT_COUNT ;
for (int i = 0; i < hitCount; i += hitsPerPage) { for (int i = 0; i < hitCount; i += hitsPerPage) {
params.put(PARAM_START_INDEX, String.valueOf(i)); params.put(PARAM_START_INDEX, String.valueOf(i));
if ( i < maxHitCount - hitsPerPage) { if ( i < maxHitCount - hitsPerPage) {
int pageNumber = i/hitsPerPage + 1; int pageNumber = i/hitsPerPage + 1;
if (i >= startIndex && i < (startIndex + hitsPerPage)) { boolean iIsCurrentPage = (i >= startIndex && i < (startIndex + hitsPerPage));
if ( iIsCurrentPage ) {
pagingLinks.add(new PagingLink(pageNumber)); pagingLinks.add(new PagingLink(pageNumber));
} else { } else {
pagingLinks.add(new PagingLink(pageNumber, baseUrl, params)); pagingLinks.add(new PagingLink(pageNumber, baseUrl, params));
} }
} else { } else {
pagingLinks.add(new PagingLink("more...", baseUrl, params)); pagingLinks.add(new PagingLink("more...", baseUrl, params));
break;
} }
} }
@ -535,7 +548,7 @@ public class PagedSearchController extends FreemarkerHttpServlet {
return UrlBuilder.getUrl(baseUrl, params); return UrlBuilder.getUrl(baseUrl, params);
} }
private class PagingLink extends LinkTemplateModel { protected static class PagingLink extends LinkTemplateModel {
PagingLink(int pageNumber, String baseUrl, ParamMap params) { PagingLink(int pageNumber, String baseUrl, ParamMap params) {
super(String.valueOf(pageNumber), baseUrl, params); super(String.valueOf(pageNumber), baseUrl, params);

4
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -195,8 +195,8 @@ public class IndexBuilder extends Thread {
if(! stopRequested && log != null )//may be null on shutdown if(! stopRequested && log != null )//may be null on shutdown
log.info("Stopping IndexBuilder thread"); log.info("Stopping IndexBuilder thread");
} }
public static void checkIndexOnRootLogin(HttpServletRequest req){ public static void checkIndexOnRootLogin(HttpServletRequest req){
HttpSession session = req.getSession(); HttpSession session = req.getSession();
ServletContext context = session.getServletContext(); ServletContext context = session.getServletContext();

41
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -2,6 +2,7 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup; package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.FileInputStream;
import java.io.InputStream; import java.io.InputStream;
import java.io.File; import java.io.File;
import java.sql.SQLException; import java.sql.SQLException;
@ -15,6 +16,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.graph.Graph; import com.hp.hpl.jena.graph.Graph;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.OntModelSpec; import com.hp.hpl.jena.ontology.OntModelSpec;
import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -57,6 +59,12 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
//these files are loaded everytime the system starts up //these files are loaded everytime the system starts up
public static String APPPATH_LOAD = APPPATH + "menuload/"; public static String APPPATH_LOAD = APPPATH + "menuload/";
protected static String SUBMODELS = "/WEB-INF/submodels/"; protected static String SUBMODELS = "/WEB-INF/submodels/";
//All files in this directory will be reloaded every startup
//and attached as sub-models to the displayOntModel.
static final String DISPLAY_MODEL_LOAD_AT_STARTUP_DIR =
APPPATH + "loadedAtStartup";
protected static boolean firstStartup = false; protected static boolean firstStartup = false;
String DB_USER = "jenatest"; // database user id String DB_USER = "jenatest"; // database user id
@ -406,6 +414,39 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
} }
/**
* Read all the files in the directory as RDF files
* and return a model build from all RDF data found in those files.
* This will attempt to load formats supported by getRdfFormat().
*/
public static OntModel getModelFromDir( File dir){
if( dir == null )
throw new IllegalStateException("Must pass a File to getModelFromDir()");
if( !dir.isDirectory() )
throw new IllegalStateException("Directory must be a File object for a directory");
if( !dir.canRead() )
throw new IllegalStateException("getModelFromDir(): Directory " +
" must be readable, check premissions on " + dir.getAbsolutePath());
OntModel model = ModelFactory.createOntologyModel();
for( File file : dir.listFiles()){
if( file.isFile()
&& file.canRead()
&& file.getName() != null ){
String format = getRdfFormat( file.getName() );
try{
model.read( new FileInputStream(file), null, format);
}catch( Throwable th){
log.warn("Could not load file " +
file.getAbsolutePath() + file.separator + file.getName() +
" check that it contains valid " + format + " data.",
th);
}
}
}
return model;
}
public static void setVitroJenaModelMaker(VitroJenaModelMaker vjmm, public static void setVitroJenaModelMaker(VitroJenaModelMaker vjmm,
ServletContext ctx){ ServletContext ctx){
ctx.setAttribute(rdbModelMaker, vjmm); ctx.setAttribute(rdbModelMaker, vjmm);

169
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaPersistentDataSourceSetup.java
View file

@ -2,14 +2,23 @@
package edu.cornell.mannlib.vitro.webapp.servlet.setup; package edu.cornell.mannlib.vitro.webapp.servlet.setup;
import java.io.File;
import java.io.FileOutputStream;
import javax.servlet.ServletContext; import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener; import javax.servlet.ServletContextListener;
import org.apache.commons.logging.Log; import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory; import org.apache.commons.logging.LogFactory;
import org.joda.time.DateTime;
import org.joda.time.format.ISODateTimeFormat;
import com.hp.hpl.jena.ontology.OntModel; import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.rdf.model.Model; import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory; import com.hp.hpl.jena.rdf.model.ModelFactory;
@ -21,8 +30,8 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
private static final Log log = LogFactory.getLog( private static final Log log = LogFactory.getLog(
JenaPersistentDataSourceSetup.class.getName()); JenaPersistentDataSourceSetup.class.getName());
@Override @Override
public void contextInitialized(ServletContextEvent sce) { public void contextInitialized(ServletContextEvent sce) {
ServletContext ctx = sce.getServletContext(); ServletContext ctx = sce.getServletContext();
if (AbortStartup.isStartupAborted(ctx)) { if (AbortStartup.isStartupAborted(ctx)) {
@ -35,7 +44,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_USER_ACCOUNTS_MODEL, DB_ONT_MODEL_SPEC, ctx);
if (userAccountsDbModel.size() == 0) { if (userAccountsDbModel.size() == 0) {
firstStartup = true; firstStartup = true;
readOntologyFilesInPathSet(AUTHPATH, sce.getServletContext(), readOntologyFilesInPathSet(AUTHPATH, ctx,
userAccountsDbModel); userAccountsDbModel);
} }
OntModel userAccountsModel = ModelFactory.createOntologyModel( OntModel userAccountsModel = ModelFactory.createOntologyModel(
@ -43,8 +52,8 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
userAccountsModel.add(userAccountsDbModel); userAccountsModel.add(userAccountsDbModel);
userAccountsModel.getBaseModel().register( userAccountsModel.getBaseModel().register(
new ModelSynchronizer(userAccountsDbModel)); new ModelSynchronizer(userAccountsDbModel));
sce.getServletContext().setAttribute( ctx.setAttribute("userAccountsOntModel", userAccountsModel);
"userAccountsOntModel", userAccountsModel);
if (userAccountsModel.isEmpty()) { if (userAccountsModel.isEmpty()) {
initializeUserAccounts(ctx, userAccountsModel); initializeUserAccounts(ctx, userAccountsModel);
} }
@ -54,22 +63,20 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
// display, editing and navigation Model // display, editing and navigation Model
try { try {
Model appDbModel = makeDBModelFromConfigurationProperties( Model displayDbModel = makeDBModelFromConfigurationProperties(
JENA_DISPLAY_METADATA_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_METADATA_MODEL, DB_ONT_MODEL_SPEC, ctx);
log.debug("Display model size is " + appDbModel.size()); if (displayDbModel.size() == 0) {
if (appDbModel.size() == 0) { readOntologyFilesInPathSet(APPPATH, ctx,displayDbModel);
readOntologyFilesInPathSet(
APPPATH, sce.getServletContext(),appDbModel);
log.debug("Loaded ontology files from " + APPPATH + " into display model");
} }
OntModel appModel = ModelFactory.createOntologyModel( OntModel displayModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
MEM_ONT_MODEL_SPEC); displayModel.add(displayDbModel);
appModel.add(appDbModel); displayModel.getBaseModel().register(new ModelSynchronizer(displayDbModel));
appModel.getBaseModel().register(new ModelSynchronizer(appDbModel)); ctx.setAttribute("displayOntModel", displayModel);
ctx.setAttribute("displayOntModel", appModel);
//at each startup load all RDF files from directory to sub-models of display model
initializeDisplayLoadedAtStartup(ctx, displayModel);
} catch (Throwable t) { } catch (Throwable t) {
log.error("Unable to load user application configuration model from DB", t); log.error("Unable to load user application configuration model", t);
} }
//display tbox - currently reading in every time //display tbox - currently reading in every time
@ -78,7 +85,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
JENA_DISPLAY_TBOX_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_TBOX_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time //Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time? //TODO: Check if original needs to be cleared/removed every time?
readOntologyFileFromPath(APPPATH_LOAD + "displayTBOX.n3", displayTboxModel, sce.getServletContext()); readOntologyFileFromPath(APPPATH_LOAD + "displayTBOX.n3", displayTboxModel, ctx);
OntModel appTBOXModel = ModelFactory.createOntologyModel( OntModel appTBOXModel = ModelFactory.createOntologyModel(
MEM_ONT_MODEL_SPEC); MEM_ONT_MODEL_SPEC);
appTBOXModel.add(displayTboxModel); appTBOXModel.add(displayTboxModel);
@ -86,15 +93,16 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
ctx.setAttribute("displayOntModelTBOX", appTBOXModel); ctx.setAttribute("displayOntModelTBOX", appTBOXModel);
log.debug("Loaded file " + APPPATH_LOAD + "displayTBOX.n3 into display tbox model"); log.debug("Loaded file " + APPPATH_LOAD + "displayTBOX.n3 into display tbox model");
} catch (Throwable t) { } catch (Throwable t) {
log.error("Unable to load user application configuration model TBOX from DB", t); log.error("Unable to load user application configuration model TBOX", t);
} }
//Display Display model, currently empty, create if doesn't exist but no files to load //Display Display model, currently empty, create if doesn't exist but no files to load
try { try {
Model displayDisplayModel = makeDBModelFromConfigurationProperties( Model displayDisplayModel = makeDBModelFromConfigurationProperties(
JENA_DISPLAY_DISPLAY_MODEL, DB_ONT_MODEL_SPEC, ctx); JENA_DISPLAY_DISPLAY_MODEL, DB_ONT_MODEL_SPEC, ctx);
//Reading in single file every time //Reading in single file every time
//TODO: Check if original needs to be cleared/removed every time? //TODO: Check if original needs to be cleared/removed every time?
readOntologyFileFromPath(APPPATH_LOAD + "displayDisplay.n3", displayDisplayModel, sce.getServletContext()); readOntologyFileFromPath(APPPATH_LOAD + "displayDisplay.n3", displayDisplayModel, ctx);
OntModel appDisplayDisplayModel = ModelFactory.createOntologyModel( OntModel appDisplayDisplayModel = ModelFactory.createOntologyModel(
MEM_ONT_MODEL_SPEC); MEM_ONT_MODEL_SPEC);
@ -103,7 +111,7 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
ctx.setAttribute("displayOntModelDisplayModel", appDisplayDisplayModel); ctx.setAttribute("displayOntModelDisplayModel", appDisplayDisplayModel);
log.debug("Loaded file " + APPPATH_LOAD + "displayDisplay.n3 into display display model"); log.debug("Loaded file " + APPPATH_LOAD + "displayDisplay.n3 into display display model");
} catch (Throwable t) { } catch (Throwable t) {
log.error("Unable to load user application configuration model Display Model from DB", t); log.error("Unable to load user application configuration model Display Model", t);
} }
} }
@ -117,4 +125,123 @@ public class JenaPersistentDataSourceSetup extends JenaDataSourceSetupBase
readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsModel); readOntologyFilesInPathSet(AUTHPATH, ctx, userAccountsModel);
} }
/**
* Load the RDF found in the directory DISPLAY_MODEL_LOAD_AT_STARTUP_DIR
* a sub-models of displayModel. The RDF from thes files will not be saved
* in the database and it will be reloaded each time the system starts up.
*/
private void initializeDisplayLoadedAtStartup(ServletContext ctx, OntModel displayModel){
Model displayLoadAtStartup = readInDisplayModelLoadAtStartup(ctx);
if( log.isDebugEnabled() ){
log.debug("loaded display model from files in " + ctx.getRealPath(DISPLAY_MODEL_LOAD_AT_STARTUP_DIR) );
displayLoadAtStartup.write(System.out, "N3-PP");
}
checkForOldListViews(ctx,displayModel,displayLoadAtStartup);
displayModel.addSubModel( displayLoadAtStartup );
}
/**
* All of the list views should now reside in files in DISPLAY_MODEL_LOAD_AT_STARTUP_DIR.
* This will check for custom list view annotation statements in the displayModel, check
* if they exist in the files in DISPLAY_MODEL_LOAD_AT_STARTUP_DIR, and write any that don't
* exist there to a file in DISPLAY_MODEL_LOAD_AT_STARTUP_DIR. After that the statements
* will be removed from the displayDBModel.
*
* returns true if there were old list view statements in the DB, returns false
* if there were none. displayLoadAlways should be reloaded from the file system
* if this returns true as this method may have changed the files.
*
* displayLoadAtStartup and displayModel may be modified.
*/
private void checkForOldListViews( ServletContext ctx, OntModel displayModel, Model displayLoadAtStartup){
// run construct for old custom list view statements from displayModel
Model oldListViewModel = getOldListViewStatements( displayModel );
if( log.isDebugEnabled() ){
log.debug("Printing the old list view statements from the display model to System.out.");
oldListViewModel.write(System.out,"N3-PP");
}
// find statements in old stmts that are not in loadedAtStartup and
// save them in a new file in DISPLAY_MODEL_LOAD_AT_STARTUP_DIR
// so that in the future they will be in loadedAtStartup
Model stmtsInOldAndFiles = displayLoadAtStartup.intersection( displayModel );
Model unhandledOldListViewStmts = oldListViewModel.difference( stmtsInOldAndFiles );
boolean saved = false;
boolean neededSave = false;
if( unhandledOldListViewStmts != null && !unhandledOldListViewStmts.isEmpty() ){
log.debug("need to deal with old list view statements from the display model");
neededSave = true;
try{
//create a file for the old statements in the loadAtStartup directory
String newFileName = ctx.getRealPath(
DISPLAY_MODEL_LOAD_AT_STARTUP_DIR + File.separator
+ new DateTime().toString(ISODateTimeFormat.basicDateTime()) + ".n3" );
File file = new File( newFileName );
file.createNewFile();
log.info("Relocating " + unhandledOldListViewStmts.size() + " custom list view statements from DB and saving to "
+ file.getAbsolutePath()+ File.separator + file.getName()
+ ". These will be loaded from this file when the system starts up.");
FileOutputStream fileOut = new FileOutputStream(file);
unhandledOldListViewStmts.write(fileOut, "N3-PP");
fileOut.close();
saved = true;
}catch(Throwable th){
log.warn("Could not save old list view statements. Leaving them in the DB",th);
}
//need to reload displayLoadAlways because DISPLAY_MODEL_LOAD_AT_STARTUP_DIR may have changed
displayLoadAtStartup.removeAll().add(readInDisplayModelLoadAtStartup(ctx));
}
if( oldListViewModel != null && ! oldListViewModel.isEmpty() ){
//At this point, there are old list view statements in the DB but they
//should are all redundant with ones in DISPLAY_MODEL_LOAD_AT_STARTUP_DIR
if( (neededSave && saved) || (!neededSave) ){
//if there was nothing to save, just remove the old stuff
//if there was stuff to save, only remove if it was saved.
log.debug("removing old statements from displayModel");
displayModel.remove(oldListViewModel);
}
}
}
private Model getOldListViewStatements(OntModel displayModel) {
//run a construct on displayModel to get all list view statements
Query query = QueryFactory.create ( listViewQuery );
QueryExecution qexec = QueryExecutionFactory.create(query, displayModel) ;
Model oldModel = null;
try {
oldModel = qexec.execConstruct();
} catch( Throwable th ){
log.error("could not check for old custom list views, query exception",th);
}finally {
qexec.close() ;
}
if( oldModel != null)
return oldModel;
else
return ModelFactory.createDefaultModel();
}
private static final String listViewQuery = "" +
"PREFIX d: <http://vitro.mannlib.cornell.edu/ontologies/display/1.1#>\n" +
"CONSTRUCT { \n" +
" ?a d:listViewConfigFile ?b . \n" +
"} WHERE {\n" +
" ?a d:listViewConfigFile ?b . \n" +
"} ";
protected Model readInDisplayModelLoadAtStartup( ServletContext ctx ){
return getModelFromDir( new File( ctx.getRealPath( DISPLAY_MODEL_LOAD_AT_STARTUP_DIR )));
}
} }

7
webapp/src/edu/cornell/mannlib/vitro/webapp/utils/pageDataGetter/DataGetterUtils.java
View file

@ -246,11 +246,8 @@ public class DataGetterUtils {
jo.put("imageUrl", ind.getImageUrl()); jo.put("imageUrl", ind.getImageUrl());
jo.put("profileUrl", UrlBuilder.getIndividualProfileUrl(ind, vreq)); jo.put("profileUrl", UrlBuilder.getIndividualProfileUrl(ind, vreq));
jo.put("mostSpecificTypes", JsonServlet.getMostSpecificTypes(ind,fullWdf)); jo.put("mostSpecificTypes", JsonServlet.getMostSpecificTypes(ind,fullWdf));
jo.put("preferredTitle", JsonServlet.getDataPropertyValue(ind, preferredTitleDp, fullWdf));
jo.put("preferredTitle", JsonServlet.getDataPropertyValue(ind, preferredTitleDp, fullWdf));
jo.put("firstName", JsonServlet.getDataPropertyValue(ind, fNameDp, fullWdf));
jo.put("lastName", JsonServlet.getDataPropertyValue(ind, lNameDp, fullWdf));
jInds.put(jo); jInds.put(jo);
} }

61
webapp/test/edu/cornell/mannlib/vitro/webapp/search/controller/PagedSearchControllerTest.java Normal file
View file

@ -0,0 +1,61 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
/**
*
*/
package edu.cornell.mannlib.vitro.webapp.search.controller;
import java.util.List;
import junit.framework.Assert;
import org.junit.Before;
import org.junit.Test;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.UrlBuilder.ParamMap;
import edu.cornell.mannlib.vitro.webapp.search.controller.PagedSearchController.PagingLink;
public class PagedSearchControllerTest {
@Test
public void testGetPagingLinks() {
ParamMap pm = new ParamMap();
int hitsPerPage = 25;
int totalHits = 500;
int currentStartIndex = 0;
List<PagingLink> pageLinks = PagedSearchController.getPagingLinks(currentStartIndex, hitsPerPage, totalHits, "baseURL", pm);
Assert.assertNotNull(pageLinks);
Assert.assertEquals(500 / 25, pageLinks.size());
//test for no page links on a very short result
hitsPerPage = 25;
totalHits = 10;
currentStartIndex = 0;
pageLinks = PagedSearchController.getPagingLinks(currentStartIndex, hitsPerPage, totalHits, "baseURL", pm);
Assert.assertNotNull(pageLinks);
Assert.assertEquals(0, pageLinks.size());
}
@Test
public void testGetPagingLinksForLargeResults() {
ParamMap pm = new ParamMap();
int hitsPerPage = 25;
int totalHits = 349909;
int currentStartIndex = 0;
List<PagingLink> pageLinks = PagedSearchController.getPagingLinks(currentStartIndex, hitsPerPage, totalHits, "baseURL", pm);
Assert.assertNotNull(pageLinks);
Assert.assertEquals( PagedSearchController.DEFAULT_MAX_HIT_COUNT / hitsPerPage, pageLinks.size());
//test for large sets of results with high start index
hitsPerPage = 25;
totalHits = PagedSearchController.DEFAULT_MAX_HIT_COUNT + 20329;
currentStartIndex = PagedSearchController.DEFAULT_MAX_HIT_COUNT + 5432;
pageLinks = PagedSearchController.getPagingLinks(currentStartIndex, hitsPerPage, totalHits, "baseURL", pm);
Assert.assertNotNull(pageLinks);
Assert.assertEquals(
(currentStartIndex / hitsPerPage) + //all the pages that are before the current page
(PagedSearchController.DEFAULT_MAX_HIT_COUNT / hitsPerPage) + //some pages after the current apge
1, //for the more... page
pageLinks.size());
}
}