litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Removed unused method Individual.isMemberOfClassProhibitedFromSearch()

Browse source 
NIHVIVO-3696 Fixed linked data responses so correctly filtered by public display settings
NIHVIVO-3758 Change search name boost to be additive
Removed unused search field PROHIBITED_FROM_TEXT_RESULTS
NIHVIVO-3759 Refactored individual exclusion in search 
NIHVIVO-549 Allow for local product to add custom code for search
This commit is contained in:
briancaruso 2012-05-11 20:59:34 +00:00
parent a7fdf30a85
commit e3158dadaa
25 changed files with 593 additions and 323 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
webapp/src/edu/cornell/mannlib/vitro/webapp/beans/Individual.java
View file

@ -3,15 +3,12 @@
package edu.cornell.mannlib.vitro.webapp.beans;
import java.sql.Timestamp;
import java.util.Date;
import java.util.List;
import java.util.Map;
import org.json.JSONException;
import org.json.JSONObject;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
/**
* User: bdc34
* Date: Oct 18, 2007
@ -69,9 +66,7 @@ public interface Individual extends ResourceBean, Comparable<Individual> {
void setVClasses(List<VClass> vClassList, boolean direct);
/** Does the individual belong to this class? */
boolean isVClass(String uri);
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs);
boolean isVClass(String uri);
void setObjectPropertyStatements(List<ObjectPropertyStatement> list);
List<ObjectPropertyStatement> getObjectPropertyStatements();

5
webapp/src/edu/cornell/mannlib/vitro/webapp/beans/IndividualImpl.java
View file

@ -192,11 +192,6 @@ public class IndividualImpl extends BaseResourceBean implements Individual, Comp
}
return false;
}
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs) {
throw new UnsupportedOperationException(this.getClass().getName() +
".isMemberOfClassProhibitedFromSearch must be overriden by a subclass");
}
public List<VClass> getVClasses(boolean direct) {
if (direct) {

30
webapp/src/edu/cornell/mannlib/vitro/webapp/controller/individual/IndividualRdfAssembler.java
View file

@ -125,22 +125,20 @@ public class IndividualRdfAssembler {
}
newModel = getLabelAndTypes(entity, contextModel, newModel );
// get all the statements not covered by the object property / datatype property code above
// note implication that extendedLinkedData individuals will only be evaluated for the
// recognized object properties.
contextModel.enterCriticalSection(Lock.READ);
try {
StmtIterator iter = contextModel.listStatements(subj, (Property) null, (RDFNode) null);
while (iter.hasNext()) {
Statement stmt = iter.next();
if (!newModel.contains(stmt)) {
newModel.add(stmt);
}
}
} finally {
contextModel.leaveCriticalSection();
}
//bdc34: The following code adds all triples where entity is the Subject.
// contextModel.enterCriticalSection(Lock.READ);
// try {
// StmtIterator iter = contextModel.listStatements(subj, (Property) null, (RDFNode) null);
// while (iter.hasNext()) {
// Statement stmt = iter.next();
// if (!newModel.contains(stmt)) {
// newModel.add(stmt);
// }
// }
// } finally {
// contextModel.leaveCriticalSection();
// }
if (recurseDepth == 0 && includes != null && entity.isVClass(PERSON_CLASS_URI)) {

5
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/filtering/IndividualFiltering.java
View file

@ -368,11 +368,6 @@ public class IndividualFiltering implements Individual {
public boolean isVClass(String uri) {
return _innerIndividual.isVClass(uri);
}
@Override
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs) {
return _innerIndividual.isMemberOfClassProhibitedFromSearch(pfs);
}
@Override
public void setDataPropertyMap(Map<String, DataProperty> propertyMap) {

26
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/IndividualJena.java
View file

@ -28,7 +28,6 @@ import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.util.iterator.ClosableIterator;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.beans.DataProperty;
import edu.cornell.mannlib.vitro.webapp.beans.DataPropertyStatement;
@ -41,7 +40,6 @@ import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.filestorage.model.ImageInfo;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
public class IndividualJena extends IndividualImpl implements Individual {
@ -508,30 +506,6 @@ public class IndividualJena extends IndividualImpl implements Individual {
}
return false;
}
@Override
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs) {
ind.getModel().enterCriticalSection(Lock.READ);
try {
StmtIterator stmtIt = ind.listProperties(RDF.type);
try {
while(stmtIt.hasNext()) {
Statement stmt = stmtIt.nextStatement();
if (stmt.getObject().isURIResource()) {
String typeURI = ((Resource)stmt.getObject()).getURI();
if (pfs.isClassProhibitedFromSearch(typeURI)) {
return true;
}
}
}
} finally {
stmtIt.close();
}
return false;
} finally {
ind.getModel().leaveCriticalSection();
}
}
/**
* Overriding the base method so that we can do the sorting by arbitrary property here. An

17
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/IndividualSDB.java
View file

@ -53,7 +53,6 @@ import edu.cornell.mannlib.vitro.webapp.dao.VClassDao;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.WebappDaoFactorySDB.SDBDatasetMode;
import edu.cornell.mannlib.vitro.webapp.filestorage.model.ImageInfo;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
public class IndividualSDB extends IndividualImpl implements Individual {
@ -870,22 +869,6 @@ public class IndividualSDB extends IndividualImpl implements Individual {
return false;
}
@Override
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs) {
List<VClass> types = getVClasses(false);
Iterator<VClass> itr = types.iterator();
while(itr.hasNext()) {
String typeURI = itr.next().getURI();
if (pfs.isClassProhibitedFromSearch(typeURI)) {
return true;
}
}
return false;
}
/**
* Overriding the base method so that we can do the sorting by arbitrary property here. An
* IndividualSDB has a reference back to the model; everything else is just a dumb bean (for now).

8
webapp/src/edu/cornell/mannlib/vitro/webapp/dao/jena/ModelContext.java
View file

@ -7,6 +7,8 @@ import javax.servlet.ServletContext;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
public class ModelContext {
private static final String ONT_MODEL_SELECTOR = "ontModelSelector";
@ -106,4 +108,10 @@ public class ModelContext {
}
public static OntModel getDisplayModel(ServletContext ctx){
return(OntModel) ctx.getAttribute( DisplayVocabulary.DISPLAY_ONT_MODEL );
}
public static void setDisplayModel(OntModel ontModel, ServletContext ctx){
ctx.setAttribute(DisplayVocabulary.DISPLAY_ONT_MODEL,ontModel);
}
}

2
webapp/src/edu/cornell/mannlib/vitro/webapp/search/VitroSearchTermNames.java
View file

@ -25,8 +25,6 @@ public class VitroSearchTermNames {
public static String ALLTEXTUNSTEMMED = "ALLTEXTUNSTEMMED";
/** Does the individual have a thumbnail image? 1=yes 0=no */
public static final String THUMBNAIL = "THUMBNAIL";
/** Should individual be included in full text search results? 1=yes 0=no */
public static final String PROHIBITED_FROM_TEXT_RESULTS = "PROHIBITED_FROM_TEXT_RESULTS";
/** class names in human readable form of an individual*/
public static final String CLASSLOCALNAMELOWERCASE = "classLocalNameLowerCase";
/** class names in human readable form of an individual*/

8
webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/IndividualProhibitedFromSearch.java
View file

@ -1,8 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.beans;
public interface IndividualProhibitedFromSearch {
public boolean isIndividualProhibited(String uri);
}

84
webapp/src/edu/cornell/mannlib/vitro/webapp/search/beans/IndividualProhibitedFromSearchImpl.java
View file

@ -1,84 +0,0 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.beans;
import javax.servlet.ServletContext;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.shared.Lock;
import com.hp.hpl.jena.vocabulary.OWL;
import com.hp.hpl.jena.vocabulary.RDF;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
public class IndividualProhibitedFromSearchImpl implements IndividualProhibitedFromSearch {
protected OntModel fullModel;
protected static Log log = LogFactory.getLog(IndividualProhibitedFromSearchImpl.class);
public IndividualProhibitedFromSearchImpl( ServletContext context ){
this.fullModel = ModelContext.getUnionOntModelSelector(context).getFullModel();
}
public IndividualProhibitedFromSearchImpl( OntModel fullModel ){
this.fullModel = fullModel;
}
public boolean isIndividualProhibited(String uri){
if( uri == null || uri.isEmpty() )
return true;
boolean prohibited = false;
QueryExecution qexec = null;
try {
fullModel.getLock().enterCriticalSection(Lock.READ);
Query query = makeAskQueryForUri( uri );
qexec = QueryExecutionFactory.create( query, fullModel);
prohibited = qexec.execAsk();
} finally {
if( qexec != null ) qexec.close();
fullModel.getLock().leaveCriticalSection();
}
if( prohibited )
log.debug("prohibited " + uri);
return prohibited;
}
private Query makeAskQueryForUri( String uri ){
String queryString =
"PREFIX fn: <http://www.w3.org/2005/xpath-functions#> \n" +
"ASK { \n" +
" <"+uri+"> <" + RDF.type.getURI() + "> ?type . \n" +
" FILTER ( \n" +
" ( fn:starts-with( str(?type), \"" + VitroVocabulary.vitroURI + "\" ) \n" +
" && \n"+
" ! fn:starts-with( str(?type), \"" + VitroVocabulary.vitroURI + "Flag\" ) ) || \n" +
" fn:starts-with( str(?type), \"" + VitroVocabulary.PUBLIC + "\" ) || \n" +
" str(?type) = \"" + OWL.ObjectProperty.getURI() + "\" || \n" +
" str(?type) = \"" + OWL.DatatypeProperty.getURI() + "\" || \n" +
" str(?type) = \"" + OWL.AnnotationProperty.getURI() + "\" \n" +
" )\n" +
"}" ;
return QueryFactory.create( queryString );
}
}

34
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/ExcludeBasedOnNamespace.java Normal file
View file

@ -0,0 +1,34 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.Arrays;
import java.util.List;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
/**
* Skip individual if its URI is from any of the excludeNamepsaces
*
*/
public class ExcludeBasedOnNamespace implements SearchIndexExcluder {
List<String> excludeNamepsaces;
public ExcludeBasedOnNamespace(String ... excludeNamepsaces) {
super();
this.excludeNamepsaces = Arrays.asList(excludeNamepsaces);
}
@Override
public String checkForExclusion(Individual ind) {
for( String ns: excludeNamepsaces){
if( ns.equals( ind.getNamespace() ) ){
return "skipping because of namespace " ;
}
}
return null;
}
}

60
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/ExcludeBasedOnType.java Normal file
View file

@ -0,0 +1,60 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
/**
* Exclude individual from search index if
* it is a member of any of the the types.
* @author bdc34
*
*/
public class ExcludeBasedOnType implements SearchIndexExcluder {
List<String> typeURIs;
public ExcludeBasedOnType(String ... typeURIs) {
setExcludedTypes( typeURIs );
}
@Override
public String checkForExclusion(Individual ind) {
if( ind != null ) {
List<VClass> vclasses = ind.getVClasses();
if( vclasses != null && ! Collections.disjoint(vclasses, typeURIs) ){
return("skipping due to type.");
}
}
return null;
}
public void setExcludedTypes(String ... typeURIs){
setExcludedTypes(Arrays.asList(typeURIs));
}
public void setExcludedTypes(List<String> typeURIs){
synchronized(this){
this.typeURIs = new ArrayList<String>(typeURIs);
}
}
protected void addTypeToExclude(String typeURI){
if( typeURI != null && !typeURI.isEmpty()){
synchronized(this){
typeURIs.add(typeURI);
}
}
}
protected void removeTypeToExclude(String typeURI){
synchronized(this){
typeURIs.remove(typeURI);
}
}
}

58
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/ExcludeBasedOnTypeNamespace.java Normal file
View file

@ -0,0 +1,58 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
/**
* Exclude individuals based on the namespaces of their types.
*/
public class ExcludeBasedOnTypeNamespace implements SearchIndexExcluder {
final List<String> namespaces;
Pattern nsRegexPattern;
public ExcludeBasedOnTypeNamespace(String ... namespaces) {
super();
this.namespaces = Collections.unmodifiableList(Arrays.asList( namespaces ));
String nsOrPattern = "";
for( int i=0; i<namespaces.length; i++){
String ns = namespaces[i];
nsOrPattern = nsOrPattern + (i!=0?"|":"") + Pattern.quote(ns) + "[^/#]*$";
}
this.nsRegexPattern = Pattern.compile(nsOrPattern);
}
@Override
public String checkForExclusion(Individual ind) {
if( ind != null && ind.getVClasses() != null ){
for( VClass vclass : ind.getVClasses() ){
String excludeMsg = checkForSkip(ind, vclass);
if(excludeMsg != null)
return excludeMsg;
}
}
return null;
}
String checkForSkip(Individual individual, VClass vclass) {
if( vclass != null && vclass.getURI() != null ){
Matcher match=nsRegexPattern.matcher( vclass.getURI() );
if( match.matches() ){
return "Skipping because it is of a type in an excluded namespace.";
}
}
return null;
}
}

44
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/ExcludeNonFlagVitro.java Normal file
View file

@ -0,0 +1,44 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.List;
import org.apache.solr.common.SolrInputDocument;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
/**
* Exclude individuals with types from the Vitro namespace from the
* search index. (Other than old vitro Flag types).
*/
public class ExcludeNonFlagVitro implements SearchIndexExcluder {
@Override
public String checkForExclusion(Individual ind) {
if( ind != null && ind.getVClasses() != null ) {
String excludeMsg = skipIfVitro(ind, ind.getVClasses() );
if( excludeMsg != null)
return excludeMsg;
}
return null;
}
String skipIfVitro(Individual ind, List<VClass> vclasses) {
for( VClass type: vclasses ){
if( type != null && type.getURI() != null ){
String typeURI = type.getURI();
if(typeURI.startsWith( VitroVocabulary.vitroURI )
&& ! typeURI.startsWith(VitroVocabulary.vitroURI + "Flag") ){
return "Skipped " + ind.getURI()+" because in "
+ VitroVocabulary.vitroURI + " namespace";
}
}
}
return null;
}
}

167
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java
View file

@ -2,8 +2,6 @@
package edu.cornell.mannlib.vitro.webapp.search.solr;
import org.jsoup.Jsoup;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
@ -14,6 +12,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.joda.time.DateTime;
import org.jsoup.Jsoup;
import com.hp.hpl.jena.vocabulary.OWL;
@ -26,50 +25,35 @@ import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.search.IndexingException;
import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
import edu.cornell.mannlib.vitro.webapp.search.beans.ClassProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.beans.IndividualProhibitedFromSearch;
public class IndividualToSolrDocument {
public static final Log log = LogFactory.getLog(IndividualToSolrDocument.class.getName());
public static VitroSearchTermNames term = new VitroSearchTermNames();
protected static String entClassName = Individual.class.getName();
protected ClassProhibitedFromSearch classesProhibitedFromSearch;
protected IndividualProhibitedFromSearch individualProhibitedFromSearch;
public static VitroSearchTermNames term = new VitroSearchTermNames();
protected final String label = "http://www.w3.org/2000/01/rdf-schema#label";
protected List<DocumentModifier> documentModifiers = new ArrayList<DocumentModifier>();
public IndividualToSolrDocument(
ClassProhibitedFromSearch classesProhibitedFromSearch,
IndividualProhibitedFromSearch individualProhibitedFromSearch){
this( classesProhibitedFromSearch,
individualProhibitedFromSearch,
Collections.EMPTY_LIST);
}
public IndividualToSolrDocument(
ClassProhibitedFromSearch classesProhibitedFromSearch,
IndividualProhibitedFromSearch individualProhibitedFromSearch,
List<DocumentModifier> docModifiers){
this.classesProhibitedFromSearch = classesProhibitedFromSearch;
this.individualProhibitedFromSearch = individualProhibitedFromSearch;
this.documentModifiers = docModifiers;
}
protected List<SearchIndexExcluder> excludes;
public IndividualToSolrDocument(List<SearchIndexExcluder> excludes, List<DocumentModifier> docModifiers){
this.excludes = excludes;
this.documentModifiers = docModifiers;
}
@SuppressWarnings("static-access")
public SolrInputDocument translate(Individual ind) throws IndexingException{
try{
log.debug("translating " + ind.getURI());
checkForSkipBasedOnNS( ind );
String excludeMsg = checkExcludes( ind );
if( excludeMsg != DONT_EXCLUDE){
log.debug(excludeMsg);
return null;
}
SolrInputDocument doc = new SolrInputDocument();
//DocID
@ -77,44 +61,30 @@ public class IndividualToSolrDocument {
//vitro id
doc.addField(term.URI, ind.getURI());
//java class
doc.addField(term.JCLASS, entClassName);
//Individual Label
addLabel( ind, doc );
//add classes, classgroups get if prohibied becasue of its class
StringBuffer classPublicNames = new StringBuffer("");
boolean prohibited = addClasses(ind, doc, classPublicNames);
//filter out class groups, owl:ObjectProperties etc..
if(individualProhibitedFromSearch.isIndividualProhibited( ind.getURI() )){
return null;
}
addClasses(ind, doc, classPublicNames);
// collecting URIs and rdfs:labels of objects of statements
StringBuffer objectNames = new StringBuffer("");
StringBuffer addUri = new StringBuffer("");
addObjectPropertyText(ind, doc, objectNames, addUri);
//add if the individual has a thumbnail or not.
addThumbnailExistance(ind, doc);
addObjectPropertyText(ind, doc, objectNames, addUri);
//time of index in millis past epoc
doc.addField(term.INDEXEDTIME, new Long( (new DateTime()).getMillis() ) );
if(!prohibited){
addAllText( ind, doc, classPublicNames, objectNames );
addAllText( ind, doc, classPublicNames, objectNames );
runAdditionalDocModifers(ind,doc,addUri);
//boost for entity
if(documentModifiers == null || documentModifiers.isEmpty() &&
(ind.getSearchBoost() != null && ind.getSearchBoost() != 0)) {
doc.setDocumentBoost(ind.getSearchBoost());
}
}
//boost for entity
if(ind.getSearchBoost() != null && ind.getSearchBoost() != 0) {
doc.setDocumentBoost(ind.getSearchBoost());
}
runAdditionalDocModifers(ind,doc,addUri);
return doc;
}catch(SkipIndividualException ex){
@ -122,7 +92,7 @@ public class IndividualToSolrDocument {
log.debug(ex);
return null;
}catch(Throwable th){
//Odd exceptions from jena get thrown on shutdown
//Odd exceptions can get thrown on shutdown
if( log != null )
log.debug(th);
return null;
@ -130,7 +100,20 @@ public class IndividualToSolrDocument {
}
protected void runAdditionalDocModifers( Individual ind, SolrInputDocument doc, StringBuffer addUri )
protected String checkExcludes(Individual ind) {
for( SearchIndexExcluder excluder : excludes){
try{
String msg = excluder.checkForExclusion(ind);
if( msg != DONT_EXCLUDE)
return msg;
}catch (Exception e) {
return e.getMessage();
}
}
return DONT_EXCLUDE;
}
protected void runAdditionalDocModifers( Individual ind, SolrInputDocument doc, StringBuffer addUri )
throws SkipIndividualException{
//run the document modifiers
if( documentModifiers != null && !documentModifiers.isEmpty()){
@ -140,18 +123,6 @@ public class IndividualToSolrDocument {
}
}
protected void checkForSkipBasedOnNS(Individual ind) throws SkipIndividualException {
String id = ind.getURI();
if(id == null){
throw new SkipIndividualException("cannot add individuals without URIs to search index");
}else if( id.startsWith(VitroVocabulary.vitroURI) ||
id.startsWith(VitroVocabulary.VITRO_PUBLIC) ||
id.startsWith(VitroVocabulary.PSEUDO_BNODE_NS) ||
id.startsWith(OWL.NS)){
throw new SkipIndividualException("not indexing because of namespace:" + id);
}
}
protected void addAllText(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames, StringBuffer objectNames) {
String t=null;
//ALLTEXT, all of the 'full text'
@ -210,19 +181,7 @@ public class IndividualToSolrDocument {
// NAME_LOWERCASE, NAME_UNSTEMMED, NAME_STEMMED, NAME_PHONETIC, AC_NAME_UNTOKENIZED, AC_NAME_STEMMED
}
/**
* Adds if the individual has a thumbnail image or not.
*/
protected void addThumbnailExistance(Individual ind, SolrInputDocument doc) {
try{
if(ind.hasThumb())
doc.addField(term.THUMBNAIL, "1");
else
doc.addField(term.THUMBNAIL, "0");
}catch(Exception ex){
log.debug("could not index thumbnail: " + ex);
}
}
/**
* Get the rdfs:labes for objects of statements and put in objectNames.
@ -258,32 +217,24 @@ public class IndividualToSolrDocument {
* @returns true if prohibited from search
* @throws SkipIndividualException
*/
protected boolean addClasses(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{
protected void addClasses(Individual ind, SolrInputDocument doc, StringBuffer classPublicNames) throws SkipIndividualException{
ArrayList<String> superClassNames = null;
// Types and classgroups
boolean prohibited = false;
List<VClass> vclasses = ind.getVClasses(false);
superClassNames = new ArrayList<String>();
if( vclasses == null || vclasses.isEmpty() ){
throw new SkipIndividualException("Not indexing because individual has no super classes");
}
for(VClass clz : vclasses){
String superLclName = clz.getLocalName();
superClassNames.add(superLclName);
if(clz.getURI() == null){
continue;
}else if(OWL.Thing.getURI().equals(clz.getURI())){
//index individuals of type owl:Thing, just don't add owl:Thing as the type field in the index
//don't add owl:Thing as the type in the index
continue;
} else if(clz.getURI().startsWith(OWL.NS)){
throw new SkipIndividualException("not indexing " + ind.getURI() + " because of type " + clz.getURI() );
}
// do not index individuals of type Role, AdvisingRelationShip, Authorship, etc.(see search.n3 for more information)
else if(classesProhibitedFromSearch.isClassProhibitedFromSearch(clz.getURI())){
throw new SkipIndividualException("not indexing " + ind.getURI() + " because of prohibited type " + clz.getURI() );
} else {
if( !prohibited && classesProhibitedFromSearch.isClassProhibitedFromSearch(clz.getURI()))
prohibited = true;
if( clz.getSearchBoost() != null)
} else {
if( clz.getSearchBoost() != null){
doc.setDocumentBoost(doc.getDocumentBoost() + clz.getSearchBoost());
}
doc.addField(term.RDFTYPE, clz.getURI());
@ -302,14 +253,7 @@ public class IndividualToSolrDocument {
doc.addField(term.CLASSGROUP_URI,clz.getGroupURI());
}
}
}
if(superClassNames.isEmpty()){
throw new SkipIndividualException("Not indexing because individual has no super classes");
}
doc.addField(term.PROHIBITED_FROM_TEXT_RESULTS, prohibited?"1":"0");
return prohibited;
}
}
public Object getIndexId(Object obj) {
@ -318,7 +262,7 @@ public class IndividualToSolrDocument {
public String getIdForUri(String uri){
if( uri != null ){
return entClassName + uri;
return "vitroIndividual:" + uri;
}else{
return null;
}
@ -352,6 +296,5 @@ public class IndividualToSolrDocument {
}
}
public static float NAME_BOOST = 1.2F;
protected static final String DONT_EXCLUDE =null;
}

20
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/NameBoost.java
View file

@ -10,10 +10,21 @@ import edu.cornell.mannlib.vitro.webapp.search.VitroSearchTermNames;
public class NameBoost implements DocumentModifier {
static VitroSearchTermNames term = new VitroSearchTermNames();
/**
* These are the fields in the solr Document that
* are related to the name. If you modify the schema,
* please consider if you need to change this list
* of name fields to boost.
*/
static final VitroSearchTermNames term = new VitroSearchTermNames();
String[] fieldsToBoost = {term.NAME_RAW,term.NAME_LOWERCASE,term.NAME_UNSTEMMED,term.NAME_STEMMED};
static final float NAME_BOOST = (float) 1.2;
final float boost;
public NameBoost(float boost){
this.boost = boost;
}
@Override
public void modifyDocument(Individual individual, SolrInputDocument doc,
@ -21,8 +32,9 @@ public class NameBoost implements DocumentModifier {
for( String fieldName : fieldsToBoost){
SolrInputField field = doc.getField(fieldName);
if( field != null )
field.setBoost(field.getBoost() * NAME_BOOST);
if( field != null ){
field.setBoost(field.getBoost() + boost);
}
}
}

19
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SearchIndexExcluder.java Normal file
View file

@ -0,0 +1,19 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
/**
* Interface for classes that check to see if an individual
* should be excluded from the search index.
*/
public interface SearchIndexExcluder {
/**
* REturn a string message if the individual should
* be excluded from the index.
*
* Return null if ind should not be excluded.
*/
public String checkForExclusion(Individual ind);
}

70
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SolrSetup.java
View file

@ -18,17 +18,19 @@ import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
import org.apache.solr.client.solrj.impl.XMLResponseParser;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.vocabulary.OWL;
import edu.cornell.mannlib.vitro.webapp.config.ConfigurationProperties;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.IndividualDao;
import edu.cornell.mannlib.vitro.webapp.dao.VitroVocabulary;
import edu.cornell.mannlib.vitro.webapp.dao.WebappDaoFactory;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.WebappDaoFactoryFiltering;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilterUtils;
import edu.cornell.mannlib.vitro.webapp.dao.filtering.filters.VitroFilters;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.search.beans.FileBasedProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.beans.IndividualProhibitedFromSearchImpl;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
import edu.cornell.mannlib.vitro.webapp.search.beans.StatementToURIsToUpdate;
import edu.cornell.mannlib.vitro.webapp.search.indexing.AdditionalURIsForContextNodes;
@ -40,11 +42,32 @@ import edu.cornell.mannlib.vitro.webapp.search.indexing.SearchReindexingListener
import edu.cornell.mannlib.vitro.webapp.search.indexing.URIsForClassGroupChange;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
public class SolrSetup implements javax.servlet.ServletContextListener{
private static final Log log = LogFactory.getLog(SolrSetup.class.getName());
public class SolrSetup implements javax.servlet.ServletContextListener{
public static final String SOLR_SERVER = "vitro.local.solr.server";
public static final String PROHIBITED_FROM_SEARCH = "edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch";
/** Exclude from the search index Individuals with types from these namespaces */
private static final String[] TYPE_NS_EXCLUDES = {
VitroVocabulary.PUBLIC
//if you do OWL.NS here you will exclude all of owl:Thing.
};
/** Exclude from the search index individuals who's URIs start with these namespaces. */
private static final String[] INDIVIDUAL_NS_EXCLUDES={
VitroVocabulary.vitroURI,
VitroVocabulary.VITRO_PUBLIC,
VitroVocabulary.PSEUDO_BNODE_NS,
OWL.NS
};
/** Individuals of these types will be excluded from the search index */
private static final String[] OWL_TYPES_EXCLUDES = {
OWL.ObjectProperty.getURI(),
OWL.DatatypeProperty.getURI(),
OWL.AnnotationProperty.getURI()
};
@Override
public void contextInitialized(ServletContextEvent sce) {
@ -85,31 +108,42 @@ public class SolrSetup implements javax.servlet.ServletContextListener{
/* set up the individual to solr doc translation */
OntModel jenaOntModel = ModelContext.getJenaOntModel(context);
Model displayModel = ModelContext.getDisplayModel(context);
/* try to get context attribute DocumentModifiers
* and use that as the start of the list of DocumentModifier
* objects. This allows other listeners to add to the basic set of
* DocumentModifiers. */
List<DocumentModifier> modifiers = (List<DocumentModifier>)context.getAttribute("DocumentModifiers");
* objects. This allows other ContextListeners to add to
* the basic set of DocumentModifiers. */
@SuppressWarnings("unchecked")
List<DocumentModifier> modifiers =
(List<DocumentModifier>)context.getAttribute("DocumentModifiers");
if( modifiers == null )
modifiers = new ArrayList<DocumentModifier>();
modifiers.add(new NameBoost());
modifiers.add(new ThumbnailImageURL(jenaOntModel));
modifiers.add(new NameBoost( 1.2f ));
modifiers.add(new ThumbnailImageURL(jenaOntModel));
// setup prohibited from search based on N3 files in the directory WEB-INF/ontologies/search
/* try to get context attribute SearchIndexExcludes
* and use that as the start of the list of exclude
* objects. This allows other ContextListeners to add to
* the basic set of SearchIndexExcludes . */
@SuppressWarnings("unchecked")
List<SearchIndexExcluder> excludes =
(List<SearchIndexExcluder>)context.getAttribute("SearchIndexExcludes");
if( excludes == null )
excludes = new ArrayList<SearchIndexExcluder>();
File dir = new File(sce.getServletContext().getRealPath("/WEB-INF/ontologies/search"));
ProhibitedFromSearch pfs = new FileBasedProhibitedFromSearch(DisplayVocabulary.SEARCH_INDEX_URI, dir);
context.setAttribute(PROHIBITED_FROM_SEARCH,pfs);
excludes.add(new ExcludeBasedOnNamespace( INDIVIDUAL_NS_EXCLUDES ));
excludes.add(new ExcludeBasedOnTypeNamespace( TYPE_NS_EXCLUDES ) );
excludes.add(new ExcludeBasedOnType( OWL_TYPES_EXCLUDES) );
excludes.add(new ExcludeNonFlagVitro() );
excludes.add( new SyncingExcludeBasedOnType( displayModel ) );
IndividualToSolrDocument indToSolrDoc = new IndividualToSolrDocument(
pfs,
new IndividualProhibitedFromSearchImpl(context),
modifiers);
/* setup solr indexer */
IndividualToSolrDocument indToSolrDoc =
new IndividualToSolrDocument(excludes, modifiers);
/* setup solr indexer */
SolrIndexer solrIndexer = new SolrIndexer(server, indToSolrDoc);
// This is where the builder gets the list of places to try to

199
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/SyncingExcludeBasedOnType.java Normal file
View file

@ -0,0 +1,199 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.solr;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.Query;
import com.hp.hpl.jena.query.QueryExecution;
import com.hp.hpl.jena.query.QueryExecutionFactory;
import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.query.QuerySolution;
import com.hp.hpl.jena.query.QuerySolutionMap;
import com.hp.hpl.jena.query.ResultSet;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelChangedListener;
import com.hp.hpl.jena.rdf.model.RDFNode;
import com.hp.hpl.jena.rdf.model.Resource;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import com.hp.hpl.jena.rdf.model.StmtIterator;
import com.hp.hpl.jena.shared.Lock;
import edu.cornell.mannlib.vitro.webapp.dao.DisplayVocabulary;
/**
* This excludes based on types defined as EXCLUDE_CLASS in the
* configuration RDF model.
*/
public class SyncingExcludeBasedOnType extends ExcludeBasedOnType implements ModelChangedListener{
static final Log log = LogFactory.getLog(SyncingExcludeBasedOnType.class);
private static final String queryForProhibitedClasses =
"SELECT ?prohibited WHERE{" +
"?searchConfig <" + DisplayVocabulary.EXCLUDE_CLASS + "> ?prohibited . " +
"}";
String searchIndexURI = DisplayVocabulary.SEARCH_INDEX_URI;
public SyncingExcludeBasedOnType( Model model){
this.setExcludedTypes( buildProhibitedClassesList(searchIndexURI, model) );
}
private List<String> buildProhibitedClassesList( String URI, Model model){
List<String> newProhibitedClasses = new ArrayList<String>();
QuerySolutionMap initialBinding = new QuerySolutionMap();
Resource searchConfig = ResourceFactory.createResource(URI);
initialBinding.add("searchConfig", searchConfig);
Query query = QueryFactory.create(queryForProhibitedClasses);
model.enterCriticalSection(Lock.READ);
try{
QueryExecution qExec = QueryExecutionFactory.create(query,model,initialBinding);
try{
ResultSet results = qExec.execSelect();
for(;results.hasNext();){
QuerySolution soln = results.nextSolution();
RDFNode n = soln.get("prohibited");
if( n.isResource() && !n.isAnon()){
newProhibitedClasses.add(((Resource) n).getURI());
}else{
log.warn("unexpected node in object position for prohibited classes: " + n.toString());
}
}
}catch(Throwable t){
log.error(t,t);
}finally{ qExec.close(); }
}finally{ model.leaveCriticalSection(); }
return newProhibitedClasses;
}
/* ************* Methods for ModelChangeListener *************** */
@Override
public void addedStatement(Statement s) {
try{
if( isExcludeClassPredicate( s ) && isAboutSearchIndex(s)){
if( s.getObject() != null && s.getObject().canAs(Resource.class)){
String classURI = ((Resource)s.getObject().as(Resource.class)).getURI();
this.addTypeToExclude(classURI);
}
}
}catch(Exception ex){
log.error("could not add statement",ex);
}
}
@Override
public void removedStatement(Statement s) {
try{
if( isExcludeClassPredicate( s ) && isAboutSearchIndex(s)){
if( s.getObject() != null && s.getObject().canAs(Resource.class)){
String classURI = ((Resource)s.getObject().as(Resource.class)).getURI();
this.removeTypeToExclude(classURI);
}
}
}catch(Exception ex){
log.error("could not remove statement",ex);
}
}
private boolean isExcludeClassPredicate(Statement s){
return s != null
&& s.getPredicate() != null
&& DisplayVocabulary.EXCLUDE_CLASS.getURI().equals( s.getPredicate().getURI());
}
private boolean isAboutSearchIndex(Statement s){
if( s.getSubject() != null ){
String subURI = ((Resource) s.getSubject()).getURI() ;
return this.searchIndexURI.equals(subURI);
}else{
return false;
}
}
@Override
public void addedStatements(Statement[] stmts) {
if( stmts != null ){
for( Statement stmt : stmts){
addedStatement(stmt);
}
}
}
@Override
public void addedStatements(List<Statement> stmts) {
if( stmts != null ){
for( Statement stmt : stmts){
addedStatement(stmt);
}
}
}
@Override
public void addedStatements(StmtIterator it) {
while(it.hasNext()){
Statement stmt = it.nextStatement();
addedStatement(stmt);
}
}
@Override
public void addedStatements(Model model) {
if( model != null){
addedStatements(model.listStatements(
model.createResource(searchIndexURI),
DisplayVocabulary.EXCLUDE_CLASS,
(RDFNode)null));
}
}
@Override
public void notifyEvent(Model arg0, Object arg1) {
//nothing
}
@Override
public void removedStatements(Statement[] stmts) {
if( stmts != null ){
for( Statement stmt : stmts){
removedStatement(stmt);
}
}
}
@Override
public void removedStatements(List<Statement> stmts) {
if( stmts != null ){
for( Statement stmt : stmts){
removedStatement(stmt);
}
}
}
@Override
public void removedStatements(StmtIterator it) {
while(it.hasNext()){
Statement stmt = it.nextStatement();
removedStatement(stmt);
}
}
@Override
public void removedStatements(Model model) {
if( model != null){
removedStatements(model.listStatements(
model.createResource(searchIndexURI),
DisplayVocabulary.EXCLUDE_CLASS,
(RDFNode)null));
}
}
}

16
webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/ThumbnailImageURL.java
View file

@ -61,9 +61,23 @@ public class ThumbnailImageURL implements DocumentModifier {
//add a field for storing the location of thumbnail for the individual.
doc.addField(fieldForThumbnailURL, runQueryForThumbnailLocation(individual));
addThumbnailExistance(individual, doc);
}
/**
* Adds if the individual has a thumbnail image or not.
*/
protected void addThumbnailExistance(Individual ind, SolrInputDocument doc) {
try{
if(ind.hasThumb())
doc.addField(term.THUMBNAIL, "1");
else
doc.addField(term.THUMBNAIL, "0");
}catch(Exception ex){
log.debug("could not index thumbnail: " + ex);
}
}
protected String runQueryForThumbnailLocation(Individual individual) {
StringBuffer result = new StringBuffer();

3
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/ApplicationModelSetup.java
View file

@ -25,6 +25,7 @@ import com.hp.hpl.jena.query.QueryFactory;
import com.hp.hpl.jena.rdf.model.Model;
import com.hp.hpl.jena.rdf.model.ModelFactory;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelContext;
import edu.cornell.mannlib.vitro.webapp.dao.jena.ModelSynchronizer;
import edu.cornell.mannlib.vitro.webapp.startup.StartupStatus;
@ -65,7 +66,7 @@ implements ServletContextListener {
OntModel displayModel = ModelFactory.createOntologyModel(MEM_ONT_MODEL_SPEC);
displayModel.add(displayDbModel);
displayModel.getBaseModel().register(new ModelSynchronizer(displayDbModel));
ctx.setAttribute(DISPLAY_ONT_MODEL, displayModel);
ModelContext.setDisplayModel(displayModel, ctx);
//at each startup load all RDF files from directory to sub-models of display model
initializeDisplayLoadedAtStartup(ctx, displayModel);

8
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/JenaDataSourceSetupBase.java
View file

@ -287,6 +287,10 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
return firstStartup;
}
public static void thisIsFirstStartup(){
firstStartup = true;
}
protected Model makeDBModel(BasicDataSource ds,
String jenaDbModelname,
OntModelSpec jenaDbOntModelSpec,
@ -378,10 +382,10 @@ public class JenaDataSourceSetupBase extends JenaBaseDaoCon {
}
} else {
if(!f.exists()) {
log.debug("File for path " + p + " does not exist");
log.info("File for path " + p + " does not exist");
}
else if(f.isDirectory()) {
log.debug("Path " + p +
log.info("Path " + p +
" corresponds to directory and not file so was not read in");
}
}

15
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/SDBSetup.java
View file

@ -68,13 +68,11 @@ implements javax.servlet.ServletContextListener {
Store store = connectStore(bds, storeDesc);
setApplicationStore(store, ctx);
if (!isSetUp(store)) {
log.info("Initializing SDB store");
if (isFirstStartup()) {
setupSDB(ctx, store);
} else {
migrateToSDBFromExistingRDBStore(ctx, store);
}
if (!isSetUp(store)) {
JenaPersistentDataSourceSetup.thisIsFirstStartup();
setupSDB(ctx, store);
}else{
migrateToSDBFromExistingRDBStore(ctx, store);
}
}
@ -124,7 +122,8 @@ implements javax.servlet.ServletContextListener {
protected static void setupSDB(ServletContext ctx, Store store,
Model memModel, Model inferenceModel) {
log.info("Initializing SDB store");
store.getTableFormatter().create();
store.getTableFormatter().truncate();

4
webapp/src/edu/cornell/mannlib/vitro/webapp/servlet/setup/WebappDaoSDBSetup.java
View file

@ -249,11 +249,13 @@ public class WebappDaoSDBSetup extends JenaDataSourceSetupBase
private void loadDataFromFilesystem(OntModelSelector baseOms,
ServletContext ctx) {
Long startTime = System.currentTimeMillis();
log.debug("Initializing models from RDF files");
log.info("Initializing models from RDF files");
readOntologyFilesInPathSet(USER_ABOX_PATH, ctx, baseOms.getABoxModel());
readOntologyFilesInPathSet(USER_TBOX_PATH, ctx, baseOms.getTBoxModel());
readOntologyFilesInPathSet(
USER_APPMETA_PATH, ctx, baseOms.getApplicationMetadataModel());
log.debug(((System.currentTimeMillis() - startTime) / 1000)
+ " seconds to read RDF files ");
}

7
webapp/test/stubs/edu/cornell/mannlib/vitro/webapp/beans/IndividualStub.java
View file

@ -21,7 +21,6 @@ import edu.cornell.mannlib.vitro.webapp.beans.ObjectProperty;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatement;
import edu.cornell.mannlib.vitro.webapp.beans.ObjectPropertyStatementImpl;
import edu.cornell.mannlib.vitro.webapp.beans.VClass;
import edu.cornell.mannlib.vitro.webapp.search.beans.ProhibitedFromSearch;
/**
* Mock the basic functions of Individual for unit tests.
@ -380,12 +379,6 @@ public class IndividualStub implements Individual {
throw new RuntimeException("Individual.setVClasses() not implemented.");
}
@Override
public boolean isMemberOfClassProhibitedFromSearch(ProhibitedFromSearch pfs) {
throw new RuntimeException(
"Individual.isMemberOfClassProhibitedFromSearch() not implemented.");
}
@Override
public void setObjectPropertyStatements(List<ObjectPropertyStatement> list) {
throw new RuntimeException(