Changing solr analysis for name, NIHVIVO-3332. Attempting to remove duplicate strings due to copy fields in Solr index.

solr/homeDirectoryTemplate/conf/schema.xml

@@ -1,43 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-
-<!--  
- PERFORMANCE NOTE: this schema includes many optional features and should not
- be used for benchmarking.  To improve performance one could
-  - set stored="false" for all fields possible (esp large fields) when you
-    only need to search on the field but don't need to return the original
-    value.
-  - set indexed="false" if you don't need to search on the field, but only
-    return the field as a result of searching on other indexed fields.
-  - remove all unneeded copyField statements
-  - for best index size and searching performance, set "index" to false
-    for all general text fields, use copyField to copy them to the
-    catchall "text" field, and use that for searching.
-  - For maximum indexing performance, use the StreamingUpdateSolrServer
-    java client.
-  - Remember to run the JVM in server mode, and use a higher logging level
-    that avoids logging every request
--->
-
 <schema name="example" version="1.3">
-  <!-- attribute "name" is the name of this schema and is only used for display purposes.
-       Applications should change this to reflect the nature of the search collection.
-       version="1.2" is Solr's version number for the schema syntax and semantics.  It should
-       not normally be changed by applications.
-       1.0: multiValued attribute did not exist, all fields are multiValued by nature
-       1.1: multiValued attribute introduced, false by default 
-       1.2: omitTermFreqAndPositions attribute introduced, true by default except for text fields.
-       1.3: removed optional field compress feature
-     -->
   
   <types>
-    <!-- field type definitions. The "name" attribute is
-       just a label to be used by field definitions.  The "class"
-       attribute and any other attributes determine the real
-       behavior of the fieldType.
-         Class names starting with "solr" refer to java classes in the
-       org.apache.solr.analysis package.
-    -->
-
     <!-- The StrField type is not analyzed, but indexed/stored verbatim. -->
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
 
@@ -47,107 +11,21 @@
     <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
     <fieldtype name="binary" class="solr.BinaryField"/>
     
-    <!-- The optional sortMissingLast and sortMissingFirst attributes are
-         currently supported on types that are sorted internally as strings.
-	       This includes "string","boolean","sint","slong","sfloat","sdouble","pdate"
-       - If sortMissingLast="true", then a sort on this field will cause documents
-         without the field to come after documents with the field,
-         regardless of the requested sort order (asc or desc).
-       - If sortMissingFirst="true", then a sort on this field will cause documents
-         without the field to come before documents with the field,
-         regardless of the requested sort order.
-       - If sortMissingLast="false" and sortMissingFirst="false" (the default),
-         then default lucene sorting will be used which places docs without the
-         field first in an ascending sort and last in a descending sort.
-    -->    
-
-    <!--
-      Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
-    -->
     <fieldType name="int" class="solr.TrieIntField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
     <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
     <fieldType name="long" class="solr.TrieLongField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/>
     <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" omitNorms="true" positionIncrementGap="0"/> 
    
-    <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
-         is a more restricted form of the canonical representation of dateTime
-         http://www.w3.org/TR/xmlschema-2/#dateTime    
-         The trailing "Z" designates UTC time and is mandatory.
-         Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
-         All other components are mandatory.
-
-         Expressions can also be used to denote calculations that should be
-         performed relative to "NOW" to determine the value, ie...
-
-               NOW/HOUR
-                  ... Round to the start of the current hour
-               NOW-1DAY
-                  ... Exactly 1 day prior to now
-               NOW/DAY+6MONTHS+3DAYS
-                  ... 6 months and 3 days in the future from the start of
-                      the current day
-                      
-         Consult the DateField javadocs for more information.
-
-         Note: For faster range queries, consider the tdate type
-      -->
     <fieldType name="date" class="solr.TrieDateField" omitNorms="true" precisionStep="0" positionIncrementGap="0"/>
-
     <!-- A Trie based date field for faster date range queries and date faceting. -->
     <fieldType name="tdate" class="solr.TrieDateField" omitNorms="true" precisionStep="6" positionIncrementGap="0"/>
 
-    <!-- The "RandomSortField" is not used to store or search any
-         data.  You can declare fields of this type it in your schema
-         to generate pseudo-random orderings of your docs for sorting 
-         purposes.  The ordering is generated based on the field name 
-         and the version of the index, As long as the index version
-         remains unchanged, and the same field name is reused,
-         the ordering of the docs will be consistent.  
-         If you want different psuedo-random orderings of documents,
-         for the same version of the index, use a dynamicField and
-         change the name
-     -->
     <fieldType name="random" class="solr.RandomSortField" indexed="true" />
   
-    <!-- solr.TextField allows the specification of custom text analyzers
-         specified as a tokenizer and a list of token filters. Different
-         analyzers may be specified for indexing and querying.
-
-         The optional positionIncrementGap puts space between multiple fields of
-         this type on the same document, with the purpose of preventing false phrase
-         matching across fields.
-
-         For more info on customizing your analyzer chain, please see
-         http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
-     -->    
-
-    <!-- A text field that uses WordDelimiterFilter to enable splitting and matching of
-        words on case-change, alpha numeric boundaries, and non-alphanumeric chars,
-        so that a query of "wifi" or "wi fi" could match a document containing "Wi-Fi".
-        Synonyms and stopwords are customized by external files, and stemming is enabled.
-        The attribute autoGeneratePhraseQueries="true" (the default) causes words that get split to
-        form phrase queries. For example, WordDelimiterFilter splitting text:pdp-11 will cause the parser
-        to generate text:"pdp 11" rather than (text:PDP OR text:11).
-        NOTE: autoGeneratePhraseQueries="true" tends to not work well for non whitespace delimited languages.
-    -->
-    <!--
-    <fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
-      <analyzer>
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>       
-        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />      
-        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" 
-          catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
-        <filter class="solr.ASCIIFoldingFilterFactory"/>
-        <filter class="solr.LowerCaseFilterFactory"/>
-        <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>        
-        <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>        
-      </analyzer>
-    </fieldType>-->
-  
     <fieldType name="text" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
     <analyzer type="index">
       <tokenizer class="solr.WhitespaceTokenizerFactory"/>      
-      <filter class="solr.ISOLatin1AccentFilterFactory"/>
+      <filter class="solr.ASCIIFoldingFilterFactory"/>
       <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
       <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" 
               catenateWords="1" catenateNumbers="1" catenateAll="0"/>
@@ -157,7 +35,7 @@
     </analyzer>
     <analyzer type="query">
       <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-      <filter class="solr.ISOLatin1AccentFilterFactory"/>      
+      <filter class="solr.ASCIIFoldingFilterFactory"/>      
       <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
       <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"/>
       <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" 
@@ -168,32 +46,24 @@
     </analyzer>
     </fieldType>
         
-    <!-- Like text, but without synonyms and stemming. Good for autocomplete where we want to remove
+    <!--  This field intentionally lacks solr.WordDelimiterFilterFactory because of issue NIHVIVO-3332 -->
-      stop words but not stem. -->
     <fieldType name="text_unstemmed" class="solr.TextField" positionIncrementGap="100">
       <analyzer>
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-        <filter class="solr.StopFilterFactory" ignoreCase="true"
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
-          words="stopwords.txt"  enablePositionIncrements="true" />               
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt"  enablePositionIncrements="true" />                                                                             
-        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1"                 
-          generateNumberParts="1" catenateWords="0"                 
-          catenateNumbers="0" catenateAll="0"                
-          splitOnCaseChange="1" />                                               
         <filter class="solr.LowerCaseFilterFactory"/> 
       </analyzer>  
     </fieldType>
 
-    <!-- Like text, but without synonyms. Good for autocomplete matching of book/grant titles, etc., where we want to remove
+    <!-- Like text, but without synonyms. Good for autocomplete matching of book/grant titles, etc.,
-      stop words and stem. -->
+         where we want to remove stop words and stem. -->
     <fieldType name="text_stemmed" class="solr.TextField" positionIncrementGap="100">
       <analyzer>
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true"
           words="stopwords-name.txt"  enablePositionIncrements="true" />               
-        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1"                 
+        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1"  generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1" />
-          generateNumberParts="1" catenateWords="0"                 
-          catenateNumbers="0" catenateAll="0"                
-          splitOnCaseChange="1" />
         <filter class="solr.ASCIIFoldingFilterFactory"/>        
         <filter class="solr.LowerCaseFilterFactory"/>        
         <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>  
@@ -207,17 +77,15 @@
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="0"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
-        <filter class="solr.StopFilterFactory"
+        <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
-                ignoreCase="true"
-                words="stopwords.txt"
-                enablePositionIncrements="true"
-                />
         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.LowerCaseFilterFactory"/>
       </analyzer>
     </fieldType>
@@ -234,6 +102,7 @@
     <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
       <analyzer>
         <tokenizer class="solr.KeywordTokenizerFactory"/>
+        <filter class="solr.ASCIIFoldingFilterFactory"/>
         <filter class="solr.LowerCaseFilterFactory" />
       </analyzer>
     </fieldType>
@@ -280,34 +149,6 @@
       </analyzer>      
     </fieldtype>    
     
-    <!-- RY: Commenting this fieldtype out for now because we have no use case for 
-      a tokenized, unstemmed autocomplete field. Identical to edgengram_stemmed but without
-      the stemming.
-    <fieldtype name="edgengram_unstemmed" class="solr.TextField">
-      <analyzer type="index">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>   
-        <filter class="solr.StopFilterFactory" ignoreCase="true"
-          words="stopwords.txt"  enablePositionIncrements="true" /> 
-        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1"                 
-          generateNumberParts="1" catenateWords="0"                 
-          catenateNumbers="0" catenateAll="0"                
-          splitOnCaseChange="1" />  
-        <filter class="solr.LowerCaseFilterFactory" />
-        <filter class="solr.EdgeNGramFilterFactory" minGramSize="1" maxGramSize="25" side="front"/>
-      </analyzer>
-      <analyzer type="query">
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>   
-        <filter class="solr.StopFilterFactory" ignoreCase="true"
-          words="stopwords.txt"  enablePositionIncrements="true" />  
-        <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1"                 
-          generateNumberParts="1" catenateWords="0"                 
-          catenateNumbers="0" catenateAll="0"                
-          splitOnCaseChange="1" />   
-        <filter class="solr.LowerCaseFilterFactory" /> 
-      </analyzer>      
-    </fieldtype>
-    -->
-    
  </types>
 
 
@@ -358,7 +199,6 @@
      
   <!-- Autocomplete search fields -->
   <field name="acNameUntokenized" type="edgengram_untokenized" indexed="true" stored="false" multiValued="true" />
-  <!-- <field name="acNameUnstemmed" type="edgengram_unstemmed" indexed="true" stored="false" multiValued="true" /> -->
   <field name="acNameStemmed" type="edgengram_stemmed" indexed="true" stored="false" multiValued="true" />
   
   <field name="indexedTime" type="long" indexed="true" stored="true"/>
@@ -370,7 +210,6 @@
   
   <field name="THUMBNAIL" type="string" indexed="true" stored="true"/>
     
-  <!-- <field name="targetInfo" type="text" indexed="true" stored="false" multiValued="true"/> -->
   <field name="BETA" type="float" indexed="true" stored="true" multiValued="false"/>
   <!-- <field name="PHI" type="float" indexed="true" stored="true" multiValued="false"/>
   <field name="ADJACENT_NODES" type="string" indexed="true" stored="true" multiValued="true"/> -->
@@ -390,62 +229,34 @@
   <!--  field for preferred title -->
   <field name="PREFERRED_TITLE" type="string" indexed="true" stored="true"/>
   
-  <!-- Copy nameLowercase to sortable field. -->  
+  <!-- Copy nameRaw into several other fields -->  
   <copyField source="nameRaw" dest="nameStemmed" />
   <copyField source="nameRaw" dest="nameUnstemmed" />
   <copyField source="nameRaw" dest="nameLowercase" />
+  <copyField source="nameRaw" dest="NAME_PHONETIC" />
+  <copyField source="nameRaw" dest="acNameUntokenized" />
+  <copyField source="nameRaw" dest="acNameStemmed" />
   <copyField source="nameLowercase" dest="nameLowercaseSingleValued" />
      
   <!-- ****************************  End Vitro Fields *************************** -->
    
-
-   
-   <!-- Uncommenting the following will create a "timestamp" field using
-        a default value of "NOW" to indicate when each document was indexed.
-     -->
    <field name="timestamp" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>     
  </fields>
 
  <!-- Field to use to determine and enforce document uniqueness. 
-      Unless this field is marked with required="false", it will be a required field
+      Unless this field is marked with required="false", it will be a required field -->
-   -->
  <uniqueKey>DocId</uniqueKey>
 
  <!-- field for the QueryParser to use when an explicit fieldname is absent -->
-<!-- <defaultSearchField>ALLTEXT</defaultSearchField> -->
+ <!--  This is now set in solrconfig.xml  -->
+ <!-- <defaultSearchField>ALLTEXT</defaultSearchField> -->
 
  <!-- SolrQueryParser configuration: defaultOperator="AND|OR" -->
  <solrQueryParser defaultOperator="AND"/>
 
-  <!-- copyField commands copy one field to another at the time a document
-        is added to the index.  It's used either to index the same field differently,
-        or to add multiple fields to the same field for easier/faster searching.  -->
-	
-   <!-- Above, multiple source fields are copied to the [text] field. 
-	  Another way to map multiple source fields to the same 
-	  destination field is to use the dynamic field syntax. 
-	  copyField also supports a maxChars to copy setting.  -->
-	   
-   <!-- <copyField source="*_t" dest="text" maxChars="3000"/> -->
-
-   <!-- copy name to alphaNameSort, a field designed for sorting by name -->
-   <!-- <copyField source="name" dest="alphaNameSort"/> -->
- 
-
  <!-- Similarity is the scoring routine for each document vs. a query.
       A custom similarity may be specified here, but the default is fine
       for most applications.  -->
-<similarity class="org.apache.lucene.search.DefaultSimilarity"/>
+ <similarity class="org.apache.lucene.search.DefaultSimilarity"/>
-<!-- <similarity class="edu.cornell.mannlib.vitro.webapp.search.lucene.CustomSimilarity"/> -->
- <!-- ... OR ...
-      Specify a SimilarityFactory class name implementation
-      allowing parameters to be used.
- -->
- <!--
- <similarity class="com.example.solr.CustomSimilarityFactory">
-   <str name="paramkey">param value</str>
- </similarity>
- -->
-
 
 </schema>

webapp/src/edu/cornell/mannlib/vitro/webapp/search/solr/IndividualToSolrDocument.java

@@ -204,13 +204,10 @@ public class IndividualToSolrDocument {
         } else {
             value = ind.getLocalName();
         }            
+
         doc.addField(term.NAME_RAW, value);
-        doc.addField(term.NAME_LOWERCASE, value);
+        // NAME_RAW will be copied by solr into the following fields:
-        doc.addField(term.NAME_UNSTEMMED, value);
+        // NAME_LOWERCASE, NAME_UNSTEMMED, NAME_STEMMED, NAME_PHONETIC, AC_NAME_UNTOKENIZED, AC_NAME_STEMMED
-        doc.addField(term.NAME_STEMMED, value);
-        doc.addField(term.NAME_PHONETIC, value);
-        doc.addField(term.AC_NAME_UNTOKENIZED, value);
-        doc.addField(term.AC_NAME_STEMMED, value);         
     }
 
     /**