Handle BibTeX files with non-ascii encodings

git-svn-id: svn://svn.code.sf.net/p/writer2latex/code/trunk@263 f0f2a975-2e09-46c8-9428-3b39399b9f3c

build.xml

@@ -2,7 +2,7 @@
   ############################################################################
   # This is the Ant build file for writer2latex
   # Original: Sep 2004 (mgn)
-  # version 1.6 (2015-05-28)
+  # version 1.6 (2015-07-22)
   ############################################################################
 -->
 <project name="w2l" default="help" basedir=".">
@@ -36,8 +36,8 @@
 
     <!-- configure the directories -->
     <property name="jarfile" value="writer2latex"/>
-    <property name="basename" value="writer2latex15"/>
+    <property name="basename" value="writer2latex16"/>
-    <property name="distrofile" value="${basename}3alpha.zip" />
+    <property name="distrofile" value="${basename}beta.zip" />
     <!--<property name="sourcedistrofile" value="${basename}source.zip" />-->
     <property name="src" location="source/java"/>
     <property name="source.distro" location="source/distro" />

source/distro/Readme.txt

@@ -1,7 +1,7 @@
-Writer2LaTeX version 1.5.3 (alpha test release)
+Writer2LaTeX version 1.6 (beta test release)
-===============================================
+============================================
 
-This is the distribution of Writer2LaTeX version 1.5.3 alpha
+This is the distribution of Writer2LaTeX version 1.6 beta
 
 Latest version can be found at the web site
   http://writer2latex.sourceforge.net
@@ -14,5 +14,5 @@ Bugs and feature requests should be reported to
   writer2latex (at) gmail.com
   
 
-May 2015
+July 2015
 Henrik Just

source/distro/changelog.txt

@@ -1,5 +1,15 @@
 Changelog for Writer2LaTeX version 1.4 -> 1.6
 
+---------- version 1.6 beta ----------
+
+[w2l] Added support for non-ASCII BibTeX files. The new option bibtex_encoding is used to specify the encoding of
+  the BibTeX files loaded via the option external_bibtex_files. The default value is "document" which implies the
+  same encoding as the LaTeX document. Otherwise the same values as for inputencoding are supported.
+  In the UI, the encoding can be selected on the bibliography page of the toolbar settings.
+  The setting is ignored if the backend is XeTeX, in this case all files are assumed to be UTF-8.
+  Also, the default bibtex application has been changed to bibtex8 (but currently no attempt is made to select the
+  proper .csf file)
+
 ---------- version 1.5.3 ----------
 
 [w2l] The command \textsubscript is now defined with \providecommand to avoid problems with other packages that may

source/java/org/openoffice/da/comp/writer2latex/BibTeXDialog.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-05-29)
+ *  Version 1.6 (2015-07-24)
  *
  */ 
  
@@ -61,6 +61,7 @@ import com.sun.star.text.XTextViewCursor;
 import com.sun.star.text.XTextViewCursorSupplier;
 import com.sun.star.ui.dialogs.ExecutableDialogResults;
 import com.sun.star.uno.AnyConverter;
+import com.sun.star.uno.Exception;
 import com.sun.star.uno.UnoRuntime;
 import com.sun.star.uno.XComponentContext;
 
@@ -68,7 +69,10 @@ import org.jbibtex.ParseException;
 import org.openoffice.da.comp.w2lcommon.helper.DialogAccess;
 import org.openoffice.da.comp.w2lcommon.helper.DialogBase;
 import org.openoffice.da.comp.w2lcommon.helper.MessageBox;
+import org.openoffice.da.comp.w2lcommon.helper.RegistryHelper;
+import org.openoffice.da.comp.w2lcommon.helper.XPropertySetHelper;
 
+import writer2latex.latex.i18n.ClassicI18n;
 import writer2latex.office.BibMark;
 import writer2latex.office.BibMark.EntryType;
 import writer2latex.util.Misc;
@@ -95,6 +99,9 @@ public class BibTeXDialog extends DialogBase implements com.sun.star.lang.XIniti
     // The BibTeX directory (passed at initialization)
     File bibTeXDirectory = null;
     
+    // The encoding for BibTeX files (set in constructor from the registry)
+    String sBibTeXJavaEncoding = null;
+    
     // Cache of BibTeX files in the BibTeX directory
     File[] files = null;
     
@@ -121,6 +128,22 @@ public class BibTeXDialog extends DialogBase implements com.sun.star.lang.XIniti
     /** Create a new BibTeXDialog */
     public BibTeXDialog(XComponentContext xContext) {
         super(xContext);
+        sBibTeXJavaEncoding = getBibTeXJavaEncoding();
+    }
+    
+    private String getBibTeXJavaEncoding() {
+    	RegistryHelper registry = new RegistryHelper(xContext);
+    	try {
+    		Object view = registry.getRegistryView(BibliographyDialog.REGISTRY_PATH, false);
+    		XPropertySet xProps = (XPropertySet) UnoRuntime.queryInterface(XPropertySet.class,view);
+        	int nBibTeXEncoding = XPropertySetHelper.getPropertyValueAsShort(xProps, "BibTeXEncoding"); //$NON-NLS-1$
+        	registry.disposeRegistryView(view);
+        	return ClassicI18n.writeJavaEncoding(nBibTeXEncoding);
+    	}
+    	catch (Exception e) {
+    		// Failed to get registry view
+    	}
+    	return null;
     }
 	
     /** Return the name of the library containing the dialog
@@ -257,11 +280,12 @@ public class BibTeXDialog extends DialogBase implements com.sun.star.lang.XIniti
     	int nFile = getListBoxSelectedItem("File"); //$NON-NLS-1$
     	if (nFile>=0) {
     		try {
-				currentFile = new BibTeXReader(files[nFile]);
+				currentFile = new BibTeXReader(files[nFile],sBibTeXJavaEncoding);
 			} catch (IOException e) {
+				System.err.println(e.getMessage());
 				currentFile = null;
 			} catch (ParseException e) {
-				System.out.println(e.getMessage());
+				System.err.println(e.getMessage());
 				currentFile = null;
 			}
     		
@@ -502,10 +526,12 @@ public class BibTeXDialog extends DialogBase implements com.sun.star.lang.XIniti
     	BibTeXReader[] readers = new BibTeXReader[nFiles];
     	for (int i=0; i<nFiles; i++) {
     		try {
-				readers[i] = new BibTeXReader(files[i]);
+				readers[i] = new BibTeXReader(files[i],sBibTeXJavaEncoding);
 			} catch (IOException e) {
+				System.err.println(e.getMessage());
 				readers[i] = null;
 			} catch (ParseException e) {
+				System.err.println(e.getMessage());
 				readers[i] = null;
  			}
     	}

source/java/org/openoffice/da/comp/writer2latex/BibTeXReader.java

@@ -16,19 +16,20 @@
  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
  *  MA  02111-1307  USA
  *
- *  Copyright: 2002-2014 by Henrik Just
+ *  Copyright: 2002-2015 by Henrik Just
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2014-12-27) 
+ *  Version 1.6 (2015-07-24)
  *
  */
 
 package org.openoffice.da.comp.writer2latex;
 
 import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.Collection;
@@ -43,6 +44,7 @@ import org.jbibtex.Key;
 import org.jbibtex.LaTeXParser;
 import org.jbibtex.LaTeXPrinter;
 import org.jbibtex.ParseException;
+import org.jbibtex.TokenMgrException;
 import org.jbibtex.Value;
 
 import writer2latex.bibtex.BibTeXEntryMap;
@@ -56,16 +58,19 @@ import writer2latex.office.BibMark.EntryType;
 public class BibTeXReader {
 
 	private File file;
+	private String sEncoding;
 	private Map<String, BibMark> entries;
 	
 	/** Construct a new <code>BibTeXReader</code> based on a file
 	 * 
 	 * @param file the file to read
+	 * @param sEncoding the character encoding of the file
 	 * @throws IOException if any error occurs reading the file
 	 * @throws ParseException if any error occurs interpreting the contents of the file
 	 */
-	public BibTeXReader(File file) throws IOException, ParseException {
+	public BibTeXReader(File file, String sEncoding) throws IOException, ParseException {
 		this.file = file;
+		this.sEncoding = sEncoding;
 		reload();
 	}
 
@@ -73,7 +78,7 @@ public class BibTeXReader {
 	 */
 	public void reload() throws IOException, ParseException {
 		entries = new HashMap<String, BibMark>();
-		BibTeXDatabase database = parseBibTeX(file);
+		BibTeXDatabase database = parseBibTeX(file,sEncoding);
 		readEntries(database);
 	}
 	
@@ -93,8 +98,9 @@ public class BibTeXReader {
 		return entries;
 	}
 
-	private static BibTeXDatabase parseBibTeX(File file) throws ParseException, IOException {
+	private static BibTeXDatabase parseBibTeX(File file, String sEncoding) throws ParseException, IOException {
-		Reader reader = new FileReader(file);
+		FileInputStream is = new FileInputStream(file);
+		Reader reader = new InputStreamReader(is,sEncoding);
 		try {
 			BibTeXParser parser = new BibTeXParser() {
 				@Override
@@ -114,7 +120,8 @@ public class BibTeXReader {
 			};
 			return parser.parse(reader);
 		} finally {
-			reader.close();
+			if (reader!=null) { reader.close(); }
+			if (is!=null) { is.close(); }
 		}
 	}
 
@@ -148,6 +155,9 @@ public class BibTeXReader {
 		} catch (ParseException e) {
 			// If parsing fails, return the original string
 			return string;
+		} catch (TokenMgrException e) {
+			// If the string contains invalid characters, return the original string
+			return string;
 		} finally {
 			try {
 				reader.close();

source/java/org/openoffice/da/comp/writer2latex/BibliographyDialog.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-05-29)
+ *  Version 1.6 (2015-07-23)
  *
  */ 
  
@@ -179,6 +179,8 @@ public final class BibliographyDialog
         			XPropertySetHelper.getPropertyValueAsShort(xProps, "BibTeXLocation")); //$NON-NLS-1$
         	dlg.setTextFieldText("BibTeXDir", //$NON-NLS-1$
         			XPropertySetHelper.getPropertyValueAsString(xProps, "BibTeXDir")); //$NON-NLS-1$
+        	dlg.setListBoxSelectedItem("BibTeXEncoding", //$NON-NLS-1$
+        			XPropertySetHelper.getPropertyValueAsShort(xProps, "BibTeXEncoding")); //$NON-NLS-1$
     		dlg.setCheckBoxStateAsBoolean("UseNatbib", //$NON-NLS-1$
     				XPropertySetHelper.getPropertyValueAsBoolean(xProps, "UseNatbib")); //$NON-NLS-1$
     		dlg.setTextFieldText("NatbibOptions", //$NON-NLS-1$
@@ -206,6 +208,7 @@ public final class BibliographyDialog
     		XPropertySetHelper.setPropertyValue(xProps, "IncludeOriginalCitations", dlg.getCheckBoxStateAsBoolean("IncludeOriginalCitations")); //$NON-NLS-1$ //$NON-NLS-2$
    			XPropertySetHelper.setPropertyValue(xProps, "BibTeXLocation", dlg.getListBoxSelectedItem("BibTeXLocation")); //$NON-NLS-1$ //$NON-NLS-2$
    			XPropertySetHelper.setPropertyValue(xProps, "BibTeXDir", dlg.getTextFieldText("BibTeXDir")); //$NON-NLS-1$ //$NON-NLS-2$
+   			XPropertySetHelper.setPropertyValue(xProps, "BibTeXEncoding", dlg.getListBoxSelectedItem("BibTeXEncoding")); //$NON-NLS-1$ //$NON-NLS-2$
     		XPropertySetHelper.setPropertyValue(xProps, "UseNatbib", dlg.getCheckBoxStateAsBoolean("UseNatbib")); //$NON-NLS-1$ //$NON-NLS-2$
    			XPropertySetHelper.setPropertyValue(xProps, "NatbibOptions", dlg.getTextFieldText("NatbibOptions")); //$NON-NLS-1$ //$NON-NLS-2$
    			
@@ -263,6 +266,8 @@ public final class BibliographyDialog
 		dlg.setControlEnabled("BibTeXDirLabel", bEnableSettings && bEnableDir); //$NON-NLS-1$
 		dlg.setControlEnabled("BibTeXDir",  bEnableSettings && bEnableDir); //$NON-NLS-1$
 		dlg.setControlEnabled("BibTeXDirButton",  bEnableSettings && bEnableDir); //$NON-NLS-1$
+		dlg.setControlEnabled("BibTeXEncodingLabel",  bEnableSettings); //$NON-NLS-1$
+		dlg.setControlEnabled("BibTeXEncoding",  bEnableSettings); //$NON-NLS-1$
 		dlg.setControlEnabled("ConvertZoteroCitations", bEnableSettings); //$NON-NLS-1$
 		dlg.setControlEnabled("ConvertJabRefCitations", bEnableSettings); //$NON-NLS-1$
 		dlg.setControlEnabled("IncludeOriginalCitations", bEnableSettings && bEnableOriginalCitations); //$NON-NLS-1$
@@ -348,6 +353,3 @@ public final class BibliographyDialog
 	}
 	
 }
-
-
-

source/java/org/openoffice/da/comp/writer2latex/LaTeXUNOPublisher.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  *  
- *  Version 1.6 (2015-05-29)
+ *  Version 1.6 (2015-07-23)
  *  
  */
 package org.openoffice.da.comp.writer2latex;
@@ -34,6 +34,7 @@ import org.openoffice.da.comp.w2lcommon.helper.PropertyHelper;
 import org.openoffice.da.comp.w2lcommon.helper.RegistryHelper;
 import org.openoffice.da.comp.w2lcommon.helper.XPropertySetHelper;
 
+import writer2latex.latex.i18n.ClassicI18n;
 import writer2latex.util.CSVList;
 import writer2latex.util.Misc;
 
@@ -107,6 +108,8 @@ public class LaTeXUNOPublisher extends UNOPublisher {
         				XPropertySetHelper.getPropertyValueAsString(xProps, "BibTeXDir")); //$NON-NLS-1$
         		if (XPropertySetHelper.getPropertyValueAsBoolean(xProps, "UseExternalBibTeXFiles")) { //$NON-NLS-1$
         			filterHelper.put("external_bibtex_files", sBibTeXFiles); //$NON-NLS-1$
+        			filterHelper.put("bibtex_encoding", ClassicI18n.writeInputenc(  //$NON-NLS-1$
+        					XPropertySetHelper.getPropertyValueAsShort(xProps, "BibTeXEncoding")));  //$NON-NLS-1$
             		if (XPropertySetHelper.getPropertyValueAsBoolean(xProps, "ConvertZoteroCitations")) { //$NON-NLS-1$
             			filterHelper.put("zotero_bibtex_files", sBibTeXFiles); //$NON-NLS-1$
             		}

source/java/writer2latex/api/ConverterFactory.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-07-01)
+ *  Version 1.6 (2015-07-27)
  *
  */
  
@@ -32,8 +32,8 @@ package writer2latex.api;
 public class ConverterFactory {
 
     // Version information
-    private static final String VERSION = "1.5.3";
+    private static final String VERSION = "1.6";
-    private static final String DATE = "2015-07-01";
+    private static final String DATE = "2015-07-27";
 	
     /** Return the Writer2LaTeX version in the form
      *  (major version).(minor version).(patch level)<br/>

source/java/writer2latex/latex/BibConverter.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-07-01)
+ *  Version 1.6 (2015-07-27)
  *
  */
 
@@ -33,6 +33,7 @@ import org.w3c.dom.Element;
 import writer2latex.base.BibliographyGenerator;
 import writer2latex.bibtex.BibTeXDocument;
 
+import writer2latex.latex.i18n.ClassicI18n;
 import writer2latex.latex.util.BeforeAfter;
 import writer2latex.latex.util.Context;
 
@@ -60,6 +61,8 @@ class BibConverter extends ConverterHelper {
 
     private BibTeXDocument bibDoc = null;
     private boolean bUseBibTeX;
+    private String sBibTeXEncoding = null;
+    private String sDocumentEncoding = null;
     
     /** Construct a new BibConverter.
      * 
@@ -74,6 +77,16 @@ class BibConverter extends ConverterHelper {
         	bibDoc = new BibTeXDocument(palette.getOutFileName(),false,ofr);
         }
         
+        // We need to use a different encoding for the BibTeX files
+        if (config.externalBibtexFiles().length()>0) {
+        	int nBibTeXEncoding = config.getBibtexEncoding();
+        	int nDocumentEncoding = config.getInputencoding();
+        	if (config.getBackend()!=LaTeXConfig.XETEX && nBibTeXEncoding>-1 && nBibTeXEncoding!=nDocumentEncoding) {
+        		sBibTeXEncoding = ClassicI18n.writeInputenc(nBibTeXEncoding);
+            	sDocumentEncoding = ClassicI18n.writeInputenc(nDocumentEncoding);
+        	}
+        }
+        
         // We need to export it 
         bUseBibTeX = config.useBibtex();
     }
@@ -199,10 +212,17 @@ class BibConverter extends ConverterHelper {
            .append("}").nl();
 
         // Use BibTeX file from configuration, or exported BibTeX file
+        // TODO: For XeTeX, probably use \XeTeXdefaultencoding?
         if (config.externalBibtexFiles().length()>0) {
+        	if (sBibTeXEncoding!=null) {
+        		ldp.append("\\inputencoding{").append(sBibTeXEncoding).append("}").nl();
+        	}
     		ldp.append("\\bibliography{")
                .append(config.externalBibtexFiles())
                .append("}").nl();
+        	if (sBibTeXEncoding!=null) {
+        		ldp.append("\\inputencoding{").append(sDocumentEncoding).append("}").nl();
+        	}
         }
         else {
             ldp.append("\\bibliography{")

source/java/writer2latex/latex/LaTeXConfig.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-06-23)
+ *  Version 1.6 (2015-07-23)
  *
  */
 
@@ -49,7 +49,7 @@ public class LaTeXConfig extends writer2latex.base.ConfigBase {
 	/////////////////////////////////////////////////////////////////////////
 	// I. Define items needed by ConfigBase
 	
-    protected int getOptionCount() { return 72; }
+    protected int getOptionCount() { return 73; }
     protected String getDefaultConfigPath() { return "/writer2latex/latex/config/"; } 
     
 	/////////////////////////////////////////////////////////////////////////
@@ -144,45 +144,46 @@ public class LaTeXConfig extends writer2latex.base.ConfigBase {
     private static final int USE_BIBTEX = 30;
     private static final int BIBTEX_STYLE = 31;
     private static final int EXTERNAL_BIBTEX_FILES = 32;
-    private static final int ZOTERO_BIBTEX_FILES = 33;
+    private static final int BIBTEX_ENCODING = 33;
-    private static final int JABREF_BIBTEX_FILES = 34;
+    private static final int ZOTERO_BIBTEX_FILES = 34;
-    private static final int INCLUDE_ORIGINAL_CITATIONS = 35;
+    private static final int JABREF_BIBTEX_FILES = 35;
-    private static final int USE_NATBIB = 36;
+    private static final int INCLUDE_ORIGINAL_CITATIONS = 36;
-    private static final int NATBIB_OPTIONS = 37;
+    private static final int USE_NATBIB = 37;
-    private static final int FONT = 38;
+    private static final int NATBIB_OPTIONS = 38;
-    private static final int FORMATTING = 39;
+    private static final int FONT = 39;
-    private static final int PAGE_FORMATTING = 40;
+    private static final int FORMATTING = 40;
-    private static final int OTHER_STYLES = 41;
+    private static final int PAGE_FORMATTING = 41;
-    private static final int IMAGE_CONTENT = 42;
+    private static final int OTHER_STYLES = 42;
-	private static final int TABLE_CONTENT = 43;
+    private static final int IMAGE_CONTENT = 43;
-	private static final int TABLE_FIRST_HEAD_STYLE = 44;
+	private static final int TABLE_CONTENT = 44;
-	private static final int TABLE_HEAD_STYLE = 45;
+	private static final int TABLE_FIRST_HEAD_STYLE = 45;
-	private static final int TABLE_FOOT_STYLE = 46;
+	private static final int TABLE_HEAD_STYLE = 46;
-	private static final int TABLE_LAST_FOOT_STYLE = 47;
+	private static final int TABLE_FOOT_STYLE = 47;
-    private static final int IGNORE_HARD_PAGE_BREAKS = 48;
+	private static final int TABLE_LAST_FOOT_STYLE = 48;
-    private static final int IGNORE_HARD_LINE_BREAKS = 49;
+    private static final int IGNORE_HARD_PAGE_BREAKS = 49;
-    private static final int IGNORE_EMPTY_PARAGRAPHS =50;
+    private static final int IGNORE_HARD_LINE_BREAKS = 50;
-    private static final int IGNORE_DOUBLE_SPACES = 51;
+    private static final int IGNORE_EMPTY_PARAGRAPHS =51;
-    private static final int DISPLAY_HIDDEN_TEXT = 52;
+    private static final int IGNORE_DOUBLE_SPACES = 52;
-    private static final int ALIGN_FRAMES = 53;
+    private static final int DISPLAY_HIDDEN_TEXT = 53;
-    private static final int FLOAT_FIGURES = 54; 
+    private static final int ALIGN_FRAMES = 54;
-    private static final int FLOAT_TABLES = 55; 
+    private static final int FLOAT_FIGURES = 55; 
-    private static final int FLOAT_OPTIONS = 56;
+    private static final int FLOAT_TABLES = 56; 
-    private static final int FIGURE_SEQUENCE_NAME = 57; 
+    private static final int FLOAT_OPTIONS = 57;
-    private static final int TABLE_SEQUENCE_NAME = 58; 
+    private static final int FIGURE_SEQUENCE_NAME = 58; 
-    private static final int IMAGE_OPTIONS = 59;
+    private static final int TABLE_SEQUENCE_NAME = 59; 
-    private static final int REMOVE_GRAPHICS_EXTENSION = 60;
+    private static final int IMAGE_OPTIONS = 60;
-    private static final int ORIGINAL_IMAGE_SIZE = 61;
+    private static final int REMOVE_GRAPHICS_EXTENSION = 61;
-    private static final int SIMPLE_TABLE_LIMIT = 62;
+    private static final int ORIGINAL_IMAGE_SIZE = 62;
-    private static final int NOTES = 63;
+    private static final int SIMPLE_TABLE_LIMIT = 63;
-    private static final int METADATA = 64;
+    private static final int NOTES = 64;
-    private static final int TABSTOP = 65;
+    private static final int METADATA = 65;
-    private static final int WRAP_LINES_AFTER = 66;
+    private static final int TABSTOP = 66;
-    private static final int SPLIT_LINKED_SECTIONS = 67;
+    private static final int WRAP_LINES_AFTER = 67;
-    private static final int SPLIT_TOPLEVEL_SECTIONS = 68;
+    private static final int SPLIT_LINKED_SECTIONS = 68;
-    private static final int SAVE_IMAGES_IN_SUBDIR = 69;
+    private static final int SPLIT_TOPLEVEL_SECTIONS = 69;
-    private static final int OLD_MATH_COLORS = 70;
+    private static final int SAVE_IMAGES_IN_SUBDIR = 70;
-    private static final int DEBUG = 71;
+    private static final int OLD_MATH_COLORS = 71;
+    private static final int DEBUG = 72;
     
 	/////////////////////////////////////////////////////////////////////////
     // IV. Our options data
@@ -254,6 +255,13 @@ public class LaTeXConfig extends writer2latex.base.ConfigBase {
         options[USE_BIBTEX] = new BooleanOption("use_bibtex","false");
         options[BIBTEX_STYLE] = new Option("bibtex_style","plain");
         options[EXTERNAL_BIBTEX_FILES] = new Option("external_bibtex_files","");
+        options[BIBTEX_ENCODING] =  new IntegerOption("bibtex_encoding","document") {
+            public void setString(String sValue) {
+                super.setString(sValue);
+                if ("document".equals(sValue)) { nValue = -1; }
+                else { nValue = ClassicI18n.readInputenc(sValue); }
+            }
+        };
         options[ZOTERO_BIBTEX_FILES] = new Option("zotero_bibtex_files","");
         options[JABREF_BIBTEX_FILES] = new Option("jabref_bibtex_files","");
         options[INCLUDE_ORIGINAL_CITATIONS] = new BooleanOption("include_original_citations","false");
@@ -683,6 +691,7 @@ public class LaTeXConfig extends writer2latex.base.ConfigBase {
     public boolean useBibtex() { return ((BooleanOption) options[USE_BIBTEX]).getValue(); }
     public String bibtexStyle() { return options[BIBTEX_STYLE].getString(); }
     public String externalBibtexFiles() { return options[EXTERNAL_BIBTEX_FILES].getString(); }
+    public int getBibtexEncoding() { return ((IntegerOption) options[BIBTEX_ENCODING]).getValue(); }
     public String zoteroBibtexFiles() { return options[ZOTERO_BIBTEX_FILES].getString(); }
     public String jabrefBibtexFiles() { return options[JABREF_BIBTEX_FILES].getString(); }
     public boolean includeOriginalCitations() { return ((BooleanOption) options[INCLUDE_ORIGINAL_CITATIONS]).getValue(); }

source/java/writer2latex/xhtml/TOCConverter.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-06-16)
+ *  Version 1.6 (2015-07-23)
  *
  */
 package writer2latex.xhtml;
@@ -61,6 +61,8 @@ final class IndexData {
  */
 class TOCConverter extends IndexConverterHelper {
 	
+	private static final String TOC_LINK_PREFIX = "toc";
+	
     private List<IndexData> indexes = new ArrayList<IndexData>(); // All tables of content
 	private List<TocEntry> tocEntries = new ArrayList<TocEntry>(); // All potential(!) toc items
 	private int nTocFileIndex = -1; // file index for main toc
@@ -94,7 +96,7 @@ class TOCConverter extends IndexConverterHelper {
 	 */
 	void handleHeading(Element onode, Element heading, String sLabel) {
 		int nLevel = getTextCv().getOutlineLevel(onode);
-		String sTarget = "toc"+(++nTocIndex);
+		String sTarget = TOC_LINK_PREFIX+(++nTocIndex);
 		converter.addTarget(heading,sTarget);
 		
 		this.currentChapter = onode;
@@ -126,7 +128,7 @@ class TOCConverter extends IndexConverterHelper {
 			if (nLevel>config.getXhtmlSplitLevel()) {
 				Element div = converter.createElement("div");        			
 				hnode.appendChild(div);
-				sTarget = "toc"+(++nTocIndex);
+				sTarget = TOC_LINK_PREFIX+(++nTocIndex);
 				converter.addTarget(div,sTarget);
 			}
 			converter.addContentEntry(sLabel+converter.getPlainInlineText(onode), nLevel, sTarget);
@@ -136,7 +138,7 @@ class TOCConverter extends IndexConverterHelper {
 	void handleParagraph(Element onode, Element par, String sCurrentListLabel) {
         String sStyleName = Misc.getAttribute(onode,XMLString.TEXT_STYLE_NAME);
 		if (ofr.isIndexSourceStyle(getParSc().getRealParStyleName(sStyleName))) {
-	        converter.addTarget(par,"toc"+(++nTocIndex));
+	        converter.addTarget(par,TOC_LINK_PREFIX+(++nTocIndex));
 	        TocEntry entry = new TocEntry();
 	        entry.onode = (Element) onode;
 	        entry.sLabel = sCurrentListLabel;  
@@ -151,7 +153,7 @@ class TOCConverter extends IndexConverterHelper {
      * @param hnode the link target will be added to this inline HTML node
      */
     void handleTocMark(Node onode, Node hnode) {
-        hnode.appendChild(converter.createTarget("toc"+(++nTocIndex)));
+        hnode.appendChild(converter.createTarget(TOC_LINK_PREFIX+(++nTocIndex)));
         TocEntry entry = new TocEntry();
         entry.onode = (Element) onode;
         entry.nFileIndex = converter.getOutFileIndex();
@@ -243,7 +245,7 @@ class TOCConverter extends IndexConverterHelper {
                         span.setAttribute("class","SectionNumber");
                         span.appendChild(converter.createTextNode(entry.sLabel));
                     }
-                    Element a = converter.createLink("toc"+i);
+                    Element a = converter.createLink(TOC_LINK_PREFIX+i);
                     p.appendChild(a);
                     getTextCv().traverseInlineText(entry.onode,a);
                 }
@@ -255,7 +257,7 @@ class TOCConverter extends IndexConverterHelper {
                         if (entry.sLabel!=null) {
                             p.appendChild(converter.createTextNode(entry.sLabel));
                         }
-                        Element a = converter.createLink("toc"+i);
+                        Element a = converter.createLink(TOC_LINK_PREFIX+i);
                         p.appendChild(a);
                         getTextCv().traverseInlineText(entry.onode,a);
                     }
@@ -269,7 +271,7 @@ class TOCConverter extends IndexConverterHelper {
                     if (entry.sLabel!=null) {
                         p.appendChild(converter.createTextNode(entry.sLabel));
                     }
-                    Element a = converter.createLink("toc"+i);
+                    Element a = converter.createLink(TOC_LINK_PREFIX+i);
                     p.appendChild(a);
                     getTextCv().traverseInlineText(entry.onode,a);
                 }
@@ -278,7 +280,7 @@ class TOCConverter extends IndexConverterHelper {
                 int nLevel = Misc.getPosInteger(entry.onode.getAttribute(XMLString.TEXT_OUTLINE_LEVEL),1);
                 if (tocReader.useIndexMarks() && nLevel<=tocReader.getOutlineLevel()) {
                     Element p = getTextCv().createParagraph(li,sEntryStyleName[nLevel]);
-                    Element a = converter.createLink("toc"+i);
+                    Element a = converter.createLink(TOC_LINK_PREFIX+i);
                     p.appendChild(a);
                     a.appendChild(converter.createTextNode(IndexMark.getIndexValue(entry.onode)));
                 }
@@ -287,7 +289,7 @@ class TOCConverter extends IndexConverterHelper {
                 int nLevel = Misc.getPosInteger(entry.onode.getAttribute(XMLString.TEXT_OUTLINE_LEVEL),1);
                 if (tocReader.useIndexMarks() && nLevel<=tocReader.getOutlineLevel()) {
                     Element p = getTextCv().createParagraph(li,sEntryStyleName[nLevel]);
-                    Element a = converter.createLink("toc"+i);
+                    Element a = converter.createLink(TOC_LINK_PREFIX+i);
                     p.appendChild(a);
                     a.appendChild(converter.createTextNode(IndexMark.getIndexValue(entry.onode)));
                 }
@@ -353,7 +355,7 @@ class TOCConverter extends IndexConverterHelper {
                     String sNodeName = entry.onode.getTagName();
                     if (XMLString.TEXT_H.equals(sNodeName)) {
 
-                        // Determine wether or not to include this heading
+                        // Determine whether or not to include this heading
                         // Note that this condition misses the case where
                         // a heading of level n is followed by a heading of
                         // level n+2. This is considered a bug in the document!

source/java/writer2latex/xhtml/TextConverter.java

@@ -20,7 +20,7 @@
  *
  *  All Rights Reserved.
  * 
- *  Version 1.6 (2015-06-16)
+ *  Version 1.6 (2015-07-23)
  *
  */
 
@@ -50,7 +50,7 @@ public class TextConverter extends ConverterHelper {
     // TODO: Accessor methods for sections
 	// Some (Sony?) EPUB readers have a limit on the file size of individual files
 	// In any case very large files could be a performance problem, hence we do automatic splitting
-	// after this number of characters. TODO: Make configurable.
+	// after this number of characters.
 	private int nSplitAfter = 150000;
 	private int nPageBreakSplit = XhtmlConfig.NONE; // Should we split at page breaks?
 	// TODO: Collect soft page breaks between table rows

source/oxt/w2lconfig/description.xml

@@ -4,7 +4,7 @@
              xmlns:xlink="http://www.w3.org/1999/xlink">
 
   <identifier value="org.openoffice.da.w2lconfig.oxt" />
-  <version value="1.5.3" />
+  <version value="1.6" />
   <dependencies>
     <OpenOffice.org-minimal-version value="3.0" d:name="OpenOffice.org 3.0"/>
   </dependencies>

source/oxt/writer2latex/ToolbarOptions.xcs

@@ -33,6 +33,7 @@
             <prop oor:name="IncludeOriginalCitations" oor:type="xs:boolean" />
             <prop oor:name="BibTeXLocation" oor:type="xs:short" />
             <prop oor:name="BibTeXDir" oor:type="xs:string" />
+            <prop oor:name="BibTeXEncoding" oor:type="xs:short" />
             <prop oor:name="UseNatbib" oor:type="xs:boolean" />
             <prop oor:name="NatbibOptions" oor:type="xs:string" />
         </group>

source/oxt/writer2latex/ToolbarOptions.xcu

@@ -46,7 +46,7 @@
                 <value>false</value>
             </prop>
             <prop oor:name="Executable">
-                <value>bibtex</value>
+                <value>bibtex8</value>
             </prop>
             <prop oor:name="Options">
                 <value>%s</value>
@@ -138,6 +138,9 @@
         <prop oor:name="BibTeXDir" oor:type="xs:string">
             <value></value>
         </prop>
+        <prop oor:name="BibTeXEncoding" oor:type="xs:short">
+            <value>0</value><!-- US ASCII -->
+        </prop>
         <prop oor:name="UseNatbib" oor:type="xs:boolean">
             <value>false</value>
         </prop>

source/oxt/writer2latex/W2LDialogs2/Bibliography.xdl

@@ -3,13 +3,11 @@
 <dlg:window xmlns:dlg="http://openoffice.org/2000/dialog" xmlns:script="http://openoffice.org/2000/script" dlg:id="Bibliography" dlg:left="139" dlg:top="84" dlg:width="260" dlg:height="185" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyDialog" dlg:closeable="true" dlg:moveable="true" dlg:title="writer2latex Bibliography Configuration" dlg:withtitlebar="false">
  <dlg:bulletinboard>
   <dlg:text dlg:id="BibliographyLabel" dlg:tab-index="0" dlg:left="6" dlg:top="4" dlg:width="244" dlg:height="12" dlg:value="Citations and BibTeX files"/>
-
   <dlg:checkbox dlg:id="UseExternalBibTeXFiles" dlg:tab-index="1" dlg:left="12" dlg:top="18" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyUseExternalBibTeXFiles" dlg:value="Use BibTeX as bibliography database" dlg:checked="false">
    <script:event script:event-name="on-itemstatechange" script:macro-name="vnd.sun.star.UNO:UseExternalBibTeXFilesChange" script:language="UNO"/>
   </dlg:checkbox>
-
   <dlg:text dlg:id="BibTeXLocationLabel" dlg:tab-index="2" dlg:left="12" dlg:top="32" dlg:width="55" dlg:height="12" dlg:value="BibTeX location"/>
-  <dlg:menulist dlg:id="BibTeXLocation" dlg:tab-index="3" dlg:left="70" dlg:top="30" dlg:width="180" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXLocation" dlg:spin="true" dlg:linecount="3">
+  <dlg:menulist dlg:id="BibTeXLocation" dlg:tab-index="3" dlg:left="70" dlg:top="30" dlg:width="130" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXLocation" dlg:spin="true" dlg:linecount="3">
    <dlg:menupopup>
     <dlg:menuitem dlg:value="In central folder"/>
     <dlg:menuitem dlg:value="In subfolder of document folder"/>
@@ -17,31 +15,36 @@
    </dlg:menupopup>
    <script:event script:event-name="on-itemstatechange" script:macro-name="vnd.sun.star.UNO:BibTeXLocationChange" script:language="UNO"/>
   </dlg:menulist>
-
   <dlg:text dlg:id="BibTeXDirLabel" dlg:tab-index="4" dlg:left="12" dlg:top="46" dlg:width="55" dlg:height="12" dlg:value="BibTeX folder"/>
   <dlg:textfield dlg:id="BibTeXDir" dlg:tab-index="5" dlg:left="70" dlg:top="44" dlg:width="130" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXDir"/>
   <dlg:button dlg:id="BibTeXDirButton" dlg:tab-index="6" dlg:left="210" dlg:top="44" dlg:width="40" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXDirButton" dlg:value="Browse...">
    <script:event script:event-name="on-mouseup" script:macro-name="vnd.sun.star.UNO:BibTeXDirClick" script:language="UNO"/>
   </dlg:button>
-
+  <dlg:checkbox dlg:id="ConvertZoteroCitations" dlg:tab-index="8" dlg:left="12" dlg:top="74" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyConvertZoteroCitations" dlg:value="Convert Zotero citations" dlg:checked="false">
-  <dlg:checkbox dlg:id="ConvertZoteroCitations" dlg:tab-index="7" dlg:left="12" dlg:top="60" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyConvertZoteroCitations" dlg:value="Convert Zotero citations" dlg:checked="false">
    <script:event script:event-name="on-itemstatechange" script:macro-name="vnd.sun.star.UNO:ConvertZoteroCitationsChange" script:language="UNO"/>
   </dlg:checkbox>
-  
+  <dlg:checkbox dlg:id="ConvertJabRefCitations" dlg:tab-index="9" dlg:left="12" dlg:top="88" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyConvertJabRefCitations" dlg:value="Convert JabRef citations" dlg:checked="false">
-  <dlg:checkbox dlg:id="ConvertJabRefCitations" dlg:tab-index="8" dlg:left="12" dlg:top="74" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyConvertJabRefCitations" dlg:value="Convert JabRef citations" dlg:checked="false">
    <script:event script:event-name="on-itemstatechange" script:macro-name="vnd.sun.star.UNO:ConvertJabRefCitationsChange" script:language="UNO"/>
   </dlg:checkbox>
-
+  <dlg:checkbox dlg:id="IncludeOriginalCitations" dlg:tab-index="10" dlg:left="12" dlg:top="102" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyIncludeOriginalCitations" dlg:value="Include original citations as comments" dlg:checked="false"/>
-  <dlg:checkbox dlg:id="IncludeOriginalCitations" dlg:tab-index="9" dlg:left="12" dlg:top="88" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyIncludeOriginalCitations" dlg:value="Include original citations as comments" dlg:checked="false"/>
+  <dlg:text dlg:id="NatbibLabel" dlg:tab-index="11" dlg:left="6" dlg:top="116" dlg:width="244" dlg:height="11" dlg:value="Natbib"/>
-
+  <dlg:checkbox dlg:id="UseNatbib" dlg:tab-index="12" dlg:left="12" dlg:top="130" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyUseNatbib" dlg:value="Use natbib.sty" dlg:checked="false">
-  <dlg:text dlg:id="NatbibLabel" dlg:tab-index="10" dlg:left="6" dlg:top="102" dlg:width="244" dlg:height="11" dlg:value="Natbib"/>
-
-  <dlg:checkbox dlg:id="UseNatbib" dlg:tab-index="11" dlg:left="12" dlg:top="116" dlg:width="240" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyUseNatbib" dlg:value="Use natbib.sty" dlg:checked="false">
    <script:event script:event-name="on-itemstatechange" script:macro-name="vnd.sun.star.UNO:UseNatbibChange" script:language="UNO"/>
   </dlg:checkbox>
-
+  <dlg:text dlg:id="NatbibOptionsLabel" dlg:tab-index="13" dlg:left="22" dlg:top="144" dlg:width="45" dlg:height="12" dlg:value="Options"/>
-  <dlg:text dlg:id="NatbibOptionsLabel" dlg:tab-index="12" dlg:left="22" dlg:top="130" dlg:width="45" dlg:height="12" dlg:value="Options"/>
+  <dlg:textfield dlg:id="NatbibOptions" dlg:tab-index="14" dlg:left="70" dlg:top="142" dlg:width="180" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyNatbibOptions"/>
-
+  <dlg:text dlg:id="BibTeXEncodingLabel" dlg:tab-index="15" dlg:left="12" dlg:top="60" dlg:width="55" dlg:height="12" dlg:value="File encoding"/>
-  <dlg:textfield dlg:id="NatbibOptions" dlg:tab-index="13" dlg:left="70" dlg:top="128" dlg:width="180" dlg:height="12" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyNatbibOptions"/>
+  <dlg:menulist dlg:id="BibTeXEncoding" dlg:tab-index="7" dlg:left="70" dlg:top="58" dlg:width="130" dlg:height="12" dlg:spin="true" dlg:linecount="8" dlg:help-url="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXEncoding">
+   <dlg:menupopup>
+    <dlg:menuitem dlg:value="US ASCII"/>
+    <dlg:menuitem dlg:value="ISO-8859-1"/>
+    <dlg:menuitem dlg:value="ISO-8859-2"/>
+    <dlg:menuitem dlg:value="ISO-8859-7"/>
+    <dlg:menuitem dlg:value="Cp1250"/>
+    <dlg:menuitem dlg:value="Cp1251"/>
+    <dlg:menuitem dlg:value="koi8-r"/>
+    <dlg:menuitem dlg:value="UTF8"/>
+   </dlg:menupopup>
+  </dlg:menulist>
  </dlg:bulletinboard>
 </dlg:window>

source/oxt/writer2latex/description.xml

@@ -5,7 +5,7 @@
 
   <identifier value="org.openoffice.da.writer2latex.oxt"/>
 
-  <version value="1.5.3" />
+  <version value="1.6" />
 
   <dependencies>
     <OpenOffice.org-minimal-version value="3.0" d:name="OpenOffice.org 3.0"/>

source/oxt/writer2latex/help/en/org.openoffice.da.writer2latex.oxt/bibliography.xhp

@@ -73,6 +73,11 @@
 		<bookmark xml-lang="en-US" branch="hid/org.openoffice.da.writer2latex.oxt:BibliographyBibTeXDirButton" id="bm_bibtexdirbutton"/>
 		<paragraph role="paragraph" xml-lang="en-US"><ahelp hid="org.openoffice.da.writer2latex.oxt:BibliographyBibTeXDirButton" visibility="hidden">Click to select the path to the folder containing the BibTeX files</ahelp></paragraph>
 
+		<bookmark xml-lang="en-US" branch="hid/org.openoffice.da.writer2latex.oxt:BibliographyBibTeXEncoding" id="bm_bibtexencoding"/>
+		<paragraph role="heading" level="3" xml-lang="en-US">File encoding</paragraph>
+		<paragraph role="paragraph" xml-lang="en-US">Select the encoding of the BibTeX files (this will depend on your
+		BibTeX editor). If you use XeTeX as a backend, you should always use UTF-8.</paragraph>
+
 		<bookmark xml-lang="en-US" branch="hid/org.openoffice.da.writer2latex.oxt:BibliographyConvertZoteroCitations" id="bm_convertzoterocitations"/>
 		<paragraph role="heading" level="2" xml-lang="en-US">Convert Zotero citations</paragraph>
 		<paragraph role="paragraph" xml-lang="en-US"><link href="http://www.zotero.org" name="Link to the Zotero Website">Zotero</link>

source/oxt/writer2xhtml/description.xml

@@ -5,7 +5,7 @@
 			 
   <identifier value="org.openoffice.da.writer2xhtml.oxt" />
   
-  <version value="1.5.3" />
+  <version value="1.6" />
   
   <dependencies>
     <OpenOffice.org-minimal-version value="3.0" d:name="OpenOffice.org 3.0"/>

source/oxt/xhtml-config-sample/description.xml

@@ -2,5 +2,5 @@
 <description xmlns="http://openoffice.org/extensions/description/2006"
              xmlns:d="http://openoffice.org/extensions/description/2006">
   <identifier value="org.openoffice.da.writer2latex.xhtml-config-sample.oxt" />
-  <version value="1.5.3" />
+  <version value="1.6" />
 </description> 

source/readme-source.txt

@@ -1,5 +1,5 @@
-Writer2LaTeX source version 1.5.3 alpha
+Writer2LaTeX source version 1.6 beta
-=======================================
+====================================
 
 Writer2LaTeX is (c) 2002-2015 by Henrik Just.
 The source is available under the terms and conditions of the
@@ -49,8 +49,8 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRA
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 
 
 
-Third-party software:
+Third-party software: BibTeX API
----------------------------
+--------------------------------
 
 Villu Ruusmanns Java BibTeX API is included in binary form as jbibtex-1.0.14.jar.
 The source code is available from https://code.google.com/p/java-bibtex/
@@ -124,7 +124,7 @@ In addition to oxt, the build file supports the following targets:
     clean
 
 
-Henrik Just, May 2015
+Henrik Just, July 2015
 
 
 Thanks to Michael Niedermair for writing the original ant build file