/************************************************************************
 *
 *  Mouth.java
 *
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License version 2.1, as published by the Free Software Foundation.
 *
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 *
 *  You should have received a copy of the GNU Lesser General Public
 *  License along with this library; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *  MA  02111-1307  USA
 *
 *  Copyright: 2002-2010 by Henrik Just
 *
 *  All Rights Reserved.
 * 
 *  Version 1.2 (2010-10-25)
 *
 */

package org.openoffice.da.comp.w2lcommon.tex.tokenizer;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.Reader;

enum State {
	N, // new line
	M, // middle of line
	S; // ignoring spaces
}

/** <p>The Mouth is the main class of this package. It is a tokenizer to TeX files: According to "The TeXBook", the
 *  "eyes" and "mouth" of TeX are responsible for turning the input to TeX into a sequence of tokens.
 *  We are not going to reimplement TeX, but rather providing a service for parsing high-level languages based on
 *  TeX (eg. LaTeX, ConTeXt). For this reason the tokenizer deviates slightly from TeX: We're not reading a stream
 *  of bytes but rather a stream of characters (which makes no difference for ASCII files).</p>
 *  
 *  <p>In tribute to Donald E. Knuths digestive metaphors, we divide the process in four levels</p>
 *  <ul>
 *  <li>The parser should provide a <em>pair of glasses</em> to translate the stream of bytes into a stream of characters</li>
 *  <li>The <em>eyes</em> sees the stream of characters as a sequence of lines</li>
 *  <li>The <em>mouth</em> chews a bit on the characters to turn them into tokens</li>
 *  <li>The <em>tongue</em> reports the "taste" of the token to the parser</li>
 *  </ul>
 */
public class Mouth {
	private Reader reader; // The input
	private CatcodeTable catcodes; // The current catcode table
	private char cEndlinechar; // The current value of \endlinechar
	private Token token; // The token object
	private State state; // The current state of the tokenizer
	private Eyes eyes; // sic!

	/** Construct a new <code>Mouth</code> based on a character stream
	 * 
	 * @param reader the character stream to tokenize
	 * @throws IOException if we fail to read the character stream
	 */
	public Mouth(Reader reader) throws IOException {
		this.reader = reader;
		catcodes = new CatcodeTable();
		cEndlinechar = '\r';
		token = new Token();
		state = State.N;
		eyes = new Eyes();
	}
	
	private class Eyes {
		private BufferedReader br; // The input
		private String sLine; // The current line
		private int nLen; // The length of the current line
		private int nIndex; // The current index in the current line
		
		Eyes() throws IOException {
			br = new BufferedReader(reader);
			nextLine();
		}
		
		/** Start looking at the next line of input
		 * 
		 * @throws IOException if we fail to read the underlying stream
		 */
		void nextLine() throws IOException {
			sLine = br.readLine();
			if (sLine!=null) {
				nLen = sLine.length();
				nIndex = 0;
				// Delete trailing spaces
				while (nLen>0 && sLine.charAt(nLen-1)==' ') { nLen--; }
			}
			else { // end of stream
				nLen = 0;
				nIndex = 1;
			}
		}
		
		/** Test whether the eyes are looking at a character
		 * 
		 * @return true if the current line still has characters to look at
		 */
		boolean lookingAtChar() {
			return nIndex<=nLen;
		}
		
		/** Test whether the eyes a looking at a line
		 * 
		 * @return true if a current line is available
		 */ 
		boolean lookingAtLine() {
			return sLine!=null;
		}
		
		/** Get the character that the eyes currently sees
		 * 
		 * @return the character or U+FFFF if the eyes are not looking at a character
		 */
		char peekChar() {
			return getChar(false);
		}
		
		/** Get the character that the eyes currently sees and start looking at the next character
		 * 
		 * @return the character or U+FFFF if the eyes are not looking at a character
		 */
		char getChar() {
			return getChar(true);
		}
		
		private char getChar(boolean bMove) {
			if (nIndex<nLen) {
				char c = sLine.charAt(nIndex);
				if (catcodes.get(c)==Catcode.SUPERSCRIPT && nIndex+2<nLen && catcodes.get(sLine.charAt(nIndex+1))==Catcode.SUPERSCRIPT) {
					// Found ^^ and at least one more character
					char c1 = sLine.charAt(nIndex+2);
					if (nIndex+3<nLen && isHex(c1)) {
						char c2 = sLine.charAt(nIndex+3);
						if (isHex(c2)) {
							// Found ^^ and a lower case hexadecimal number
							if (bMove) { nIndex+=4; }
							char[] digits = {c1, c2};
							return (char) Integer.parseInt(new String(digits), 16);
						}
					}
					else if (c1<128) {
						// Found ^^ and an ASCII character
						if (bMove) { nIndex+=3; }
						if (c1<64) { return (char)(c1+64); }
						else { return (char)(c1-64); }
					}
				}
				// Found an ordinary character!
				if (bMove) { nIndex++; }
				return c;
			}
			else if (nIndex==nLen) {
				// Add \endlinechar at the end of the line
				if (bMove) { nIndex++; }
				return cEndlinechar;
			}
			else {
				// No more characters on the current line
				return '\uFFFF';
			}
		}
		
		private boolean isHex(char c) {
			return ('0'<=c && c<='9') || ('a'<=c && c<='z');
		}		
	}
	
	/** Get the currently used catcode table
	 * 
	 * @return the table
	 */
	public CatcodeTable getCatcodes() {
		return catcodes;
	}
	
	/** Set the catcode table. The catcode table can be changed at any time during tokenization.
	 * 
	 * @param catcodes the table
	 */
	public void setCatcodes(CatcodeTable catcodes) {
		this.catcodes = catcodes;
	}
	
	/** Return the current value of the \endlinechar (the character added to the end of each input line)
	 * 
	 * @return the character
	 */
	public char getEndlinechar() {
		return cEndlinechar;
	}
	
	/** Set a new \endlinechar (the character added to the end of each input line). The character can be changed at
	 * any time during tokenization.
	 * 
	 * @param c the character
	 */
	public void setEndlinechar(char c) {
		cEndlinechar = c;
	}
	
	/** Return the object used to store the current token (the "tongue" of TeX).
	 *  The same object is reused for all tokens, so for convenience the parser can keep a reference to the object.
	 *  If on the other hand the parser needs to store a token list, it must explicitly clone all tokens.
	 * 
	 * @return the token
	 */
	public Token getTokenObject() {
		return token;
	}
	
	/** Get the next token
	 * 
	 * @return the token (for convenience; the same object is returned by {@link Mouth#getTokenObject}).
	 * @throws IOException if we fail to read the underlying stream
	 */
	public Token getToken() throws IOException {
		while (eyes.lookingAtLine()) {
			while (eyes.lookingAtChar()) {
				char c = eyes.getChar();
				switch (catcodes.get(c)) {
				case ESCAPE:
					token.setType(TokenType.COMMAND_SEQUENCE);
					token.clearChars();
					// TODO: The description in the TeXBook is not completely clear (to me anyway), 
					// (as long as \r and no other character has catcode END_OF_LINE this should be correct)
					if (catcodes.get(eyes.peekChar())==Catcode.LETTER) {
						state = State.S;
						while (eyes.lookingAtChar() && catcodes.get(eyes.peekChar())==Catcode.LETTER) {
							token.addChar(eyes.getChar());
						}
					}
					else if (catcodes.get(eyes.peekChar())==Catcode.SPACE) {
						state = State.S;
						token.setChar(eyes.getChar());
					}
					else if (catcodes.get(eyes.peekChar())!=Catcode.END_OF_LINE) {
						state = State.M;
						token.setChar(eyes.getChar());
					}
					else {
						// Empty control sequence
						state = State.M;
					}
					return token;
				case BEGIN_GROUP:
					state = State.M;
					token.set(c, TokenType.BEGIN_GROUP);
					return token;
				case END_GROUP:
					state = State.M;
					token.set(c, TokenType.END_GROUP);
					return token;
				case MATH_SHIFT:
					state = State.M;
					token.set(c, TokenType.MATH_SHIFT);
					return token;
				case ALIGNMENT_TAB:
					state = State.M;
					token.set(c, TokenType.ALIGNMENT_TAB);
					return token;
				case END_OF_LINE:
					// Skip rest of line
					while (eyes.lookingAtChar()) { eyes.getChar(); }
					switch (state) {
					case N:
						// This terminates an empty line -> insert a \par
						token.setType(TokenType.COMMAND_SEQUENCE);
						token.clearChars();
						token.addChar('p');
						token.addChar('a');
						token.addChar('r');
						return token;
					case M:
						// Replace with a space token
						token.set(' ', TokenType.SPACE);
						return token;
					case S:
						// ignore the character
					}
					break;
				case PARAMETER:
					state = State.M;
					token.set(c, TokenType.PARAMETER);
					return token;
				case SUPERSCRIPT:
					state = State.M;
					token.set(c, TokenType.SUPERSCRIPT);
					return token;
				case SUBSCRIPT:
					state = State.M;
					token.set(c, TokenType.SUBSCRIPT);
					return token;
				case IGNORED:
					// ignore this character
					break;
				case SPACE:
					if (state==State.M) {
						state=State.S;
						token.set(' ', TokenType.SPACE);
						return token;
					}
					// In state N and S the space character is ignored
					break;
				case LETTER:
					state = State.M;
					token.set(c, TokenType.LETTER);
					return token;
				case OTHER:
					state = State.M;
					token.set(c, TokenType.OTHER);
					return token;
				case ACTIVE:
					state = State.M;
					token.set(c, TokenType.ACTIVE);
					return token;
				case COMMENT:
					// Skip rest of line
					while (eyes.lookingAtChar()) { eyes.getChar(); }
					break;
				case INVALID:
					// ignore this character (should issue an error message, but we ignore that)
				}
			}
			eyes.nextLine();
			state = State.N;
		}
		// Nothing more to read
		token.setType(TokenType.ENDINPUT);
		token.clearChars();
		return token;
	}
	
}
