import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Stack;

/**
 * Extracts Tokens from Strings.
 * To use this class, first create a Tokenizer for a specific
 * input string by calling the constructor
 * <pre>    public Tokenizer(String line)</pre>
 * The returned Tokenizer implements <code>Iterator</code>, so
 * you can use the usual methods
 * <pre>    public boolean hasNext()
 *    public Object next()</pre>
 * In addition, the method
 * <pre>    public void putBack(int howMany)</pre>
 * allows the user to "back up" in the Token stream and iterate
 * over the replaced tokens again.
 * <p>
 * Only four Token types are returned; see the <code>Type</code>
 * class for a description of these.
 * <p>
 * First Java assignment, CIT594, Spring 2006.<br>
 * Created on Jan 9, 2006
 */
public class Tokenizer implements Iterator {
    private String line;
    private int position;
    private boolean hasMoreTokens;
    private Stack<Token> previousTokens;
    private enum State { START, NAME, NUMBER };
    
    /**
     * Creates a Tokenizer for the given line.
     * 
     * @param line The String to be broken into Tokens.
     */
    public Tokenizer(String line) {
        this.line = line + " "; // extra space simplifies the next() method
        position = 0;
        hasMoreTokens = true;
        previousTokens = new Stack<Token>();
    }

    /**
     * Returns <code>true</code> if this Tokenizer can return another Token.
     * 
     * @see java.util.Iterator#hasNext()
     */
    public boolean hasNext() {
        return hasMoreTokens;
    }

    /**
     * Returns the next Token.
     * 
     * @see java.util.Iterator#next()
     */
    public Object next() {
        if (!hasMoreTokens) {
            throw new NoSuchElementException();
        }
        skipWhitespace();
        if (position >= line.length()) {
            hasMoreTokens = false;
            return pack(Type.EOL, new StringBuilder());
        }
        else {
            return parseToken();
        }
    }

    /**
     * Extracts a Token from the line being parsed by this Tokenizer.
     * 
     * @return The next Token.
     */
    private Token parseToken() {
        State state = State.START;
        StringBuilder builder = new StringBuilder();
        while (position < line.length()) {
            char ch = line.charAt(position);
            position++;
            switch (state) {
                case START:
                    if (Character.isLetter(ch)) {
                        builder.append(ch);
                        state = State.NAME;
                        break;
                    }
                    else if (Character.isDigit(ch)) {
                        builder.append(ch);
                        state = State.NUMBER;
                        break;
                    }
                    else {
                        builder.append(ch);
                        return pack(Type.SYMBOL, builder);
                        
                    }
                case NAME:
                    if (Character.isLetterOrDigit(ch) || ch == '_') {
                        builder.append(ch);
                        continue;
                    }
                    else {
                        position--;
                        return pack(Type.NAME, builder);
                    }
                case NUMBER:
                    if (Character.isDigit(ch)) {
                        builder.append(ch);
                        continue;
                    }
                    else {
                        position--;
                        return pack(Type.NUMBER, builder);
                    }
            }
        }
        return null; // Should never get here
    }
    
    /**
     * Creates a new Token from the given parameters. The new Token
     * is both saved in the <code>previousTokens</code> Stack
     * and returned.
     * 
     * @param type The type to make the new Token.
     * @param value The value to give to the new Token.
     * @return The newly created Token.
     */
    private Token pack(Type type, StringBuilder value) {
        Token result = new Token(type, value.toString());
        previousTokens.push(result);
        return result;
    }

    /**
     * Unsupported operation.
     * 
     * @see java.util.Iterator#remove()
     */
    public void remove() {
        throw new UnsupportedOperationException();
    }
    
    /**
     * Returns tokens to the Tokenizer, so that subsequent
     * calls to next() will return them again.
     * 
     * @param howMany The number of Tokens to restore to the Tokenizer.
     */
    public void putBack(int howMany) {
        if (howMany > previousTokens.size()) {
            throw new IllegalArgumentException();
        }
        hasMoreTokens = true;
        line = line.substring(position);
        for (int i = 0; i < howMany; i++) {
            Token t = previousTokens.pop();
            line = t.value + " " + line;
        }
        position = 0;
    }
    
    /**
     * Skips over whitespace in the String being tokenized.
     */
    private void skipWhitespace() {
        while (position < line.length() &&
               Character.isWhitespace(line.charAt(position))) {
            position++;
        }
    }
}