7096014: Javac tokens should retain state
authormcimadamore
Mon, 24 Oct 2011 13:00:20 +0100
changeset 10815 a719aa5f1631
parent 10814 7a74d080c7bf
child 10816 ce8a7e9d8882
7096014: Javac tokens should retain state Summary: Refactor javac tokens from enum constants to stateful instances (to keep track of position, comments, etc.) Reviewed-by: jjg
langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java
langtools/src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java
langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java
langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java
langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java
langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java
langtools/src/share/classes/com/sun/tools/javac/parser/Keywords.java
langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java
langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java
langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java
langtools/src/share/classes/com/sun/tools/javac/parser/Token.java
langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java
langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java
langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java
langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java
langtools/test/tools/javac/api/TestJavacTaskScanner.java
langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java
langtools/test/tools/javac/tree/DocCommentToplevelTest.java
--- a/langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java	Mon Oct 24 13:00:20 2011 +0100
@@ -42,7 +42,6 @@
 import com.sun.tools.apt.comp.*;
 import com.sun.tools.apt.util.Bark;
 import com.sun.mirror.apt.AnnotationProcessorFactory;
-import com.sun.tools.javac.parser.DocCommentScanner;
 
 /**
  *  <p><b>This is NOT part of any supported API.
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java	Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,420 +0,0 @@
-/*
- * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.nio.*;
-
-import com.sun.tools.javac.util.*;
-import static com.sun.tools.javac.util.LayoutCharacters.*;
-
-/** An extension to the base lexical analyzer that captures
- *  and processes the contents of doc comments.  It does so by
- *  translating Unicode escape sequences and by stripping the
- *  leading whitespace and starts from each line of the comment.
- *
- *  <p><b>This is NOT part of any supported API.
- *  If you write code that depends on this, you do so at your own risk.
- *  This code and its internal interfaces are subject to change or
- *  deletion without notice.</b>
- */
-public class DocCommentScanner extends Scanner {
-
-    /** Create a scanner from the input buffer.  buffer must implement
-     *  array() and compact(), and remaining() must be less than limit().
-     */
-    protected DocCommentScanner(ScannerFactory fac, CharBuffer buffer) {
-        super(fac, buffer);
-    }
-
-    /** Create a scanner from the input array.  The array must have at
-     *  least a single character of extra space.
-     */
-    protected DocCommentScanner(ScannerFactory fac, char[] input, int inputLength) {
-        super(fac, input, inputLength);
-    }
-
-    /** Starting position of the comment in original source
-     */
-    private int pos;
-
-    /** The comment input buffer, index of next chacter to be read,
-     *  index of one past last character in buffer.
-     */
-    private char[] buf;
-    private int bp;
-    private int buflen;
-
-    /** The current character.
-     */
-    private char ch;
-
-    /** The column number position of the current character.
-     */
-    private int col;
-
-    /** The buffer index of the last converted Unicode character
-     */
-    private int unicodeConversionBp = 0;
-
-    /**
-     * Buffer for doc comment.
-     */
-    private char[] docCommentBuffer = new char[1024];
-
-    /**
-     * Number of characters in doc comment buffer.
-     */
-    private int docCommentCount;
-
-    /**
-     * Translated and stripped contents of doc comment
-     */
-    private String docComment = null;
-
-
-    /** Unconditionally expand the comment buffer.
-     */
-    private void expandCommentBuffer() {
-        char[] newBuffer = new char[docCommentBuffer.length * 2];
-        System.arraycopy(docCommentBuffer, 0, newBuffer,
-                         0, docCommentBuffer.length);
-        docCommentBuffer = newBuffer;
-    }
-
-    /** Convert an ASCII digit from its base (8, 10, or 16)
-     *  to its value.
-     */
-    private int digit(int base) {
-        char c = ch;
-        int result = Character.digit(c, base);
-        if (result >= 0 && c > 0x7f) {
-            ch = "0123456789abcdef".charAt(result);
-        }
-        return result;
-    }
-
-    /** Convert Unicode escape; bp points to initial '\' character
-     *  (Spec 3.3).
-     */
-    private void convertUnicode() {
-        if (ch == '\\' && unicodeConversionBp != bp) {
-            bp++; ch = buf[bp]; col++;
-            if (ch == 'u') {
-                do {
-                    bp++; ch = buf[bp]; col++;
-                } while (ch == 'u');
-                int limit = bp + 3;
-                if (limit < buflen) {
-                    int d = digit(16);
-                    int code = d;
-                    while (bp < limit && d >= 0) {
-                        bp++; ch = buf[bp]; col++;
-                        d = digit(16);
-                        code = (code << 4) + d;
-                    }
-                    if (d >= 0) {
-                        ch = (char)code;
-                        unicodeConversionBp = bp;
-                        return;
-                    }
-                }
-                // "illegal.Unicode.esc", reported by base scanner
-            } else {
-                bp--;
-                ch = '\\';
-                col--;
-            }
-        }
-    }
-
-
-    /** Read next character.
-     */
-    private void scanChar() {
-        bp++;
-        ch = buf[bp];
-        switch (ch) {
-        case '\r': // return
-            col = 0;
-            break;
-        case '\n': // newline
-            if (bp == 0 || buf[bp-1] != '\r') {
-                col = 0;
-            }
-            break;
-        case '\t': // tab
-            col = (col / TabInc * TabInc) + TabInc;
-            break;
-        case '\\': // possible Unicode
-            col++;
-            convertUnicode();
-            break;
-        default:
-            col++;
-            break;
-        }
-    }
-
-    /**
-     * Read next character in doc comment, skipping over double '\' characters.
-     * If a double '\' is skipped, put in the buffer and update buffer count.
-     */
-    private void scanDocCommentChar() {
-        scanChar();
-        if (ch == '\\') {
-            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
-                if (docCommentCount == docCommentBuffer.length)
-                    expandCommentBuffer();
-                docCommentBuffer[docCommentCount++] = ch;
-                bp++; col++;
-            } else {
-                convertUnicode();
-            }
-        }
-    }
-
-    /* Reset doc comment before reading each new token
-     */
-    public void nextToken() {
-        docComment = null;
-        super.nextToken();
-    }
-
-    /**
-     * Returns the documentation string of the current token.
-     */
-    public String docComment() {
-        return docComment;
-    }
-
-    /**
-     * Process a doc comment and make the string content available.
-     * Strips leading whitespace and stars.
-     */
-    @SuppressWarnings("fallthrough")
-    protected void processComment(CommentStyle style) {
-        if (style != CommentStyle.JAVADOC) {
-            return;
-        }
-
-        pos = pos();
-        buf = getRawCharacters(pos, endPos());
-        buflen = buf.length;
-        bp = 0;
-        col = 0;
-
-        docCommentCount = 0;
-
-        boolean firstLine = true;
-
-        // Skip over first slash
-        scanDocCommentChar();
-        // Skip over first star
-        scanDocCommentChar();
-
-        // consume any number of stars
-        while (bp < buflen && ch == '*') {
-            scanDocCommentChar();
-        }
-        // is the comment in the form /**/, /***/, /****/, etc. ?
-        if (bp < buflen && ch == '/') {
-            docComment = "";
-            return;
-        }
-
-        // skip a newline on the first line of the comment.
-        if (bp < buflen) {
-            if (ch == LF) {
-                scanDocCommentChar();
-                firstLine = false;
-            } else if (ch == CR) {
-                scanDocCommentChar();
-                if (ch == LF) {
-                    scanDocCommentChar();
-                    firstLine = false;
-                }
-            }
-        }
-
-    outerLoop:
-
-        // The outerLoop processes the doc comment, looping once
-        // for each line.  For each line, it first strips off
-        // whitespace, then it consumes any stars, then it
-        // puts the rest of the line into our buffer.
-        while (bp < buflen) {
-
-            // The wsLoop consumes whitespace from the beginning
-            // of each line.
-        wsLoop:
-
-            while (bp < buflen) {
-                switch(ch) {
-                case ' ':
-                    scanDocCommentChar();
-                    break;
-                case '\t':
-                    col = ((col - 1) / TabInc * TabInc) + TabInc;
-                    scanDocCommentChar();
-                    break;
-                case FF:
-                    col = 0;
-                    scanDocCommentChar();
-                    break;
-// Treat newline at beginning of line (blank line, no star)
-// as comment text.  Old Javadoc compatibility requires this.
-/*---------------------------------*
-                case CR: // (Spec 3.4)
-                    scanDocCommentChar();
-                    if (ch == LF) {
-                        col = 0;
-                        scanDocCommentChar();
-                    }
-                    break;
-                case LF: // (Spec 3.4)
-                    scanDocCommentChar();
-                    break;
-*---------------------------------*/
-                default:
-                    // we've seen something that isn't whitespace;
-                    // jump out.
-                    break wsLoop;
-                }
-            }
-
-            // Are there stars here?  If so, consume them all
-            // and check for the end of comment.
-            if (ch == '*') {
-                // skip all of the stars
-                do {
-                    scanDocCommentChar();
-                } while (ch == '*');
-
-                // check for the closing slash.
-                if (ch == '/') {
-                    // We're done with the doc comment
-                    // scanChar() and breakout.
-                    break outerLoop;
-                }
-            } else if (! firstLine) {
-                //The current line does not begin with a '*' so we will indent it.
-                for (int i = 1; i < col; i++) {
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ' ';
-                }
-            }
-
-            // The textLoop processes the rest of the characters
-            // on the line, adding them to our buffer.
-        textLoop:
-            while (bp < buflen) {
-                switch (ch) {
-                case '*':
-                    // Is this just a star?  Or is this the
-                    // end of a comment?
-                    scanDocCommentChar();
-                    if (ch == '/') {
-                        // This is the end of the comment,
-                        // set ch and return our buffer.
-                        break outerLoop;
-                    }
-                    // This is just an ordinary star.  Add it to
-                    // the buffer.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = '*';
-                    break;
-                case ' ':
-                case '\t':
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                    break;
-                case FF:
-                    scanDocCommentChar();
-                    break textLoop; // treat as end of line
-                case CR: // (Spec 3.4)
-                    scanDocCommentChar();
-                    if (ch != LF) {
-                        // Canonicalize CR-only line terminator to LF
-                        if (docCommentCount == docCommentBuffer.length)
-                            expandCommentBuffer();
-                        docCommentBuffer[docCommentCount++] = (char)LF;
-                        break textLoop;
-                    }
-                    /* fall through to LF case */
-                case LF: // (Spec 3.4)
-                    // We've seen a newline.  Add it to our
-                    // buffer and break out of this loop,
-                    // starting fresh on a new line.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                    break textLoop;
-                default:
-                    // Add the character to our buffer.
-                    if (docCommentCount == docCommentBuffer.length)
-                        expandCommentBuffer();
-                    docCommentBuffer[docCommentCount++] = ch;
-                    scanDocCommentChar();
-                }
-            } // end textLoop
-            firstLine = false;
-        } // end outerLoop
-
-        if (docCommentCount > 0) {
-            int i = docCommentCount - 1;
-        trailLoop:
-            while (i > -1) {
-                switch (docCommentBuffer[i]) {
-                case '*':
-                    i--;
-                    break;
-                default:
-                    break trailLoop;
-                }
-            }
-            docCommentCount = i + 1;
-
-            // Store the text of the doc comment
-            docComment = new String(docCommentBuffer, 0 , docCommentCount);
-        } else {
-            docComment = "";
-        }
-    }
-
-    /** Build a map for translating between line numbers and
-     * positions in the input.
-     *
-     * @return a LineMap */
-    public Position.LineMap getLineMap() {
-        char[] buf = getRawCharacters();
-        return Position.makeLineMap(buf, buf.length, true);
-    }
-}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java	Mon Oct 24 13:00:20 2011 +0100
@@ -67,14 +67,14 @@
     /** {@inheritDoc} */
     @Override
     protected <T extends JCTree> T to(T t) {
-        storeEnd(t, S.endPos());
+        storeEnd(t, token.endPos);
         return t;
     }
 
     /** {@inheritDoc} */
     @Override
     protected <T extends JCTree> T toP(T t) {
-        storeEnd(t, S.prevEndPos());
+        storeEnd(t, S.prevToken().endPos);
         return t;
     }
 
@@ -88,7 +88,7 @@
     /** {@inheritDoc} */
     @Override
     JCExpression parExpression() {
-        int pos = S.pos();
+        int pos = token.pos;
         JCExpression t = super.parExpression();
         return toP(F.at(pos).Parens(t));
     }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,896 @@
+/*
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.nio.CharBuffer;
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.util.*;
+
+
+import static com.sun.tools.javac.parser.Tokens.*;
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The lexical analyzer maps an input stream consisting of
+ *  ASCII characters and Unicode escapes into a token sequence.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class JavaTokenizer {
+
+    private static boolean scannerDebug = false;
+
+    /** Allow hex floating-point literals.
+     */
+    private boolean allowHexFloats;
+
+    /** Allow binary literals.
+     */
+    private boolean allowBinaryLiterals;
+
+    /** Allow underscores in literals.
+     */
+    private boolean allowUnderscoresInLiterals;
+
+    /** The source language setting.
+     */
+    private Source source;
+
+    /** The log to be used for error reporting.
+     */
+    private final Log log;
+
+    /** The name table. */
+    private final Names names;
+
+    /** The token factory. */
+    private final Tokens tokens;
+
+    /** The token kind, set by nextToken().
+     */
+    protected TokenKind tk;
+
+    /** The token's radix, set by nextToken().
+     */
+    protected int radix;
+
+    /** The token's name, set by nextToken().
+     */
+    protected Name name;
+
+    /** The position where a lexical error occurred;
+     */
+    protected int errPos = Position.NOPOS;
+
+    /** Has a @deprecated been encountered in last doc comment?
+     *  this needs to be reset by client.
+     */
+    protected boolean deprecatedFlag = false;
+
+    /** A character buffer for saved chars.
+     */
+    protected char[] sbuf = new char[128];
+    protected int sp;
+
+    protected UnicodeReader reader;
+
+    private static final boolean hexFloatsWork = hexFloatsWork();
+    private static boolean hexFloatsWork() {
+        try {
+            Float.valueOf("0x1.0p1");
+            return true;
+        } catch (NumberFormatException ex) {
+            return false;
+        }
+    }
+
+    /**
+     * Create a scanner from the input array.  This method might
+     * modify the array.  To avoid copying the input array, ensure
+     * that {@code inputLength < input.length} or
+     * {@code input[input.length -1]} is a white space character.
+     *
+     * @param fac the factory which created this Scanner
+     * @param input the input, might be modified
+     * @param inputLength the size of the input.
+     * Must be positive and less than or equal to input.length.
+     */
+    protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
+        this(fac, new UnicodeReader(fac, buf));
+    }
+
+    protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
+        this(fac, new UnicodeReader(fac, buf, inputLength));
+    }
+
+    protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
+        log = fac.log;
+        names = fac.names;
+        tokens = fac.tokens;
+        source = fac.source;
+        this.reader = reader;
+        allowBinaryLiterals = source.allowBinaryLiterals();
+        allowHexFloats = source.allowHexFloats();
+        allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+    }
+
+    /** Report an error at the given position using the provided arguments.
+     */
+    protected void lexError(int pos, String key, Object... args) {
+        log.error(pos, key, args);
+        tk = TokenKind.ERROR;
+        errPos = pos;
+    }
+
+    /** Read next character in comment, skipping over double '\' characters.
+     */
+    protected void scanCommentChar() {
+        reader.scanChar();
+        if (reader.ch == '\\') {
+            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+                reader.skipChar();
+            } else {
+                reader.convertUnicode();
+            }
+        }
+    }
+
+    /** Append a character to sbuf.
+     */
+    private void putChar(char ch) {
+        if (sp == sbuf.length) {
+            char[] newsbuf = new char[sbuf.length * 2];
+            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
+            sbuf = newsbuf;
+        }
+        sbuf[sp++] = ch;
+    }
+
+    /** Read next character in character or string literal and copy into sbuf.
+     */
+    private void scanLitChar(int pos) {
+        if (reader.ch == '\\') {
+            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+                reader.skipChar();
+                putChar('\\');
+                reader.scanChar();
+            } else {
+                reader.scanChar();
+                switch (reader.ch) {
+                case '0': case '1': case '2': case '3':
+                case '4': case '5': case '6': case '7':
+                    char leadch = reader.ch;
+                    int oct = reader.digit(pos, 8);
+                    reader.scanChar();
+                    if ('0' <= reader.ch && reader.ch <= '7') {
+                        oct = oct * 8 + reader.digit(pos, 8);
+                        reader.scanChar();
+                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
+                            oct = oct * 8 + reader.digit(pos, 8);
+                            reader.scanChar();
+                        }
+                    }
+                    putChar((char)oct);
+                    break;
+                case 'b':
+                    putChar('\b'); reader.scanChar(); break;
+                case 't':
+                    putChar('\t'); reader.scanChar(); break;
+                case 'n':
+                    putChar('\n'); reader.scanChar(); break;
+                case 'f':
+                    putChar('\f'); reader.scanChar(); break;
+                case 'r':
+                    putChar('\r'); reader.scanChar(); break;
+                case '\'':
+                    putChar('\''); reader.scanChar(); break;
+                case '\"':
+                    putChar('\"'); reader.scanChar(); break;
+                case '\\':
+                    putChar('\\'); reader.scanChar(); break;
+                default:
+                    lexError(reader.bp, "illegal.esc.char");
+                }
+            }
+        } else if (reader.bp != reader.buflen) {
+            putChar(reader.ch); reader.scanChar();
+        }
+    }
+
+    private void scanDigits(int pos, int digitRadix) {
+        char saveCh;
+        int savePos;
+        do {
+            if (reader.ch != '_') {
+                putChar(reader.ch);
+            } else {
+                if (!allowUnderscoresInLiterals) {
+                    lexError(pos, "unsupported.underscore.lit", source.name);
+                    allowUnderscoresInLiterals = true;
+                }
+            }
+            saveCh = reader.ch;
+            savePos = reader.bp;
+            reader.scanChar();
+        } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
+        if (saveCh == '_')
+            lexError(savePos, "illegal.underscore");
+    }
+
+    /** Read fractional part of hexadecimal floating point number.
+     */
+    private void scanHexExponentAndSuffix(int pos) {
+        if (reader.ch == 'p' || reader.ch == 'P') {
+            putChar(reader.ch);
+            reader.scanChar();
+            skipIllegalUnderscores();
+            if (reader.ch == '+' || reader.ch == '-') {
+                putChar(reader.ch);
+                reader.scanChar();
+            }
+            skipIllegalUnderscores();
+            if ('0' <= reader.ch && reader.ch <= '9') {
+                scanDigits(pos, 10);
+                if (!allowHexFloats) {
+                    lexError(pos, "unsupported.fp.lit", source.name);
+                    allowHexFloats = true;
+                }
+                else if (!hexFloatsWork)
+                    lexError(pos, "unsupported.cross.fp.lit");
+            } else
+                lexError(pos, "malformed.fp.lit");
+        } else {
+            lexError(pos, "malformed.fp.lit");
+        }
+        if (reader.ch == 'f' || reader.ch == 'F') {
+            putChar(reader.ch);
+            reader.scanChar();
+            tk = TokenKind.FLOATLITERAL;
+            radix = 16;
+        } else {
+            if (reader.ch == 'd' || reader.ch == 'D') {
+                putChar(reader.ch);
+                reader.scanChar();
+            }
+            tk = TokenKind.DOUBLELITERAL;
+            radix = 16;
+        }
+    }
+
+    /** Read fractional part of floating point number.
+     */
+    private void scanFraction(int pos) {
+        skipIllegalUnderscores();
+        if ('0' <= reader.ch && reader.ch <= '9') {
+            scanDigits(pos, 10);
+        }
+        int sp1 = sp;
+        if (reader.ch == 'e' || reader.ch == 'E') {
+            putChar(reader.ch);
+            reader.scanChar();
+            skipIllegalUnderscores();
+            if (reader.ch == '+' || reader.ch == '-') {
+                putChar(reader.ch);
+                reader.scanChar();
+            }
+            skipIllegalUnderscores();
+            if ('0' <= reader.ch && reader.ch <= '9') {
+                scanDigits(pos, 10);
+                return;
+            }
+            lexError(pos, "malformed.fp.lit");
+            sp = sp1;
+        }
+    }
+
+    /** Read fractional part and 'd' or 'f' suffix of floating point number.
+     */
+    private void scanFractionAndSuffix(int pos) {
+        radix = 10;
+        scanFraction(pos);
+        if (reader.ch == 'f' || reader.ch == 'F') {
+            putChar(reader.ch);
+            reader.scanChar();
+            tk = TokenKind.FLOATLITERAL;
+        } else {
+            if (reader.ch == 'd' || reader.ch == 'D') {
+                putChar(reader.ch);
+                reader.scanChar();
+            }
+            tk = TokenKind.DOUBLELITERAL;
+        }
+    }
+
+    /** Read fractional part and 'd' or 'f' suffix of floating point number.
+     */
+    private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
+        radix = 16;
+        Assert.check(reader.ch == '.');
+        putChar(reader.ch);
+        reader.scanChar();
+        skipIllegalUnderscores();
+        if (reader.digit(pos, 16) >= 0) {
+            seendigit = true;
+            scanDigits(pos, 16);
+        }
+        if (!seendigit)
+            lexError(pos, "invalid.hex.number");
+        else
+            scanHexExponentAndSuffix(pos);
+    }
+
+    private void skipIllegalUnderscores() {
+        if (reader.ch == '_') {
+            lexError(reader.bp, "illegal.underscore");
+            while (reader.ch == '_')
+                reader.scanChar();
+        }
+    }
+
+    /** Read a number.
+     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
+     */
+    private void scanNumber(int pos, int radix) {
+        // for octal, allow base-10 digit in case it's a float literal
+        this.radix = radix;
+        int digitRadix = (radix == 8 ? 10 : radix);
+        boolean seendigit = false;
+        if (reader.digit(pos, digitRadix) >= 0) {
+            seendigit = true;
+            scanDigits(pos, digitRadix);
+        }
+        if (radix == 16 && reader.ch == '.') {
+            scanHexFractionAndSuffix(pos, seendigit);
+        } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
+            scanHexExponentAndSuffix(pos);
+        } else if (digitRadix == 10 && reader.ch == '.') {
+            putChar(reader.ch);
+            reader.scanChar();
+            scanFractionAndSuffix(pos);
+        } else if (digitRadix == 10 &&
+                   (reader.ch == 'e' || reader.ch == 'E' ||
+                    reader.ch == 'f' || reader.ch == 'F' ||
+                    reader.ch == 'd' || reader.ch == 'D')) {
+            scanFractionAndSuffix(pos);
+        } else {
+            if (reader.ch == 'l' || reader.ch == 'L') {
+                reader.scanChar();
+                tk = TokenKind.LONGLITERAL;
+            } else {
+                tk = TokenKind.INTLITERAL;
+            }
+        }
+    }
+
+    /** Read an identifier.
+     */
+    private void scanIdent() {
+        boolean isJavaIdentifierPart;
+        char high;
+        do {
+            if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
+            // optimization, was: putChar(reader.ch);
+
+            reader.scanChar();
+            switch (reader.ch) {
+            case 'A': case 'B': case 'C': case 'D': case 'E':
+            case 'F': case 'G': case 'H': case 'I': case 'J':
+            case 'K': case 'L': case 'M': case 'N': case 'O':
+            case 'P': case 'Q': case 'R': case 'S': case 'T':
+            case 'U': case 'V': case 'W': case 'X': case 'Y':
+            case 'Z':
+            case 'a': case 'b': case 'c': case 'd': case 'e':
+            case 'f': case 'g': case 'h': case 'i': case 'j':
+            case 'k': case 'l': case 'm': case 'n': case 'o':
+            case 'p': case 'q': case 'r': case 's': case 't':
+            case 'u': case 'v': case 'w': case 'x': case 'y':
+            case 'z':
+            case '$': case '_':
+            case '0': case '1': case '2': case '3': case '4':
+            case '5': case '6': case '7': case '8': case '9':
+            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
+            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
+            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
+            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
+            case '\u0015': case '\u0016': case '\u0017':
+            case '\u0018': case '\u0019': case '\u001B':
+            case '\u007F':
+                break;
+            case '\u001A': // EOI is also a legal identifier part
+                if (reader.bp >= reader.buflen) {
+                    name = names.fromChars(sbuf, 0, sp);
+                    tk = tokens.lookupKind(name);
+                    return;
+                }
+                break;
+            default:
+                if (reader.ch < '\u0080') {
+                    // all ASCII range chars already handled, above
+                    isJavaIdentifierPart = false;
+                } else {
+                    high = reader.scanSurrogates();
+                    if (high != 0) {
+                        if (sp == sbuf.length) {
+                            putChar(high);
+                        } else {
+                            sbuf[sp++] = high;
+                        }
+                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
+                            Character.toCodePoint(high, reader.ch));
+                    } else {
+                        isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                    }
+                }
+                if (!isJavaIdentifierPart) {
+                    name = names.fromChars(sbuf, 0, sp);
+                    tk = tokens.lookupKind(name);
+                    return;
+                }
+            }
+        } while (true);
+    }
+
+    /** Return true if reader.ch can be part of an operator.
+     */
+    private boolean isSpecial(char ch) {
+        switch (ch) {
+        case '!': case '%': case '&': case '*': case '?':
+        case '+': case '-': case ':': case '<': case '=':
+        case '>': case '^': case '|': case '~':
+        case '@':
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    /** Read longest possible sequence of special characters and convert
+     *  to token.
+     */
+    private void scanOperator() {
+        while (true) {
+            putChar(reader.ch);
+            Name newname = names.fromChars(sbuf, 0, sp);
+            TokenKind tk1 = tokens.lookupKind(newname);
+            if (tk1 == TokenKind.IDENTIFIER) {
+                sp--;
+                break;
+            }
+            tk = tk1;
+            reader.scanChar();
+            if (!isSpecial(reader.ch)) break;
+        }
+    }
+
+    /**
+     * Scan a documentation comment; determine if a deprecated tag is present.
+     * Called once the initial /, * have been skipped, positioned at the second *
+     * (which is treated as the beginning of the first line).
+     * Stops positioned at the closing '/'.
+     */
+    @SuppressWarnings("fallthrough")
+    private void scanDocComment() {
+        boolean deprecatedPrefix = false;
+
+        forEachLine:
+        while (reader.bp < reader.buflen) {
+
+            // Skip optional WhiteSpace at beginning of line
+            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+                scanCommentChar();
+            }
+
+            // Skip optional consecutive Stars
+            while (reader.bp < reader.buflen && reader.ch == '*') {
+                scanCommentChar();
+                if (reader.ch == '/') {
+                    return;
+                }
+            }
+
+            // Skip optional WhiteSpace after Stars
+            while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+                scanCommentChar();
+            }
+
+            deprecatedPrefix = false;
+            // At beginning of line in the JavaDoc sense.
+            if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
+                scanCommentChar();
+                if (reader.bp < reader.buflen && reader.ch == 'd') {
+                    scanCommentChar();
+                    if (reader.bp < reader.buflen && reader.ch == 'e') {
+                        scanCommentChar();
+                        if (reader.bp < reader.buflen && reader.ch == 'p') {
+                            scanCommentChar();
+                            if (reader.bp < reader.buflen && reader.ch == 'r') {
+                                scanCommentChar();
+                                if (reader.bp < reader.buflen && reader.ch == 'e') {
+                                    scanCommentChar();
+                                    if (reader.bp < reader.buflen && reader.ch == 'c') {
+                                        scanCommentChar();
+                                        if (reader.bp < reader.buflen && reader.ch == 'a') {
+                                            scanCommentChar();
+                                            if (reader.bp < reader.buflen && reader.ch == 't') {
+                                                scanCommentChar();
+                                                if (reader.bp < reader.buflen && reader.ch == 'e') {
+                                                    scanCommentChar();
+                                                    if (reader.bp < reader.buflen && reader.ch == 'd') {
+                                                        deprecatedPrefix = true;
+                                                        scanCommentChar();
+                                                    }}}}}}}}}}}
+            if (deprecatedPrefix && reader.bp < reader.buflen) {
+                if (Character.isWhitespace(reader.ch)) {
+                    deprecatedFlag = true;
+                } else if (reader.ch == '*') {
+                    scanCommentChar();
+                    if (reader.ch == '/') {
+                        deprecatedFlag = true;
+                        return;
+                    }
+                }
+            }
+
+            // Skip rest of line
+            while (reader.bp < reader.buflen) {
+                switch (reader.ch) {
+                case '*':
+                    scanCommentChar();
+                    if (reader.ch == '/') {
+                        return;
+                    }
+                    break;
+                case CR: // (Spec 3.4)
+                    scanCommentChar();
+                    if (reader.ch != LF) {
+                        continue forEachLine;
+                    }
+                    /* fall through to LF case */
+                case LF: // (Spec 3.4)
+                    scanCommentChar();
+                    continue forEachLine;
+                default:
+                    scanCommentChar();
+                }
+            } // rest of line
+        } // forEachLine
+        return;
+    }
+
+    /** Read token.
+     */
+    public Token readToken() {
+
+        sp = 0;
+        name = null;
+        deprecatedFlag = false;
+        radix = 0;
+        int pos = 0;
+        int endPos = 0;
+
+        try {
+            loop: while (true) {
+                pos = reader.bp;
+                switch (reader.ch) {
+                case ' ': // (Spec 3.6)
+                case '\t': // (Spec 3.6)
+                case FF: // (Spec 3.6)
+                    do {
+                        reader.scanChar();
+                    } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
+                    processWhiteSpace(pos, reader.bp);
+                    break;
+                case LF: // (Spec 3.4)
+                    reader.scanChar();
+                    processLineTerminator(pos, reader.bp);
+                    break;
+                case CR: // (Spec 3.4)
+                    reader.scanChar();
+                    if (reader.ch == LF) {
+                        reader.scanChar();
+                    }
+                    processLineTerminator(pos, reader.bp);
+                    break;
+                case 'A': case 'B': case 'C': case 'D': case 'E':
+                case 'F': case 'G': case 'H': case 'I': case 'J':
+                case 'K': case 'L': case 'M': case 'N': case 'O':
+                case 'P': case 'Q': case 'R': case 'S': case 'T':
+                case 'U': case 'V': case 'W': case 'X': case 'Y':
+                case 'Z':
+                case 'a': case 'b': case 'c': case 'd': case 'e':
+                case 'f': case 'g': case 'h': case 'i': case 'j':
+                case 'k': case 'l': case 'm': case 'n': case 'o':
+                case 'p': case 'q': case 'r': case 's': case 't':
+                case 'u': case 'v': case 'w': case 'x': case 'y':
+                case 'z':
+                case '$': case '_':
+                    scanIdent();
+                    break loop;
+                case '0':
+                    reader.scanChar();
+                    if (reader.ch == 'x' || reader.ch == 'X') {
+                        reader.scanChar();
+                        skipIllegalUnderscores();
+                        if (reader.ch == '.') {
+                            scanHexFractionAndSuffix(pos, false);
+                        } else if (reader.digit(pos, 16) < 0) {
+                            lexError(pos, "invalid.hex.number");
+                        } else {
+                            scanNumber(pos, 16);
+                        }
+                    } else if (reader.ch == 'b' || reader.ch == 'B') {
+                        if (!allowBinaryLiterals) {
+                            lexError(pos, "unsupported.binary.lit", source.name);
+                            allowBinaryLiterals = true;
+                        }
+                        reader.scanChar();
+                        skipIllegalUnderscores();
+                        if (reader.digit(pos, 2) < 0) {
+                            lexError(pos, "invalid.binary.number");
+                        } else {
+                            scanNumber(pos, 2);
+                        }
+                    } else {
+                        putChar('0');
+                        if (reader.ch == '_') {
+                            int savePos = reader.bp;
+                            do {
+                                reader.scanChar();
+                            } while (reader.ch == '_');
+                            if (reader.digit(pos, 10) < 0) {
+                                lexError(savePos, "illegal.underscore");
+                            }
+                        }
+                        scanNumber(pos, 8);
+                    }
+                    break loop;
+                case '1': case '2': case '3': case '4':
+                case '5': case '6': case '7': case '8': case '9':
+                    scanNumber(pos, 10);
+                    break loop;
+                case '.':
+                    reader.scanChar();
+                    if ('0' <= reader.ch && reader.ch <= '9') {
+                        putChar('.');
+                        scanFractionAndSuffix(pos);
+                    } else if (reader.ch == '.') {
+                        putChar('.'); putChar('.');
+                        reader.scanChar();
+                        if (reader.ch == '.') {
+                            reader.scanChar();
+                            putChar('.');
+                            tk = TokenKind.ELLIPSIS;
+                        } else {
+                            lexError(pos, "malformed.fp.lit");
+                        }
+                    } else {
+                        tk = TokenKind.DOT;
+                    }
+                    break loop;
+                case ',':
+                    reader.scanChar(); tk = TokenKind.COMMA; break loop;
+                case ';':
+                    reader.scanChar(); tk = TokenKind.SEMI; break loop;
+                case '(':
+                    reader.scanChar(); tk = TokenKind.LPAREN; break loop;
+                case ')':
+                    reader.scanChar(); tk = TokenKind.RPAREN; break loop;
+                case '[':
+                    reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
+                case ']':
+                    reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
+                case '{':
+                    reader.scanChar(); tk = TokenKind.LBRACE; break loop;
+                case '}':
+                    reader.scanChar(); tk = TokenKind.RBRACE; break loop;
+                case '/':
+                    reader.scanChar();
+                    if (reader.ch == '/') {
+                        do {
+                            scanCommentChar();
+                        } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
+                        if (reader.bp < reader.buflen) {
+                            processComment(pos, reader.bp, CommentStyle.LINE);
+                        }
+                        break;
+                    } else if (reader.ch == '*') {
+                        reader.scanChar();
+                        CommentStyle style;
+                        if (reader.ch == '*') {
+                            style = CommentStyle.JAVADOC;
+                            scanDocComment();
+                        } else {
+                            style = CommentStyle.BLOCK;
+                            while (reader.bp < reader.buflen) {
+                                if (reader.ch == '*') {
+                                    reader.scanChar();
+                                    if (reader.ch == '/') break;
+                                } else {
+                                    scanCommentChar();
+                                }
+                            }
+                        }
+                        if (reader.ch == '/') {
+                            reader.scanChar();
+                            processComment(pos, reader.bp, style);
+                            break;
+                        } else {
+                            lexError(pos, "unclosed.comment");
+                            break loop;
+                        }
+                    } else if (reader.ch == '=') {
+                        tk = TokenKind.SLASHEQ;
+                        reader.scanChar();
+                    } else {
+                        tk = TokenKind.SLASH;
+                    }
+                    break loop;
+                case '\'':
+                    reader.scanChar();
+                    if (reader.ch == '\'') {
+                        lexError(pos, "empty.char.lit");
+                    } else {
+                        if (reader.ch == CR || reader.ch == LF)
+                            lexError(pos, "illegal.line.end.in.char.lit");
+                        scanLitChar(pos);
+                        char ch2 = reader.ch;
+                        if (reader.ch == '\'') {
+                            reader.scanChar();
+                            tk = TokenKind.CHARLITERAL;
+                        } else {
+                            lexError(pos, "unclosed.char.lit");
+                        }
+                    }
+                    break loop;
+                case '\"':
+                    reader.scanChar();
+                    while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
+                        scanLitChar(pos);
+                    if (reader.ch == '\"') {
+                        tk = TokenKind.STRINGLITERAL;
+                        reader.scanChar();
+                    } else {
+                        lexError(pos, "unclosed.str.lit");
+                    }
+                    break loop;
+                default:
+                    if (isSpecial(reader.ch)) {
+                        scanOperator();
+                    } else {
+                        boolean isJavaIdentifierStart;
+                        if (reader.ch < '\u0080') {
+                            // all ASCII range chars already handled, above
+                            isJavaIdentifierStart = false;
+                        } else {
+                            char high = reader.scanSurrogates();
+                            if (high != 0) {
+                                if (sp == sbuf.length) {
+                                    putChar(high);
+                                } else {
+                                    sbuf[sp++] = high;
+                                }
+
+                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
+                                    Character.toCodePoint(high, reader.ch));
+                            } else {
+                                isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
+                            }
+                        }
+                        if (isJavaIdentifierStart) {
+                            scanIdent();
+                        } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
+                            tk = TokenKind.EOF;
+                            pos = reader.buflen;
+                        } else {
+                            lexError(pos, "illegal.char", String.valueOf((int)reader.ch));
+                            reader.scanChar();
+                        }
+                    }
+                    break loop;
+                }
+            }
+            endPos = reader.bp;
+            switch (tk.tag) {
+                case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
+                case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
+                case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
+                case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
+                default: throw new AssertionError();
+            }
+        }
+        finally {
+            if (scannerDebug) {
+                    System.out.println("nextToken(" + pos
+                                       + "," + endPos + ")=|" +
+                                       new String(reader.getRawCharacters(pos, endPos))
+                                       + "|");
+            }
+        }
+    }
+
+    /** Return the position where a lexical error occurred;
+     */
+    public int errPos() {
+        return errPos;
+    }
+
+    /** Set the position where a lexical error occurred;
+     */
+    public void errPos(int pos) {
+        errPos = pos;
+    }
+
+    public enum CommentStyle {
+        LINE,
+        BLOCK,
+        JAVADOC,
+    }
+
+    /**
+     * Called when a complete comment has been scanned. pos and endPos
+     * will mark the comment boundary.
+     */
+    protected void processComment(int pos, int endPos, CommentStyle style) {
+        if (scannerDebug)
+            System.out.println("processComment(" + pos
+                               + "," + endPos + "," + style + ")=|"
+                               + new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+    }
+
+    /**
+     * Called when a complete whitespace run has been scanned. pos and endPos
+     * will mark the whitespace boundary.
+     */
+    protected void processWhiteSpace(int pos, int endPos) {
+        if (scannerDebug)
+            System.out.println("processWhitespace(" + pos
+                               + "," + endPos + ")=|" +
+                               new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+    }
+
+    /**
+     * Called when a line terminator has been processed.
+     */
+    protected void processLineTerminator(int pos, int endPos) {
+        if (scannerDebug)
+            System.out.println("processTerminator(" + pos
+                               + "," + endPos + ")=|" +
+                               new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+    }
+
+    /** Build a map for translating between line numbers and
+     * positions in the input.
+     *
+     * @return a LineMap */
+    public Position.LineMap getLineMap() {
+        return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
+    }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java	Mon Oct 24 13:00:20 2011 +0100
@@ -28,6 +28,7 @@
 import java.util.*;
 
 import com.sun.tools.javac.code.*;
+import com.sun.tools.javac.parser.Tokens.*;
 import com.sun.tools.javac.tree.*;
 import com.sun.tools.javac.tree.JCTree.*;
 import com.sun.tools.javac.util.*;
@@ -36,7 +37,7 @@
 import com.sun.tools.javac.util.List;
 
 import static com.sun.tools.javac.util.ListBuffer.lb;
-import static com.sun.tools.javac.parser.Token.*;
+import static com.sun.tools.javac.parser.Tokens.TokenKind.*;
 
 /** The parser maps a token sequence into an abstract syntax
  *  tree. It operates by recursive descent, with code derived
@@ -67,9 +68,6 @@
      */
     private Log log;
 
-    /** The keyword table. */
-    private Keywords keywords;
-
     /** The Source language setting. */
     private Source source;
 
@@ -83,11 +81,10 @@
                      boolean keepDocComments,
                      boolean keepLineMap) {
         this.S = S;
-        S.nextToken(); // prime the pump
+        nextToken(); // prime the pump
         this.F = fac.F;
         this.log = fac.log;
         this.names = fac.names;
-        this.keywords = fac.keywords;
         this.source = fac.source;
         this.allowGenerics = source.allowGenerics();
         this.allowVarargs = source.allowVarargs();
@@ -178,7 +175,16 @@
      */
     private int lastmode = 0;
 
-/* ---------- error recovery -------------- */
+    /* ---------- token management -------------- */
+
+    protected Token token;
+
+    protected void nextToken() {
+        S.nextToken();
+        token = S.token();
+    }
+
+    /* ---------- error recovery -------------- */
 
     private JCErroneous errorTree;
 
@@ -186,9 +192,9 @@
      */
     private void skip(boolean stopAtImport, boolean stopAtMemberDecl, boolean stopAtIdentifier, boolean stopAtStatement) {
          while (true) {
-             switch (S.token()) {
+             switch (token.kind) {
                 case SEMI:
-                    S.nextToken();
+                    nextToken();
                     return;
                 case PUBLIC:
                 case FINAL:
@@ -249,15 +255,15 @@
                         return;
                     break;
             }
-            S.nextToken();
+            nextToken();
         }
     }
 
-    private JCErroneous syntaxError(int pos, String key, Token... args) {
+    private JCErroneous syntaxError(int pos, String key, TokenKind... args) {
         return syntaxError(pos, List.<JCTree>nil(), key, args);
     }
 
-    private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, Token... args) {
+    private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, TokenKind... args) {
         setErrorEndPos(pos);
         JCErroneous err = F.at(pos).Erroneous(errs);
         reportSyntaxError(err, key, (Object[])args);
@@ -287,16 +293,16 @@
     private void reportSyntaxError(JCDiagnostic.DiagnosticPosition diagPos, String key, Object... args) {
         int pos = diagPos.getPreferredPosition();
         if (pos > S.errPos() || pos == Position.NOPOS) {
-            if (S.token() == EOF) {
+            if (token.kind == EOF) {
                 error(diagPos, "premature.eof");
             } else {
                 error(diagPos, key, args);
             }
         }
         S.errPos(pos);
-        if (S.pos() == errorPos)
-            S.nextToken(); // guarantee progress
-        errorPos = S.pos();
+        if (token.pos == errorPos)
+            nextToken(); // guarantee progress
+        errorPos = token.pos;
     }
 
 
@@ -304,25 +310,25 @@
      *  reported at the same position.
      */
     private JCErroneous syntaxError(String key) {
-        return syntaxError(S.pos(), key);
+        return syntaxError(token.pos, key);
     }
 
     /** Generate a syntax error at current position unless one was
      *  already reported at the same position.
      */
-    private JCErroneous syntaxError(String key, Token arg) {
-        return syntaxError(S.pos(), key, arg);
+    private JCErroneous syntaxError(String key, TokenKind arg) {
+        return syntaxError(token.pos, key, arg);
     }
 
     /** If next input token matches given token, skip it, otherwise report
      *  an error.
      */
-    public void accept(Token token) {
-        if (S.token() == token) {
-            S.nextToken();
+    public void accept(TokenKind tk) {
+        if (token.kind == tk) {
+            nextToken();
         } else {
-            setErrorEndPos(S.pos());
-            reportSyntaxError(S.prevEndPos(), "expected", token);
+            setErrorEndPos(token.pos);
+            reportSyntaxError(S.prevToken().endPos, "expected", tk);
         }
     }
 
@@ -340,14 +346,14 @@
     /** Report an illegal start of expression/type error at current position.
      */
     JCExpression illegal() {
-        return illegal(S.pos());
+        return illegal(token.pos);
     }
 
     /** Diagnose a modifier flag from the set, if any. */
     void checkNoMods(long mods) {
         if (mods != 0) {
             long lowestMod = mods & -mods;
-            error(S.pos(), "mod.not.allowed.here",
+            error(token.pos, "mod.not.allowed.here",
                       Flags.asFlagSet(lowestMod));
         }
     }
@@ -435,30 +441,30 @@
      * Ident = IDENTIFIER
      */
     Name ident() {
-        if (S.token() == IDENTIFIER) {
-            Name name = S.name();
-            S.nextToken();
+        if (token.kind == IDENTIFIER) {
+            Name name = token.name();
+            nextToken();
             return name;
-        } else if (S.token() == ASSERT) {
+        } else if (token.kind == ASSERT) {
             if (allowAsserts) {
-                error(S.pos(), "assert.as.identifier");
-                S.nextToken();
+                error(token.pos, "assert.as.identifier");
+                nextToken();
                 return names.error;
             } else {
-                warning(S.pos(), "assert.as.identifier");
-                Name name = S.name();
-                S.nextToken();
+                warning(token.pos, "assert.as.identifier");
+                Name name = token.name();
+                nextToken();
                 return name;
             }
-        } else if (S.token() == ENUM) {
+        } else if (token.kind == ENUM) {
             if (allowEnums) {
-                error(S.pos(), "enum.as.identifier");
-                S.nextToken();
+                error(token.pos, "enum.as.identifier");
+                nextToken();
                 return names.error;
             } else {
-                warning(S.pos(), "enum.as.identifier");
-                Name name = S.name();
-                S.nextToken();
+                warning(token.pos, "enum.as.identifier");
+                Name name = token.name();
+                nextToken();
                 return name;
             }
         } else {
@@ -471,17 +477,17 @@
      * Qualident = Ident { DOT Ident }
      */
     public JCExpression qualident() {
-        JCExpression t = toP(F.at(S.pos()).Ident(ident()));
-        while (S.token() == DOT) {
-            int pos = S.pos();
-            S.nextToken();
+        JCExpression t = toP(F.at(token.pos).Ident(ident()));
+        while (token.kind == DOT) {
+            int pos = token.pos;
+            nextToken();
             t = toP(F.at(pos).Select(t, ident()));
         }
         return t;
     }
 
     JCExpression literal(Name prefix) {
-        return literal(prefix, S.pos());
+        return literal(prefix, token.pos);
     }
 
     /**
@@ -498,27 +504,29 @@
      */
     JCExpression literal(Name prefix, int pos) {
         JCExpression t = errorTree;
-        switch (S.token()) {
+        switch (token.kind) {
         case INTLITERAL:
             try {
                 t = F.at(pos).Literal(
                     TypeTags.INT,
-                    Convert.string2int(strval(prefix), S.radix()));
+                    Convert.string2int(strval(prefix), token.radix()));
             } catch (NumberFormatException ex) {
-                error(S.pos(), "int.number.too.large", strval(prefix));
+                error(token.pos, "int.number.too.large", strval(prefix));
             }
             break;
         case LONGLITERAL:
             try {
                 t = F.at(pos).Literal(
                     TypeTags.LONG,
-                    new Long(Convert.string2long(strval(prefix), S.radix())));
+                    new Long(Convert.string2long(strval(prefix), token.radix())));
             } catch (NumberFormatException ex) {
-                error(S.pos(), "int.number.too.large", strval(prefix));
+                error(token.pos, "int.number.too.large", strval(prefix));
             }
             break;
         case FLOATLITERAL: {
-            String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
+            String proper = token.radix() == 16 ?
+                    ("0x"+ token.stringVal()) :
+                    token.stringVal();
             Float n;
             try {
                 n = Float.valueOf(proper);
@@ -527,15 +535,17 @@
                 n = Float.NaN;
             }
             if (n.floatValue() == 0.0f && !isZero(proper))
-                error(S.pos(), "fp.number.too.small");
+                error(token.pos, "fp.number.too.small");
             else if (n.floatValue() == Float.POSITIVE_INFINITY)
-                error(S.pos(), "fp.number.too.large");
+                error(token.pos, "fp.number.too.large");
             else
                 t = F.at(pos).Literal(TypeTags.FLOAT, n);
             break;
         }
         case DOUBLELITERAL: {
-            String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
+            String proper = token.radix() == 16 ?
+                    ("0x"+ token.stringVal()) :
+                    token.stringVal();
             Double n;
             try {
                 n = Double.valueOf(proper);
@@ -544,9 +554,9 @@
                 n = Double.NaN;
             }
             if (n.doubleValue() == 0.0d && !isZero(proper))
-                error(S.pos(), "fp.number.too.small");
+                error(token.pos, "fp.number.too.small");
             else if (n.doubleValue() == Double.POSITIVE_INFINITY)
-                error(S.pos(), "fp.number.too.large");
+                error(token.pos, "fp.number.too.large");
             else
                 t = F.at(pos).Literal(TypeTags.DOUBLE, n);
             break;
@@ -554,17 +564,17 @@
         case CHARLITERAL:
             t = F.at(pos).Literal(
                 TypeTags.CHAR,
-                S.stringVal().charAt(0) + 0);
+                token.stringVal().charAt(0) + 0);
             break;
         case STRINGLITERAL:
             t = F.at(pos).Literal(
                 TypeTags.CLASS,
-                S.stringVal());
+                token.stringVal());
             break;
         case TRUE: case FALSE:
             t = F.at(pos).Literal(
                 TypeTags.BOOLEAN,
-                (S.token() == TRUE ? 1 : 0));
+                (token.kind == TRUE ? 1 : 0));
             break;
         case NULL:
             t = F.at(pos).Literal(
@@ -576,8 +586,8 @@
         }
         if (t == errorTree)
             t = F.at(pos).Erroneous();
-        storeEnd(t, S.endPos());
-        S.nextToken();
+        storeEnd(t, token.endPos);
+        nextToken();
         return t;
     }
 //where
@@ -590,7 +600,7 @@
         }
 
         String strval(Name prefix) {
-            String s = S.stringVal();
+            String s = token.stringVal();
             return prefix.isEmpty() ? s : prefix + s;
         }
 
@@ -627,17 +637,17 @@
     JCExpression term() {
         JCExpression t = term1();
         if ((mode & EXPR) != 0 &&
-            S.token() == EQ || PLUSEQ.compareTo(S.token()) <= 0 && S.token().compareTo(GTGTGTEQ) <= 0)
+            token.kind == EQ || PLUSEQ.compareTo(token.kind) <= 0 && token.kind.compareTo(GTGTGTEQ) <= 0)
             return termRest(t);
         else
             return t;
     }
 
     JCExpression termRest(JCExpression t) {
-        switch (S.token()) {
+        switch (token.kind) {
         case EQ: {
-            int pos = S.pos();
-            S.nextToken();
+            int pos = token.pos;
+            nextToken();
             mode = EXPR;
             JCExpression t1 = term();
             return toP(F.at(pos).Assign(t, t1));
@@ -653,12 +663,12 @@
         case LTLTEQ:
         case GTGTEQ:
         case GTGTGTEQ:
-            int pos = S.pos();
-            Token token = S.token();
-            S.nextToken();
+            int pos = token.pos;
+            TokenKind tk = token.kind;
+            nextToken();
             mode = EXPR;
             JCExpression t1 = term();
-            return F.at(pos).Assignop(optag(token), t, t1);
+            return F.at(pos).Assignop(optag(tk), t, t1);
         default:
             return t;
         }
@@ -670,7 +680,7 @@
      */
     JCExpression term1() {
         JCExpression t = term2();
-        if ((mode & EXPR) != 0 && S.token() == QUES) {
+        if ((mode & EXPR) != 0 && token.kind == QUES) {
             mode = EXPR;
             return term1Rest(t);
         } else {
@@ -681,9 +691,9 @@
     /** Expression1Rest = ["?" Expression ":" Expression1]
      */
     JCExpression term1Rest(JCExpression t) {
-        if (S.token() == QUES) {
-            int pos = S.pos();
-            S.nextToken();
+        if (token.kind == QUES) {
+            int pos = token.pos;
+            nextToken();
             JCExpression t1 = term();
             accept(COLON);
             JCExpression t2 = term1();
@@ -699,7 +709,7 @@
      */
     JCExpression term2() {
         JCExpression t = term3();
-        if ((mode & EXPR) != 0 && prec(S.token()) >= TreeInfo.orPrec) {
+        if ((mode & EXPR) != 0 && prec(token.kind) >= TreeInfo.orPrec) {
             mode = EXPR;
             return term2Rest(t, TreeInfo.orPrec);
         } else {
@@ -725,28 +735,23 @@
         JCExpression[] odStack = newOdStack();
         List<Token[]> savedOp = opStackSupply.elems;
         Token[] opStack = newOpStack();
-        List<int[]> savedPos = posStackSupply.elems;
-        int[] posStack = newPosStack();
+
         // optimization, was odStack = new Tree[...]; opStack = new Tree[...];
         int top = 0;
         odStack[0] = t;
-        int startPos = S.pos();
-        Token topOp = ERROR;
-        int topOpPos = Position.NOPOS;
-        while (prec(S.token()) >= minprec) {
-            posStack[top] = topOpPos;
+        int startPos = token.pos;
+        Token topOp = Tokens.DUMMY;
+        while (prec(token.kind) >= minprec) {
             opStack[top] = topOp;
             top++;
-            topOp = S.token();
-            topOpPos = S.pos();
-            S.nextToken();
-            odStack[top] = (topOp == INSTANCEOF) ? parseType() : term3();
-            while (top > 0 && prec(topOp) >= prec(S.token())) {
-                odStack[top-1] = makeOp(topOpPos, topOp, odStack[top-1],
+            topOp = token;
+            nextToken();
+            odStack[top] = (topOp.kind == INSTANCEOF) ? parseType() : term3();
+            while (top > 0 && prec(topOp.kind) >= prec(token.kind)) {
+                odStack[top-1] = makeOp(topOp.pos, topOp.kind, odStack[top-1],
                                         odStack[top]);
                 top--;
                 topOp = opStack[top];
-                topOpPos = posStack[top];
             }
         }
         Assert.check(top == 0);
@@ -761,14 +766,13 @@
 
         odStackSupply.elems = savedOd; // optimization
         opStackSupply.elems = savedOp; // optimization
-        posStackSupply.elems = savedPos; // optimization
         return t;
     }
 //where
         /** Construct a binary or type test node.
          */
         private JCExpression makeOp(int pos,
-                                    Token topOp,
+                                    TokenKind topOp,
                                     JCExpression od1,
                                     JCExpression od2)
         {
@@ -817,7 +821,6 @@
          */
         ListBuffer<JCExpression[]> odStackSupply = new ListBuffer<JCExpression[]>();
         ListBuffer<Token[]> opStackSupply = new ListBuffer<Token[]>();
-        ListBuffer<int[]> posStackSupply = new ListBuffer<int[]>();
 
         private JCExpression[] newOdStack() {
             if (odStackSupply.elems == odStackSupply.last)
@@ -835,14 +838,6 @@
             return opStack;
         }
 
-        private int[] newPosStack() {
-            if (posStackSupply.elems == posStackSupply.last)
-                posStackSupply.append(new int[infixPrecedenceLevels + 1]);
-            int[] posStack = posStackSupply.elems.head;
-            posStackSupply.elems = posStackSupply.elems.tail;
-            return posStack;
-        }
-
     /** Expression3    = PrefixOp Expression3
      *                 | "(" Expr | TypeNoParams ")" Expression3
      *                 | Primary {Selector} {PostfixOp}
@@ -871,10 +866,10 @@
      *  SuperSuffix    = Arguments | "." Ident [Arguments]
      */
     protected JCExpression term3() {
-        int pos = S.pos();
+        int pos = token.pos;
         JCExpression t;
         List<JCExpression> typeArgs = typeArgumentsOpt(EXPR);
-        switch (S.token()) {
+        switch (token.kind) {
         case QUES:
             if ((mode & TYPE) != 0 && (mode & (TYPEARG|NOPARAMS)) == TYPEARG) {
                 mode = TYPE;
@@ -883,49 +878,49 @@
                 return illegal();
         case PLUSPLUS: case SUBSUB: case BANG: case TILDE: case PLUS: case SUB:
             if (typeArgs == null && (mode & EXPR) != 0) {
-                Token token = S.token();
-                S.nextToken();
+                TokenKind tk = token.kind;
+                nextToken();
                 mode = EXPR;
-                if (token == SUB &&
-                    (S.token() == INTLITERAL || S.token() == LONGLITERAL) &&
-                    S.radix() == 10) {
+                if (tk == SUB &&
+                    (token.kind == INTLITERAL || token.kind == LONGLITERAL) &&
+                    token.radix() == 10) {
                     mode = EXPR;
                     t = literal(names.hyphen, pos);
                 } else {
                     t = term3();
-                    return F.at(pos).Unary(unoptag(token), t);
+                    return F.at(pos).Unary(unoptag(tk), t);
                 }
             } else return illegal();
             break;
         case LPAREN:
             if (typeArgs == null && (mode & EXPR) != 0) {
-                S.nextToken();
+                nextToken();
                 mode = EXPR | TYPE | NOPARAMS;
                 t = term3();
-                if ((mode & TYPE) != 0 && S.token() == LT) {
+                if ((mode & TYPE) != 0 && token.kind == LT) {
                     // Could be a cast to a parameterized type
                     int op = JCTree.LT;
-                    int pos1 = S.pos();
-                    S.nextToken();
+                    int pos1 = token.pos;
+                    nextToken();
                     mode &= (EXPR | TYPE);
                     mode |= TYPEARG;
                     JCExpression t1 = term3();
                     if ((mode & TYPE) != 0 &&
-                        (S.token() == COMMA || S.token() == GT)) {
+                        (token.kind == COMMA || token.kind == GT)) {
                         mode = TYPE;
                         ListBuffer<JCExpression> args = new ListBuffer<JCExpression>();
                         args.append(t1);
-                        while (S.token() == COMMA) {
-                            S.nextToken();
+                        while (token.kind == COMMA) {
+                            nextToken();
                             args.append(typeArgument());
                         }
                         accept(GT);
                         t = toP(F.at(pos1).TypeApply(t, args.toList()));
                         checkGenerics();
-                        while (S.token() == DOT) {
-                            S.nextToken();
+                        while (token.kind == DOT) {
+                            nextToken();
                             mode = TYPE;
-                            t = toP(F.at(S.pos()).Select(t, ident()));
+                            t = toP(F.at(token.pos).Select(t, ident()));
                             t = typeArgumentsOpt(t);
                         }
                         t = bracketsOpt(toP(t));
@@ -948,7 +943,7 @@
                     JCExpression t1 = term3();
                     return F.at(pos).TypeCast(t, t1);
                 } else if ((lastmode & TYPE) != 0) {
-                    switch (S.token()) {
+                    switch (token.kind) {
                     /*case PLUSPLUS: case SUBSUB: */
                     case BANG: case TILDE:
                     case LPAREN: case THIS: case SUPER:
@@ -969,7 +964,7 @@
             if ((mode & EXPR) != 0) {
                 mode = EXPR;
                 t = to(F.at(pos).Ident(names._this));
-                S.nextToken();
+                nextToken();
                 if (typeArgs == null)
                     t = argumentsOpt(null, t);
                 else
@@ -997,22 +992,22 @@
             if (typeArgs != null) return illegal();
             if ((mode & EXPR) != 0) {
                 mode = EXPR;
-                S.nextToken();
-                if (S.token() == LT) typeArgs = typeArguments(false);
+                nextToken();
+                if (token.kind == LT) typeArgs = typeArguments(false);
                 t = creator(pos, typeArgs);
                 typeArgs = null;
             } else return illegal();
             break;
         case IDENTIFIER: case ASSERT: case ENUM:
             if (typeArgs != null) return illegal();
-            t = toP(F.at(S.pos()).Ident(ident()));
+            t = toP(F.at(token.pos).Ident(ident()));
             loop: while (true) {
-                pos = S.pos();
-                switch (S.token()) {
+                pos = token.pos;
+                switch (token.kind) {
                 case LBRACKET:
-                    S.nextToken();
-                    if (S.token() == RBRACKET) {
-                        S.nextToken();
+                    nextToken();
+                    if (token.kind == RBRACKET) {
+                        nextToken();
                         t = bracketsOpt(t);
                         t = toP(F.at(pos).TypeArray(t));
                         t = bracketsSuffix(t);
@@ -1033,24 +1028,24 @@
                     }
                     break loop;
                 case DOT:
-                    S.nextToken();
+                    nextToken();
                     int oldmode = mode;
                     mode &= ~NOPARAMS;
                     typeArgs = typeArgumentsOpt(EXPR);
                     mode = oldmode;
                     if ((mode & EXPR) != 0) {
-                        switch (S.token()) {
+                        switch (token.kind) {
                         case CLASS:
                             if (typeArgs != null) return illegal();
                             mode = EXPR;
                             t = to(F.at(pos).Select(t, names._class));
-                            S.nextToken();
+                            nextToken();
                             break loop;
                         case THIS:
                             if (typeArgs != null) return illegal();
                             mode = EXPR;
                             t = to(F.at(pos).Select(t, names._this));
-                            S.nextToken();
+                            nextToken();
                             break loop;
                         case SUPER:
                             mode = EXPR;
@@ -1061,9 +1056,9 @@
                         case NEW:
                             if (typeArgs != null) return illegal();
                             mode = EXPR;
-                            int pos1 = S.pos();
-                            S.nextToken();
-                            if (S.token() == LT) typeArgs = typeArguments(false);
+                            int pos1 = token.pos;
+                            nextToken();
+                            if (token.kind == LT) typeArgs = typeArguments(false);
                             t = innerCreator(pos1, typeArgs, t);
                             typeArgs = null;
                             break loop;
@@ -1087,8 +1082,8 @@
         case VOID:
             if (typeArgs != null) illegal();
             if ((mode & EXPR) != 0) {
-                S.nextToken();
-                if (S.token() == DOT) {
+                nextToken();
+                if (token.kind == DOT) {
                     JCPrimitiveTypeTree ti = toP(F.at(pos).TypeIdent(TypeTags.VOID));
                     t = bracketsSuffix(ti);
                 } else {
@@ -1099,7 +1094,7 @@
                 // a void type (like other primitive types) to the next phase.
                 // The error will be reported in Attr.attribTypes or Attr.visitApply.
                 JCPrimitiveTypeTree ti = to(F.at(pos).TypeIdent(TypeTags.VOID));
-                S.nextToken();
+                nextToken();
                 return ti;
                 //return illegal();
             }
@@ -1109,14 +1104,14 @@
         }
         if (typeArgs != null) illegal();
         while (true) {
-            int pos1 = S.pos();
-            if (S.token() == LBRACKET) {
-                S.nextToken();
+            int pos1 = token.pos;
+            if (token.kind == LBRACKET) {
+                nextToken();
                 if ((mode & TYPE) != 0) {
                     int oldmode = mode;
                     mode = TYPE;
-                    if (S.token() == RBRACKET) {
-                        S.nextToken();
+                    if (token.kind == RBRACKET) {
+                        nextToken();
                         t = bracketsOpt(t);
                         t = toP(F.at(pos1).TypeArray(t));
                         return t;
@@ -1129,21 +1124,21 @@
                     t = to(F.at(pos1).Indexed(t, t1));
                 }
                 accept(RBRACKET);
-            } else if (S.token() == DOT) {
-                S.nextToken();
+            } else if (token.kind == DOT) {
+                nextToken();
                 typeArgs = typeArgumentsOpt(EXPR);
-                if (S.token() == SUPER && (mode & EXPR) != 0) {
+                if (token.kind == SUPER && (mode & EXPR) != 0) {
                     mode = EXPR;
                     t = to(F.at(pos1).Select(t, names._super));
-                    S.nextToken();
+                    nextToken();
                     t = arguments(typeArgs, t);
                     typeArgs = null;
-                } else if (S.token() == NEW && (mode & EXPR) != 0) {
+                } else if (token.kind == NEW && (mode & EXPR) != 0) {
                     if (typeArgs != null) return illegal();
                     mode = EXPR;
-                    int pos2 = S.pos();
-                    S.nextToken();
-                    if (S.token() == LT) typeArgs = typeArguments(false);
+                    int pos2 = token.pos;
+                    nextToken();
+                    if (token.kind == LT) typeArgs = typeArguments(false);
                     t = innerCreator(pos2, typeArgs, t);
                     typeArgs = null;
                 } else {
@@ -1155,11 +1150,11 @@
                 break;
             }
         }
-        while ((S.token() == PLUSPLUS || S.token() == SUBSUB) && (mode & EXPR) != 0) {
+        while ((token.kind == PLUSPLUS || token.kind == SUBSUB) && (mode & EXPR) != 0) {
             mode = EXPR;
-            t = to(F.at(S.pos()).Unary(
-                  S.token() == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
-            S.nextToken();
+            t = to(F.at(token.pos).Unary(
+                  token.kind == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
+            nextToken();
         }
         return toP(t);
     }
@@ -1167,13 +1162,13 @@
     /** SuperSuffix = Arguments | "." [TypeArguments] Ident [Arguments]
      */
     JCExpression superSuffix(List<JCExpression> typeArgs, JCExpression t) {
-        S.nextToken();
-        if (S.token() == LPAREN || typeArgs != null) {
+        nextToken();
+        if (token.kind == LPAREN || typeArgs != null) {
             t = arguments(typeArgs, t);
         } else {
-            int pos = S.pos();
+            int pos = token.pos;
             accept(DOT);
-            typeArgs = (S.token() == LT) ? typeArguments(false) : null;
+            typeArgs = (token.kind == LT) ? typeArguments(false) : null;
             t = toP(F.at(pos).Select(t, ident()));
             t = argumentsOpt(typeArgs, t);
         }
@@ -1183,15 +1178,15 @@
     /** BasicType = BYTE | SHORT | CHAR | INT | LONG | FLOAT | DOUBLE | BOOLEAN
      */
     JCPrimitiveTypeTree basicType() {
-        JCPrimitiveTypeTree t = to(F.at(S.pos()).TypeIdent(typetag(S.token())));
-        S.nextToken();
+        JCPrimitiveTypeTree t = to(F.at(token.pos).TypeIdent(typetag(token.kind)));
+        nextToken();
         return t;
     }
 
     /** ArgumentsOpt = [ Arguments ]
      */
     JCExpression argumentsOpt(List<JCExpression> typeArgs, JCExpression t) {
-        if ((mode & EXPR) != 0 && S.token() == LPAREN || typeArgs != null) {
+        if ((mode & EXPR) != 0 && token.kind == LPAREN || typeArgs != null) {
             mode = EXPR;
             return arguments(typeArgs, t);
         } else {
@@ -1203,24 +1198,24 @@
      */
     List<JCExpression> arguments() {
         ListBuffer<JCExpression> args = lb();
-        if (S.token() == LPAREN) {
-            S.nextToken();
-            if (S.token() != RPAREN) {
+        if (token.kind == LPAREN) {
+            nextToken();
+            if (token.kind != RPAREN) {
                 args.append(parseExpression());
-                while (S.token() == COMMA) {
-                    S.nextToken();
+                while (token.kind == COMMA) {
+                    nextToken();
                     args.append(parseExpression());
                 }
             }
             accept(RPAREN);
         } else {
-            syntaxError(S.pos(), "expected", LPAREN);
+            syntaxError(token.pos, "expected", LPAREN);
         }
         return args.toList();
     }
 
     JCMethodInvocation arguments(List<JCExpression> typeArgs, JCExpression t) {
-        int pos = S.pos();
+        int pos = token.pos;
         List<JCExpression> args = arguments();
         return toP(F.at(pos).Apply(typeArgs, t, args));
     }
@@ -1228,7 +1223,7 @@
     /**  TypeArgumentsOpt = [ TypeArguments ]
      */
     JCExpression typeArgumentsOpt(JCExpression t) {
-        if (S.token() == LT &&
+        if (token.kind == LT &&
             (mode & TYPE) != 0 &&
             (mode & NOPARAMS) == 0) {
             mode = TYPE;
@@ -1243,7 +1238,7 @@
     }
 
     List<JCExpression> typeArgumentsOpt(int useMode) {
-        if (S.token() == LT) {
+        if (token.kind == LT) {
             checkGenerics();
             if ((mode & useMode) == 0 ||
                 (mode & NOPARAMS) != 0) {
@@ -1258,47 +1253,37 @@
     /**  TypeArguments  = "<" TypeArgument {"," TypeArgument} ">"
      */
     List<JCExpression> typeArguments(boolean diamondAllowed) {
-        if (S.token() == LT) {
-            S.nextToken();
-            if (S.token() == GT && diamondAllowed) {
+        if (token.kind == LT) {
+            nextToken();
+            if (token.kind == GT && diamondAllowed) {
                 checkDiamond();
                 mode |= DIAMOND;
-                S.nextToken();
+                nextToken();
                 return List.nil();
             } else {
                 ListBuffer<JCExpression> args = ListBuffer.lb();
                 args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
-                while (S.token() == COMMA) {
-                    S.nextToken();
+                while (token.kind == COMMA) {
+                    nextToken();
                     args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
                 }
-                switch (S.token()) {
-                case GTGTGTEQ:
-                    S.token(GTGTEQ);
-                    break;
-                case GTGTEQ:
-                    S.token(GTEQ);
-                    break;
-                case GTEQ:
-                    S.token(EQ);
-                    break;
-                case GTGTGT:
-                    S.token(GTGT);
-                    break;
-                case GTGT:
-                    S.token(GT);
+                switch (token.kind) {
+
+                case GTGTGTEQ: case GTGTEQ: case GTEQ:
+                case GTGTGT: case GTGT:
+                    token = S.split();
                     break;
                 case GT:
-                    S.nextToken();
+                    nextToken();
                     break;
                 default:
-                    args.append(syntaxError(S.pos(), "expected", GT));
+                    args.append(syntaxError(token.pos, "expected", GT));
                     break;
                 }
                 return args.toList();
             }
         } else {
-            return List.<JCExpression>of(syntaxError(S.pos(), "expected", LT));
+            return List.<JCExpression>of(syntaxError(token.pos, "expected", LT));
         }
     }
 
@@ -1308,24 +1293,24 @@
      *               | "?" SUPER Type
      */
     JCExpression typeArgument() {
-        if (S.token() != QUES) return parseType();
-        int pos = S.pos();
-        S.nextToken();
-        if (S.token() == EXTENDS) {
+        if (token.kind != QUES) return parseType();
+        int pos = token.pos;
+        nextToken();
+        if (token.kind == EXTENDS) {
             TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.EXTENDS));
-            S.nextToken();
+            nextToken();
             JCExpression bound = parseType();
             return F.at(pos).Wildcard(t, bound);
-        } else if (S.token() == SUPER) {
+        } else if (token.kind == SUPER) {
             TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.SUPER));
-            S.nextToken();
+            nextToken();
             JCExpression bound = parseType();
             return F.at(pos).Wildcard(t, bound);
-        } else if (S.token() == IDENTIFIER) {
+        } else if (token.kind == IDENTIFIER) {
             //error recovery
             TypeBoundKind t = F.at(Position.NOPOS).TypeBoundKind(BoundKind.UNBOUND);
             JCExpression wc = toP(F.at(pos).Wildcard(t, null));
-            JCIdent id = toP(F.at(S.pos()).Ident(ident()));
+            JCIdent id = toP(F.at(token.pos).Ident(ident()));
             JCErroneous err = F.at(pos).Erroneous(List.<JCTree>of(wc, id));
             reportSyntaxError(err, "expected3", GT, EXTENDS, SUPER);
             return err;
@@ -1336,7 +1321,7 @@
     }
 
     JCTypeApply typeArguments(JCExpression t, boolean diamondAllowed) {
-        int pos = S.pos();
+        int pos = token.pos;
         List<JCExpression> args = typeArguments(diamondAllowed);
         return toP(F.at(pos).TypeApply(t, args));
     }
@@ -1344,9 +1329,9 @@
     /** BracketsOpt = {"[" "]"}
      */
     private JCExpression bracketsOpt(JCExpression t) {
-        if (S.token() == LBRACKET) {
-            int pos = S.pos();
-            S.nextToken();
+        if (token.kind == LBRACKET) {
+            int pos = token.pos;
+            nextToken();
             t = bracketsOptCont(t, pos);
             F.at(pos);
         }
@@ -1363,17 +1348,17 @@
      *  BracketsSuffixType =
      */
     JCExpression bracketsSuffix(JCExpression t) {
-        if ((mode & EXPR) != 0 && S.token() == DOT) {
+        if ((mode & EXPR) != 0 && token.kind == DOT) {
             mode = EXPR;
-            int pos = S.pos();
-            S.nextToken();
+            int pos = token.pos;
+            nextToken();
             accept(CLASS);
-            if (S.pos() == errorEndPos) {
+            if (token.pos == errorEndPos) {
                 // error recovery
                 Name name = null;
-                if (S.token() == IDENTIFIER) {
-                    name = S.name();
-                    S.nextToken();
+                if (token.kind == IDENTIFIER) {
+                    name = token.name();
+                    nextToken();
                 } else {
                     name = names.error;
                 }
@@ -1384,7 +1369,7 @@
         } else if ((mode & TYPE) != 0) {
             mode = TYPE;
         } else {
-            syntaxError(S.pos(), "dot.class.expected");
+            syntaxError(token.pos, "dot.class.expected");
         }
         return t;
     }
@@ -1392,7 +1377,7 @@
     /** Creator = Qualident [TypeArguments] ( ArrayCreatorRest | ClassCreatorRest )
      */
     JCExpression creator(int newpos, List<JCExpression> typeArgs) {
-        switch (S.token()) {
+        switch (token.kind) {
         case BYTE: case SHORT: case CHAR: case INT: case LONG: case FLOAT:
         case DOUBLE: case BOOLEAN:
             if (typeArgs == null)
@@ -1405,29 +1390,29 @@
         mode = TYPE;
         boolean diamondFound = false;
         int lastTypeargsPos = -1;
-        if (S.token() == LT) {
+        if (token.kind == LT) {
             checkGenerics();
-            lastTypeargsPos = S.pos();
+            lastTypeargsPos = token.pos;
             t = typeArguments(t, true);
             diamondFound = (mode & DIAMOND) != 0;
         }
-        while (S.token() == DOT) {
+        while (token.kind == DOT) {
             if (diamondFound) {
                 //cannot select after a diamond
                 illegal();
             }
-            int pos = S.pos();
-            S.nextToken();
+            int pos = token.pos;
+            nextToken();
             t = toP(F.at(pos).Select(t, ident()));
-            if (S.token() == LT) {
-                lastTypeargsPos = S.pos();
+            if (token.kind == LT) {
+                lastTypeargsPos = token.pos;
                 checkGenerics();
                 t = typeArguments(t, true);
                 diamondFound = (mode & DIAMOND) != 0;
             }
         }
         mode = oldmode;
-        if (S.token() == LBRACKET) {
+        if (token.kind == LBRACKET) {
             JCExpression e = arrayCreatorRest(newpos, t);
             if (diamondFound) {
                 reportSyntaxError(lastTypeargsPos, "cannot.create.array.with.diamond");
@@ -1441,17 +1426,17 @@
                     // modified to improve error recovery.
                     pos = typeArgs.head.pos;
                 }
-                setErrorEndPos(S.prevEndPos());
+                setErrorEndPos(S.prevToken().endPos);
                 JCErroneous err = F.at(pos).Erroneous(typeArgs.prepend(e));
                 reportSyntaxError(err, "cannot.create.array.with.type.arguments");
                 return toP(err);
             }
             return e;
-        } else if (S.token() == LPAREN) {
+        } else if (token.kind == LPAREN) {
             return classCreatorRest(newpos, null, typeArgs, t);
         } else {
-            setErrorEndPos(S.pos());
-            reportSyntaxError(S.pos(), "expected2", LPAREN, LBRACKET);
+            setErrorEndPos(token.pos);
+            reportSyntaxError(token.pos, "expected2", LPAREN, LBRACKET);
             t = toP(F.at(newpos).NewClass(null, typeArgs, t, List.<JCExpression>nil(), null));
             return toP(F.at(newpos).Erroneous(List.<JCTree>of(t)));
         }
@@ -1460,8 +1445,8 @@
     /** InnerCreator = Ident [TypeArguments] ClassCreatorRest
      */
     JCExpression innerCreator(int newpos, List<JCExpression> typeArgs, JCExpression encl) {
-        JCExpression t = toP(F.at(S.pos()).Ident(ident()));
-        if (S.token() == LT) {
+        JCExpression t = toP(F.at(token.pos).Ident(ident()));
+        if (token.kind == LT) {
             int oldmode = mode;
             checkGenerics();
             t = typeArguments(t, true);
@@ -1475,23 +1460,23 @@
      */
     JCExpression arrayCreatorRest(int newpos, JCExpression elemtype) {
         accept(LBRACKET);
-        if (S.token() == RBRACKET) {
+        if (token.kind == RBRACKET) {
             accept(RBRACKET);
             elemtype = bracketsOpt(elemtype);
-            if (S.token() == LBRACE) {
+            if (token.kind == LBRACE) {
                 return arrayInitializer(newpos, elemtype);
             } else {
                 JCExpression t = toP(F.at(newpos).NewArray(elemtype, List.<JCExpression>nil(), null));
-                return syntaxError(S.pos(), List.<JCTree>of(t), "array.dimension.missing");
+                return syntaxError(token.pos, List.<JCTree>of(t), "array.dimension.missing");
             }
         } else {
             ListBuffer<JCExpression> dims = new ListBuffer<JCExpression>();
             dims.append(parseExpression());
             accept(RBRACKET);
-            while (S.token() == LBRACKET) {
-                int pos = S.pos();
-                S.nextToken();
-                if (S.token() == RBRACKET) {
+            while (token.kind == LBRACKET) {
+                int pos = token.pos;
+                nextToken();
+                if (token.kind == RBRACKET) {
                     elemtype = bracketsOptCont(elemtype, pos);
                 } else {
                     dims.append(parseExpression());
@@ -1511,8 +1496,8 @@
     {
         List<JCExpression> args = arguments();
         JCClassDecl body = null;
-        if (S.token() == LBRACE) {
-            int pos = S.pos();
+        if (token.kind == LBRACE) {
+            int pos = token.pos;
             List<JCTree> defs = classOrInterfaceBody(names.empty, false);
             JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
             body = toP(F.at(pos).AnonymousClassDef(mods, defs));
@@ -1525,13 +1510,13 @@
     JCExpression arrayInitializer(int newpos, JCExpression t) {
         accept(LBRACE);
         ListBuffer<JCExpression> elems = new ListBuffer<JCExpression>();
-        if (S.token() == COMMA) {
-            S.nextToken();
-        } else if (S.token() != RBRACE) {
+        if (token.kind == COMMA) {
+            nextToken();
+        } else if (token.kind != RBRACE) {
             elems.append(variableInitializer());
-            while (S.token() == COMMA) {
-                S.nextToken();
-                if (S.token() == RBRACE) break;
+            while (token.kind == COMMA) {
+                nextToken();
+                if (token.kind == RBRACE) break;
                 elems.append(variableInitializer());
             }
         }
@@ -1542,7 +1527,7 @@
     /** VariableInitializer = ArrayInitializer | Expression
      */
     public JCExpression variableInitializer() {
-        return S.token() == LBRACE ? arrayInitializer(S.pos(), null) : parseExpression();
+        return token.kind == LBRACE ? arrayInitializer(token.pos, null) : parseExpression();
     }
 
     /** ParExpression = "(" Expression ")"
@@ -1560,19 +1545,19 @@
         accept(LBRACE);
         List<JCStatement> stats = blockStatements();
         JCBlock t = F.at(pos).Block(flags, stats);
-        while (S.token() == CASE || S.token() == DEFAULT) {
-            syntaxError("orphaned", S.token());
+        while (token.kind == CASE || token.kind == DEFAULT) {
+            syntaxError("orphaned", token.kind);
             switchBlockStatementGroups();
         }
         // the Block node has a field "endpos" for first char of last token, which is
         // usually but not necessarily the last char of the last token.
-        t.endpos = S.pos();
+        t.endpos = token.pos;
         accept(RBRACE);
         return toP(t);
     }
 
     public JCBlock block() {
-        return block(S.pos(), 0);
+        return block(token.pos, 0);
     }
 
     /** BlockStatements = { BlockStatement }
@@ -1588,8 +1573,8 @@
         int lastErrPos = -1;
         ListBuffer<JCStatement> stats = new ListBuffer<JCStatement>();
         while (true) {
-            int pos = S.pos();
-            switch (S.token()) {
+            int pos = token.pos;
+            switch (token.kind) {
             case RBRACE: case CASE: case DEFAULT: case EOF:
                 return stats.toList();
             case LBRACE: case IF: case FOR: case WHILE: case DO: case TRY:
@@ -1599,64 +1584,63 @@
                 break;
             case MONKEYS_AT:
             case FINAL: {
-                String dc = S.docComment();
+                String dc = token.docComment;
                 JCModifiers mods = modifiersOpt();
-                if (S.token() == INTERFACE ||
-                    S.token() == CLASS ||
-                    allowEnums && S.token() == ENUM) {
+                if (token.kind == INTERFACE ||
+                    token.kind == CLASS ||
+                    allowEnums && token.kind == ENUM) {
                     stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
                 } else {
                     JCExpression t = parseType();
                     stats.appendList(variableDeclarators(mods, t,
                                                          new ListBuffer<JCStatement>()));
                     // A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
-                    storeEnd(stats.elems.last(), S.endPos());
+                    storeEnd(stats.elems.last(), token.endPos);
                     accept(SEMI);
                 }
                 break;
             }
             case ABSTRACT: case STRICTFP: {
-                String dc = S.docComment();
+                String dc = token.docComment;
                 JCModifiers mods = modifiersOpt();
                 stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
                 break;
             }
             case INTERFACE:
             case CLASS:
-                stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
-                                                               S.docComment()));
+                String dc = token.docComment;
+                stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
                 break;
             case ENUM:
             case ASSERT:
-                if (allowEnums && S.token() == ENUM) {
-                    error(S.pos(), "local.enum");
-                    stats.
-                        append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
-                                                                 S.docComment()));
+                if (allowEnums && token.kind == ENUM) {
+                    error(token.pos, "local.enum");
+                    dc = token.docComment;
+                    stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
                     break;
-                } else if (allowAsserts && S.token() == ASSERT) {
+                } else if (allowAsserts && token.kind == ASSERT) {
                     stats.append(parseStatement());
                     break;
                 }
                 /* fall through to default */
             default:
-                Name name = S.name();
+                Token prevToken = token;
                 JCExpression t = term(EXPR | TYPE);
-                if (S.token() == COLON && t.getTag() == JCTree.IDENT) {
-                    S.nextToken();
+                if (token.kind == COLON && t.getTag() == JCTree.IDENT) {
+                    nextToken();
                     JCStatement stat = parseStatement();
-                    stats.append(F.at(pos).Labelled(name, stat));
+                    stats.append(F.at(pos).Labelled(prevToken.name(), stat));
                 } else if ((lastmode & TYPE) != 0 &&
-                           (S.token() == IDENTIFIER ||
-                            S.token() == ASSERT ||
-                            S.token() == ENUM)) {
-                    pos = S.pos();
+                           (token.kind == IDENTIFIER ||
+                            token.kind == ASSERT ||
+                            token.kind == ENUM)) {
+                    pos = token.pos;
                     JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
                     F.at(pos);
                     stats.appendList(variableDeclarators(mods, t,
                                                          new ListBuffer<JCStatement>()));
                     // A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
-                    storeEnd(stats.elems.last(), S.endPos());
+                    storeEnd(stats.elems.last(), token.endPos);
                     accept(SEMI);
                 } else {
                     // This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
@@ -1666,15 +1650,12 @@
             }
 
             // error recovery
-            if (S.pos() == lastErrPos)
+            if (token.pos == lastErrPos)
                 return stats.toList();
-            if (S.pos() <= errorEndPos) {
+            if (token.pos <= errorEndPos) {
                 skip(false, true, true, true);
-                lastErrPos = S.pos();
+                lastErrPos = token.pos;
             }
-
-            // ensure no dangling /** @deprecated */ active
-            S.resetDeprecatedFlag();
         }
     }
 
@@ -1700,29 +1681,29 @@
      */
     @SuppressWarnings("fallthrough")
     public JCStatement parseStatement() {
-        int pos = S.pos();
-        switch (S.token()) {
+        int pos = token.pos;
+        switch (token.kind) {
         case LBRACE:
             return block();
         case IF: {
-            S.nextToken();
+            nextToken();
             JCExpression cond = parExpression();
             JCStatement thenpart = parseStatement();
             JCStatement elsepart = null;
-            if (S.token() == ELSE) {
-                S.nextToken();
+            if (token.kind == ELSE) {
+                nextToken();
                 elsepart = parseStatement();
             }
             return F.at(pos).If(cond, thenpart, elsepart);
         }
         case FOR: {
-            S.nextToken();
+            nextToken();
             accept(LPAREN);
-            List<JCStatement> inits = S.token() == SEMI ? List.<JCStatement>nil() : forInit();
+            List<JCStatement> inits = token.kind == SEMI ? List.<JCStatement>nil() : forInit();
             if (inits.length() == 1 &&
                 inits.head.getTag() == JCTree.VARDEF &&
                 ((JCVariableDecl) inits.head).init == null &&
-                S.token() == COLON) {
+                token.kind == COLON) {
                 checkForeach();
                 JCVariableDecl var = (JCVariableDecl)inits.head;
                 accept(COLON);
@@ -1732,22 +1713,22 @@
                 return F.at(pos).ForeachLoop(var, expr, body);
             } else {
                 accept(SEMI);
-                JCExpression cond = S.token() == SEMI ? null : parseExpression();
+                JCExpression cond = token.kind == SEMI ? null : parseExpression();
                 accept(SEMI);
-                List<JCExpressionStatement> steps = S.token() == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
+                List<JCExpressionStatement> steps = token.kind == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
                 accept(RPAREN);
                 JCStatement body = parseStatement();
                 return F.at(pos).ForLoop(inits, cond, steps, body);
             }
         }
         case WHILE: {
-            S.nextToken();
+            nextToken();
             JCExpression cond = parExpression();
             JCStatement body = parseStatement();
             return F.at(pos).WhileLoop(cond, body);
         }
         case DO: {
-            S.nextToken();
+            nextToken();
             JCStatement body = parseStatement();
             accept(WHILE);
             JCExpression cond = parExpression();
@@ -1756,21 +1737,21 @@
             return t;
         }
         case TRY: {
-            S.nextToken();
+            nextToken();
             List<JCTree> resources = List.<JCTree>nil();
-            if (S.token() == LPAREN) {
+            if (token.kind == LPAREN) {
                 checkTryWithResources();
-                S.nextToken();
+                nextToken();
                 resources = resources();
                 accept(RPAREN);
             }
             JCBlock body = block();
             ListBuffer<JCCatch> catchers = new ListBuffer<JCCatch>();
             JCBlock finalizer = null;
-            if (S.token() == CATCH || S.token() == FINALLY) {
-                while (S.token() == CATCH) catchers.append(catchClause());
-                if (S.token() == FINALLY) {
-                    S.nextToken();
+            if (token.kind == CATCH || token.kind == FINALLY) {
+                while (token.kind == CATCH) catchers.append(catchClause());
+                if (token.kind == FINALLY) {
+                    nextToken();
                     finalizer = block();
                 }
             } else {
@@ -1783,7 +1764,7 @@
             return F.at(pos).Try(resources, body, catchers.toList(), finalizer);
         }
         case SWITCH: {
-            S.nextToken();
+            nextToken();
             JCExpression selector = parExpression();
             accept(LBRACE);
             List<JCCase> cases = switchBlockStatementGroups();
@@ -1792,41 +1773,41 @@
             return t;
         }
         case SYNCHRONIZED: {
-            S.nextToken();
+            nextToken();
             JCExpression lock = parExpression();
             JCBlock body = block();
             return F.at(pos).Synchronized(lock, body);
         }
         case RETURN: {
-            S.nextToken();
-            JCExpression result = S.token() == SEMI ? null : parseExpression();
+            nextToken();
+            JCExpression result = token.kind == SEMI ? null : parseExpression();
             JCReturn t = to(F.at(pos).Return(result));
             accept(SEMI);
             return t;
         }
         case THROW: {
-            S.nextToken();
+            nextToken();
             JCExpression exc = parseExpression();
             JCThrow t = to(F.at(pos).Throw(exc));
             accept(SEMI);
             return t;
         }
         case BREAK: {
-            S.nextToken();
-            Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
+            nextToken();
+            Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
             JCBreak t = to(F.at(pos).Break(label));
             accept(SEMI);
             return t;
         }
         case CONTINUE: {
-            S.nextToken();
-            Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
+            nextToken();
+            Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
             JCContinue t =  to(F.at(pos).Continue(label));
             accept(SEMI);
             return t;
         }
         case SEMI:
-            S.nextToken();
+            nextToken();
             return toP(F.at(pos).Skip());
         case ELSE:
             return toP(F.Exec(syntaxError("else.without.if")));
@@ -1835,12 +1816,12 @@
         case CATCH:
             return toP(F.Exec(syntaxError("catch.without.try")));
         case ASSERT: {
-            if (allowAsserts && S.token() == ASSERT) {
-                S.nextToken();
+            if (allowAsserts && token.kind == ASSERT) {
+                nextToken();
                 JCExpression assertion = parseExpression();
                 JCExpression message = null;
-                if (S.token() == COLON) {
-                    S.nextToken();
+                if (token.kind == COLON) {
+                    nextToken();
                     message = parseExpression();
                 }
                 JCAssert t = to(F.at(pos).Assert(assertion, message));
@@ -1851,12 +1832,12 @@
         }
         case ENUM:
         default:
-            Name name = S.name();
+            Token prevToken = token;
             JCExpression expr = parseExpression();
-            if (S.token() == COLON && expr.getTag() == JCTree.IDENT) {
-                S.nextToken();
+            if (token.kind == COLON && expr.getTag() == JCTree.IDENT) {
+                nextToken();
                 JCStatement stat = parseStatement();
-                return F.at(pos).Labelled(name, stat);
+                return F.at(pos).Labelled(prevToken.name(), stat);
             } else {
                 // This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
                 JCExpressionStatement stat = to(F.at(pos).Exec(checkExprStat(expr)));
@@ -1869,7 +1850,7 @@
     /** CatchClause     = CATCH "(" FormalParameter ")" Block
      */
     protected JCCatch catchClause() {
-        int pos = S.pos();
+        int pos = token.pos;
         accept(CATCH);
         accept(LPAREN);
         JCModifiers mods = optFinal(Flags.PARAMETER);
@@ -1886,9 +1867,9 @@
     List<JCExpression> catchTypes() {
         ListBuffer<JCExpression> catchTypes = ListBuffer.lb();
         catchTypes.add(parseType());
-        while (S.token() == BAR) {
+        while (token.kind == BAR) {
             checkMulticatch();
-            S.nextToken();
+            nextToken();
             catchTypes.add(qualident());
         }
         return catchTypes.toList();
@@ -1901,33 +1882,33 @@
     List<JCCase> switchBlockStatementGroups() {
         ListBuffer<JCCase> cases = new ListBuffer<JCCase>();
         while (true) {
-            int pos = S.pos();
-            switch (S.token()) {
+            int pos = token.pos;
+            switch (token.kind) {
             case CASE: {
-                S.nextToken();
+                nextToken();
                 JCExpression pat = parseExpression();
                 accept(COLON);
                 List<JCStatement> stats = blockStatements();
                 JCCase c = F.at(pos).Case(pat, stats);
                 if (stats.isEmpty())
-                    storeEnd(c, S.prevEndPos());
+                    storeEnd(c, S.prevToken().endPos);
                 cases.append(c);
                 break;
             }
             case DEFAULT: {
-                S.nextToken();
+                nextToken();
                 accept(COLON);
                 List<JCStatement> stats = blockStatements();
                 JCCase c = F.at(pos).Case(null, stats);
                 if (stats.isEmpty())
-                    storeEnd(c, S.prevEndPos());
+                    storeEnd(c, S.prevToken().endPos);
                 cases.append(c);
                 break;
             }
             case RBRACE: case EOF:
                 return cases.toList();
             default:
-                S.nextToken(); // to ensure progress
+                nextToken(); // to ensure progress
                 syntaxError(pos, "expected3",
                     CASE, DEFAULT, RBRACE);
             }
@@ -1941,9 +1922,9 @@
                                                                     T stats) {
         // This Exec is a "StatementExpression"; it subsumes no terminating token
         stats.append(toP(F.at(pos).Exec(checkExprStat(first))));
-        while (S.token() == COMMA) {
-            S.nextToken();
-            pos = S.pos();
+        while (token.kind == COMMA) {
+            nextToken();
+            pos = token.pos;
             JCExpression t = parseExpression();
             // This Exec is a "StatementExpression"; it subsumes no terminating token
             stats.append(toP(F.at(pos).Exec(checkExprStat(t))));
@@ -1956,13 +1937,13 @@
      */
     List<JCStatement> forInit() {
         ListBuffer<JCStatement> stats = lb();
-        int pos = S.pos();
-        if (S.token() == FINAL || S.token() == MONKEYS_AT) {
+        int pos = token.pos;
+        if (token.kind == FINAL || token.kind == MONKEYS_AT) {
             return variableDeclarators(optFinal(0), parseType(), stats).toList();
         } else {
             JCExpression t = term(EXPR | TYPE);
             if ((lastmode & TYPE) != 0 &&
-                (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM))
+                (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM))
                 return variableDeclarators(modifiersOpt(), t, stats).toList();
             else
                 return moreStatementExpressions(pos, t, stats).toList();
@@ -1972,7 +1953,7 @@
     /** ForUpdate = StatementExpression MoreStatementExpressions
      */
     List<JCExpressionStatement> forUpdate() {
-        return moreStatementExpressions(S.pos(),
+        return moreStatementExpressions(token.pos,
                                         parseExpression(),
                                         new ListBuffer<JCExpressionStatement>()).toList();
     }
@@ -1980,11 +1961,11 @@
     /** AnnotationsOpt = { '@' Annotation }
      */
     List<JCAnnotation> annotationsOpt() {
-        if (S.token() != MONKEYS_AT) return List.nil(); // optimization
+        if (token.kind != MONKEYS_AT) return List.nil(); // optimization
         ListBuffer<JCAnnotation> buf = new ListBuffer<JCAnnotation>();
-        while (S.token() == MONKEYS_AT) {
-            int pos = S.pos();
-            S.nextToken();
+        while (token.kind == MONKEYS_AT) {
+            int pos = token.pos;
+            nextToken();
             buf.append(annotation(pos));
         }
         return buf.toList();
@@ -2004,21 +1985,20 @@
         int pos;
         if (partial == null) {
             flags = 0;
-            pos = S.pos();
+            pos = token.pos;
         } else {
             flags = partial.flags;
             annotations.appendList(partial.annotations);
             pos = partial.pos;
         }
-        if (S.deprecatedFlag()) {
+        if (token.deprecatedFlag) {
             flags |= Flags.DEPRECATED;
-            S.resetDeprecatedFlag();
         }
         int lastPos = Position.NOPOS;
     loop:
         while (true) {
             long flag;
-            switch (S.token()) {
+            switch (token.kind) {
             case PRIVATE     : flag = Flags.PRIVATE; break;
             case PROTECTED   : flag = Flags.PROTECTED; break;
             case PUBLIC      : flag = Flags.PUBLIC; break;
@@ -2031,15 +2011,15 @@
             case SYNCHRONIZED: flag = Flags.SYNCHRONIZED; break;
             case STRICTFP    : flag = Flags.STRICTFP; break;
             case MONKEYS_AT  : flag = Flags.ANNOTATION; break;
-            case ERROR       : flag = 0; S.nextToken(); break;
+            case ERROR       : flag = 0; nextToken(); break;
             default: break loop;
             }
-            if ((flags & flag) != 0) error(S.pos(), "repeated.modifier");
-            lastPos = S.pos();
-            S.nextToken();
+            if ((flags & flag) != 0) error(token.pos, "repeated.modifier");
+            lastPos = token.pos;
+            nextToken();
             if (flag == Flags.ANNOTATION) {
                 checkAnnotations();
-                if (S.token() != INTERFACE) {
+                if (token.kind != INTERFACE) {
                     JCAnnotation ann = annotation(lastPos);
                     // if first modifier is an annotation, set pos to annotation's.
                     if (flags == 0 && annotations.isEmpty())
@@ -2051,7 +2031,7 @@
             }
             flags |= flag;
         }
-        switch (S.token()) {
+        switch (token.kind) {
         case ENUM: flags |= Flags.ENUM; break;
         case INTERFACE: flags |= Flags.INTERFACE; break;
         default: break;
@@ -2064,7 +2044,7 @@
 
         JCModifiers mods = F.at(pos).Modifiers(flags, annotations.toList());
         if (pos != Position.NOPOS)
-            storeEnd(mods, S.prevEndPos());
+            storeEnd(mods, S.prevToken().endPos);
         return mods;
     }
 
@@ -2077,22 +2057,22 @@
         JCTree ident = qualident();
         List<JCExpression> fieldValues = annotationFieldValuesOpt();
         JCAnnotation ann = F.at(pos).Annotation(ident, fieldValues);
-        storeEnd(ann, S.prevEndPos());
+        storeEnd(ann, S.prevToken().endPos);
         return ann;
     }
 
     List<JCExpression> annotationFieldValuesOpt() {
-        return (S.token() == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
+        return (token.kind == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
     }
 
     /** AnnotationFieldValues   = "(" [ AnnotationFieldValue { "," AnnotationFieldValue } ] ")" */
     List<JCExpression> annotationFieldValues() {
         accept(LPAREN);
         ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
-        if (S.token() != RPAREN) {
+        if (token.kind != RPAREN) {
             buf.append(annotationFieldValue());
-            while (S.token() == COMMA) {
-                S.nextToken();
+            while (token.kind == COMMA) {
+                nextToken();
                 buf.append(annotationFieldValue());
             }
         }
@@ -2104,11 +2084,11 @@
      *                          | Identifier "=" AnnotationValue
      */
     JCExpression annotationFieldValue() {
-        if (S.token() == IDENTIFIER) {
+        if (token.kind == IDENTIFIER) {
             mode = EXPR;
             JCExpression t1 = term1();
-            if (t1.getTag() == JCTree.IDENT && S.token() == EQ) {
-                int pos = S.pos();
+            if (t1.getTag() == JCTree.IDENT && token.kind == EQ) {
+                int pos = token.pos;
                 accept(EQ);
                 JCExpression v = annotationValue();
                 return toP(F.at(pos).Assign(t1, v));
@@ -2125,20 +2105,20 @@
      */
     JCExpression annotationValue() {
         int pos;
-        switch (S.token()) {
+        switch (token.kind) {
         case MONKEYS_AT:
-            pos = S.pos();
-            S.nextToken();
+            pos = token.pos;
+            nextToken();
             return annotation(pos);
         case LBRACE:
-            pos = S.pos();
+            pos = token.pos;
             accept(LBRACE);
             ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
-            if (S.token() != RBRACE) {
+            if (token.kind != RBRACE) {
                 buf.append(annotationValue());
-                while (S.token() == COMMA) {
-                    S.nextToken();
-                    if (S.token() == RBRACE) break;
+                while (token.kind == COMMA) {
+                    nextToken();
+                    if (token.kind == RBRACE) break;
                     buf.append(annotationValue());
                 }
             }
@@ -2156,7 +2136,7 @@
                                                                          JCExpression type,
                                                                          T vdefs)
     {
-        return variableDeclaratorsRest(S.pos(), mods, type, ident(), false, null, vdefs);
+        return variableDeclaratorsRest(token.pos, mods, type, ident(), false, null, vdefs);
     }
 
     /** VariableDeclaratorsRest = VariableDeclaratorRest { "," VariableDeclarator }
@@ -2174,10 +2154,10 @@
                                                                      T vdefs)
     {
         vdefs.append(variableDeclaratorRest(pos, mods, type, name, reqInit, dc));
-        while (S.token() == COMMA) {
+        while (token.kind == COMMA) {
             // All but last of multiple declarators subsume a comma
-            storeEnd((JCTree)vdefs.elems.last(), S.endPos());
-            S.nextToken();
+            storeEnd((JCTree)vdefs.elems.last(), token.endPos);
+            nextToken();
             vdefs.append(variableDeclarator(mods, type, reqInit, dc));
         }
         return vdefs;
@@ -2187,7 +2167,7 @@
      *  ConstantDeclarator = Ident ConstantDeclaratorRest
      */
     JCVariableDecl variableDeclarator(JCModifiers mods, JCExpression type, boolean reqInit, String dc) {
-        return variableDeclaratorRest(S.pos(), mods, type, ident(), reqInit, dc);
+        return variableDeclaratorRest(token.pos, mods, type, ident(), reqInit, dc);
     }
 
     /** VariableDeclaratorRest = BracketsOpt ["=" VariableInitializer]
@@ -2200,11 +2180,11 @@
                                   boolean reqInit, String dc) {
         type = bracketsOpt(type);
         JCExpression init = null;
-        if (S.token() == EQ) {
-            S.nextToken();
+        if (token.kind == EQ) {
+            nextToken();
             init = variableInitializer();
         }
-        else if (reqInit) syntaxError(S.pos(), "expected", EQ);
+        else if (reqInit) syntaxError(token.pos, "expected", EQ);
         JCVariableDecl result =
             toP(F.at(pos).VarDef(mods, name, type, init));
         attach(result, dc);
@@ -2214,11 +2194,11 @@
     /** VariableDeclaratorId = Ident BracketsOpt
      */
     JCVariableDecl variableDeclaratorId(JCModifiers mods, JCExpression type) {
-        int pos = S.pos();
+        int pos = token.pos;
         Name name = ident();
         if ((mods.flags & Flags.VARARGS) != 0 &&
-                S.token() == LBRACKET) {
-            log.error(S.pos(), "varargs.and.old.array.syntax");
+                token.kind == LBRACKET) {
+            log.error(token.pos, "varargs.and.old.array.syntax");
         }
         type = bracketsOpt(type);
         return toP(F.at(pos).VarDef(mods, name, type, null));
@@ -2229,12 +2209,12 @@
     List<JCTree> resources() {
         ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
         defs.append(resource());
-        while (S.token() == SEMI) {
+        while (token.kind == SEMI) {
             // All but last of multiple declarators must subsume a semicolon
-            storeEnd(defs.elems.last(), S.endPos());
-            int semiColonPos = S.pos();
-            S.nextToken();
-            if (S.token() == RPAREN) { // Optional trailing semicolon
+            storeEnd(defs.elems.last(), token.endPos);
+            int semiColonPos = token.pos;
+            nextToken();
+            if (token.kind == RPAREN) { // Optional trailing semicolon
                                        // after last resource
                 break;
             }
@@ -2248,7 +2228,7 @@
     protected JCTree resource() {
         JCModifiers optFinal = optFinal(Flags.FINAL);
         JCExpression type = parseType();
-        int pos = S.pos();
+        int pos = token.pos;
         Name ident = ident();
         return variableDeclaratorRest(pos, optFinal, type, ident, true, null);
     }
@@ -2256,54 +2236,61 @@
     /** CompilationUnit = [ { "@" Annotation } PACKAGE Qualident ";"] {ImportDeclaration} {TypeDeclaration}
      */
     public JCTree.JCCompilationUnit parseCompilationUnit() {
-        int pos = S.pos();
+        Token firstToken = token;
         JCExpression pid = null;
-        String dc = S.docComment();
         JCModifiers mods = null;
+        boolean consumedToplevelDoc = false;
+        boolean seenImport = false;
+        boolean seenPackage = false;
         List<JCAnnotation> packageAnnotations = List.nil();
-        if (S.token() == MONKEYS_AT)
+        if (token.kind == MONKEYS_AT)
             mods = modifiersOpt();
 
-        if (S.token() == PACKAGE) {
+        if (token.kind == PACKAGE) {
+            seenPackage = true;
             if (mods != null) {
                 checkNoMods(mods.flags);
                 packageAnnotations = mods.annotations;
                 mods = null;
             }
-            S.nextToken();
+            nextToken();
             pid = qualident();
             accept(SEMI);
         }
         ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
         boolean checkForImports = true;
-        while (S.token() != EOF) {
-            if (S.pos() <= errorEndPos) {
+        boolean firstTypeDecl = true;
+        while (token.kind != EOF) {
+            if (token.pos <= errorEndPos) {
                 // error recovery
                 skip(checkForImports, false, false, false);
-                if (S.token() == EOF)
+                if (token.kind == EOF)
                     break;
             }
-            if (checkForImports && mods == null && S.token() == IMPORT) {
+            if (checkForImports && mods == null && token.kind == IMPORT) {
+                seenImport = true;
                 defs.append(importDeclaration());
             } else {
-                JCTree def = typeDeclaration(mods);
-                if (keepDocComments && dc != null && docComments.get(def) == dc) {
-                    // If the first type declaration has consumed the first doc
-                    // comment, then don't use it for the top level comment as well.
-                    dc = null;
+                String docComment = token.docComment;
+                if (firstTypeDecl && !seenImport && !seenPackage) {
+                    docComment = firstToken.docComment;
+                    consumedToplevelDoc = true;
                 }
+                JCTree def = typeDeclaration(mods, docComment);
                 if (def instanceof JCExpressionStatement)
                     def = ((JCExpressionStatement)def).expr;
                 defs.append(def);
                 if (def instanceof JCClassDecl)
                     checkForImports = false;
                 mods = null;
+                firstTypeDecl = false;
             }
         }
-        JCTree.JCCompilationUnit toplevel = F.at(pos).TopLevel(packageAnnotations, pid, defs.toList());
-        attach(toplevel, dc);
+        JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList());
+        if (!consumedToplevelDoc)
+            attach(toplevel, firstToken.docComment);
         if (defs.elems.isEmpty())
-            storeEnd(toplevel, S.prevEndPos());
+            storeEnd(toplevel, S.prevToken().endPos);
         if (keepDocComments)
             toplevel.docComments = docComments;
         if (keepLineMap)
@@ -2314,26 +2301,26 @@
     /** ImportDeclaration = IMPORT [ STATIC ] Ident { "." Ident } [ "." "*" ] ";"
      */
     JCTree importDeclaration() {
-        int pos = S.pos();
-        S.nextToken();
+        int pos = token.pos;
+        nextToken();
         boolean importStatic = false;
-        if (S.token() == STATIC) {
+        if (token.kind == STATIC) {
             checkStaticImports();
             importStatic = true;
-            S.nextToken();
+            nextToken();
         }
-        JCExpression pid = toP(F.at(S.pos()).Ident(ident()));
+        JCExpression pid = toP(F.at(token.pos).Ident(ident()));
         do {
-            int pos1 = S.pos();
+            int pos1 = token.pos;
             accept(DOT);
-            if (S.token() == STAR) {
+            if (token.kind == STAR) {
                 pid = to(F.at(pos1).Select(pid, names.asterisk));
-                S.nextToken();
+                nextToken();
                 break;
             } else {
                 pid = toP(F.at(pos1).Select(pid, ident()));
             }
-        } while (S.token() == DOT);
+        } while (token.kind == DOT);
         accept(SEMI);
         return toP(F.at(pos).Import(pid, importStatic));
     }
@@ -2341,14 +2328,13 @@
     /** TypeDeclaration = ClassOrInterfaceOrEnumDeclaration
      *                  | ";"
      */
-    JCTree typeDeclaration(JCModifiers mods) {
-        int pos = S.pos();
-        if (mods == null && S.token() == SEMI) {
-            S.nextToken();
+    JCTree typeDeclaration(JCModifiers mods, String docComment) {
+        int pos = token.pos;
+        if (mods == null && token.kind == SEMI) {
+            nextToken();
             return toP(F.at(pos).Skip());
         } else {
-            String dc = S.docComment();
-            return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), dc);
+            return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), docComment);
         }
     }
 
@@ -2358,19 +2344,19 @@
      *  @param dc       The documentation comment for the class, or null.
      */
     JCStatement classOrInterfaceOrEnumDeclaration(JCModifiers mods, String dc) {
-        if (S.token() == CLASS) {
+        if (token.kind == CLASS) {
             return classDeclaration(mods, dc);
-        } else if (S.token() == INTERFACE) {
+        } else if (token.kind == INTERFACE) {
             return interfaceDeclaration(mods, dc);
         } else if (allowEnums) {
-            if (S.token() == ENUM) {
+            if (token.kind == ENUM) {
                 return enumDeclaration(mods, dc);
             } else {
-                int pos = S.pos();
+                int pos = token.pos;
                 List<JCTree> errs;
-                if (S.token() == IDENTIFIER) {
+                if (token.kind == IDENTIFIER) {
                     errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
-                    setErrorEndPos(S.pos());
+                    setErrorEndPos(token.pos);
                 } else {
                     errs = List.<JCTree>of(mods);
                 }
@@ -2378,16 +2364,16 @@
                                               CLASS, INTERFACE, ENUM)));
             }
         } else {
-            if (S.token() == ENUM) {
-                error(S.pos(), "enums.not.supported.in.source", source.name);
+            if (token.kind == ENUM) {
+                error(token.pos, "enums.not.supported.in.source", source.name);
                 allowEnums = true;
                 return enumDeclaration(mods, dc);
             }
-            int pos = S.pos();
+            int pos = token.pos;
             List<JCTree> errs;
-            if (S.token() == IDENTIFIER) {
+            if (token.kind == IDENTIFIER) {
                 errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
-                setErrorEndPos(S.pos());
+                setErrorEndPos(token.pos);
             } else {
                 errs = List.<JCTree>of(mods);
             }
@@ -2402,20 +2388,20 @@
      *  @param dc       The documentation comment for the class, or null.
      */
     JCClassDecl classDeclaration(JCModifiers mods, String dc) {
-        int pos = S.pos();
+        int pos = token.pos;
         accept(CLASS);
         Name name = ident();
 
         List<JCTypeParameter> typarams = typeParametersOpt();
 
         JCExpression extending = null;
-        if (S.token() == EXTENDS) {
-            S.nextToken();
+        if (token.kind == EXTENDS) {
+            nextToken();
             extending = parseType();
         }
         List<JCExpression> implementing = List.nil();
-        if (S.token() == IMPLEMENTS) {
-            S.nextToken();
+        if (token.kind == IMPLEMENTS) {
+            nextToken();
             implementing = typeList();
         }
         List<JCTree> defs = classOrInterfaceBody(name, false);
@@ -2431,15 +2417,15 @@
      *  @param dc       The documentation comment for the interface, or null.
      */
     JCClassDecl interfaceDeclaration(JCModifiers mods, String dc) {
-        int pos = S.pos();
+        int pos = token.pos;
         accept(INTERFACE);
         Name name = ident();
 
         List<JCTypeParameter> typarams = typeParametersOpt();
 
         List<JCExpression> extending = List.nil();
-        if (S.token() == EXTENDS) {
-            S.nextToken();
+        if (token.kind == EXTENDS) {
+            nextToken();
             extending = typeList();
         }
         List<JCTree> defs = classOrInterfaceBody(name, true);
@@ -2454,13 +2440,13 @@
      *  @param dc       The documentation comment for the enum, or null.
      */
     JCClassDecl enumDeclaration(JCModifiers mods, String dc) {
-        int pos = S.pos();
+        int pos = token.pos;
         accept(ENUM);
         Name name = ident();
 
         List<JCExpression> implementing = List.nil();
-        if (S.token() == IMPLEMENTS) {
-            S.nextToken();
+        if (token.kind == IMPLEMENTS) {
+            nextToken();
             implementing = typeList();
         }
 
@@ -2479,27 +2465,27 @@
     List<JCTree> enumBody(Name enumName) {
         accept(LBRACE);
         ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
-        if (S.token() == COMMA) {
-            S.nextToken();
-        } else if (S.token() != RBRACE && S.token() != SEMI) {
+        if (token.kind == COMMA) {
+            nextToken();
+        } else if (token.kind != RBRACE && token.kind != SEMI) {
             defs.append(enumeratorDeclaration(enumName));
-            while (S.token() == COMMA) {
-                S.nextToken();
-                if (S.token() == RBRACE || S.token() == SEMI) break;
+            while (token.kind == COMMA) {
+                nextToken();
+                if (token.kind == RBRACE || token.kind == SEMI) break;
                 defs.append(enumeratorDeclaration(enumName));
             }
-            if (S.token() != SEMI && S.token() != RBRACE) {
-                defs.append(syntaxError(S.pos(), "expected3",
+            if (token.kind != SEMI && token.kind != RBRACE) {
+                defs.append(syntaxError(token.pos, "expected3",
                                 COMMA, RBRACE, SEMI));
-                S.nextToken();
+                nextToken();
             }
         }
-        if (S.token() == SEMI) {
-            S.nextToken();
-            while (S.token() != RBRACE && S.token() != EOF) {
+        if (token.kind == SEMI) {
+            nextToken();
+            while (token.kind != RBRACE && token.kind != EOF) {
                 defs.appendList(classOrInterfaceBodyDeclaration(enumName,
                                                                 false));
-                if (S.pos() <= errorEndPos) {
+                if (token.pos <= errorEndPos) {
                     // error recovery
                    skip(false, true, true, false);
                 }
@@ -2512,23 +2498,22 @@
     /** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ]
      */
     JCTree enumeratorDeclaration(Name enumName) {
-        String dc = S.docComment();
+        String dc = token.docComment;
         int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM;
-        if (S.deprecatedFlag()) {
+        if (token.deprecatedFlag) {
             flags |= Flags.DEPRECATED;
-            S.resetDeprecatedFlag();
         }
-        int pos = S.pos();
+        int pos = token.pos;
         List<JCAnnotation> annotations = annotationsOpt();
         JCModifiers mods = F.at(annotations.isEmpty() ? Position.NOPOS : pos).Modifiers(flags, annotations);
         List<JCExpression> typeArgs = typeArgumentsOpt();
-        int identPos = S.pos();
+        int identPos = token.pos;
         Name name = ident();
-        int createPos = S.pos();
-        List<JCExpression> args = (S.token() == LPAREN)
+        int createPos = token.pos;
+        List<JCExpression> args = (token.kind == LPAREN)
             ? arguments() : List.<JCExpression>nil();
         JCClassDecl body = null;
-        if (S.token() == LBRACE) {
+        if (token.kind == LBRACE) {
             JCModifiers mods1 = F.at(Position.NOPOS).Modifiers(Flags.ENUM | Flags.STATIC);
             List<JCTree> defs = classOrInterfaceBody(names.empty, false);
             body = toP(F.at(identPos).AnonymousClassDef(mods1, defs));
@@ -2538,7 +2523,7 @@
         JCIdent ident = F.at(identPos).Ident(enumName);
         JCNewClass create = F.at(createPos).NewClass(null, typeArgs, ident, args, body);
         if (createPos != identPos)
-            storeEnd(create, S.prevEndPos());
+            storeEnd(create, S.prevToken().endPos);
         ident = F.at(identPos).Ident(enumName);
         JCTree result = toP(F.at(pos).VarDef(mods, name, ident, create));
         attach(result, dc);
@@ -2550,8 +2535,8 @@
     List<JCExpression> typeList() {
         ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
         ts.append(parseType());
-        while (S.token() == COMMA) {
-            S.nextToken();
+        while (token.kind == COMMA) {
+            nextToken();
             ts.append(parseType());
         }
         return ts.toList();
@@ -2562,16 +2547,16 @@
      */
     List<JCTree> classOrInterfaceBody(Name className, boolean isInterface) {
         accept(LBRACE);
-        if (S.pos() <= errorEndPos) {
+        if (token.pos <= errorEndPos) {
             // error recovery
             skip(false, true, false, false);
-            if (S.token() == LBRACE)
-                S.nextToken();
+            if (token.kind == LBRACE)
+                nextToken();
         }
         ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
-        while (S.token() != RBRACE && S.token() != EOF) {
+        while (token.kind != RBRACE && token.kind != EOF) {
             defs.appendList(classOrInterfaceBodyDeclaration(className, isInterface));
-            if (S.pos() <= errorEndPos) {
+            if (token.pos <= errorEndPos) {
                // error recovery
                skip(false, true, true, false);
            }
@@ -2598,23 +2583,23 @@
      *      ( ConstantDeclaratorsRest | InterfaceMethodDeclaratorRest ";" )
      */
     protected List<JCTree> classOrInterfaceBodyDeclaration(Name className, boolean isInterface) {
-        if (S.token() == SEMI) {
-            S.nextToken();
+        if (token.kind == SEMI) {
+            nextToken();
             return List.<JCTree>nil();
         } else {
-            String dc = S.docComment();
-            int pos = S.pos();
+            String dc = token.docComment;
+            int pos = token.pos;
             JCModifiers mods = modifiersOpt();
-            if (S.token() == CLASS ||
-                S.token() == INTERFACE ||
-                allowEnums && S.token() == ENUM) {
+            if (token.kind == CLASS ||
+                token.kind == INTERFACE ||
+                allowEnums && token.kind == ENUM) {
                 return List.<JCTree>of(classOrInterfaceOrEnumDeclaration(mods, dc));
-            } else if (S.token() == LBRACE && !isInterface &&
+            } else if (token.kind == LBRACE && !isInterface &&
                        (mods.flags & Flags.StandardFlags & ~Flags.STATIC) == 0 &&
                        mods.annotations.isEmpty()) {
                 return List.<JCTree>of(block(pos, mods.flags));
             } else {
-                pos = S.pos();
+                pos = token.pos;
                 List<JCTypeParameter> typarams = typeParametersOpt();
                 // if there are type parameters but no modifiers, save the start
                 // position of the method in the modifiers.
@@ -2622,26 +2607,26 @@
                     mods.pos = pos;
                     storeEnd(mods, pos);
                 }
-                Name name = S.name();
-                pos = S.pos();
+                Token tk = token;
+                pos = token.pos;
                 JCExpression type;
-                boolean isVoid = S.token() == VOID;
+                boolean isVoid = token.kind == VOID;
                 if (isVoid) {
                     type = to(F.at(pos).TypeIdent(TypeTags.VOID));
-                    S.nextToken();
+                    nextToken();
                 } else {
                     type = parseType();
                 }
-                if (S.token() == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
-                    if (isInterface || name != className)
+                if (token.kind == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
+                    if (isInterface || tk.name() != className)
                         error(pos, "invalid.meth.decl.ret.type.req");
                     return List.of(methodDeclaratorRest(
                         pos, mods, null, names.init, typarams,
                         isInterface, true, dc));
                 } else {
-                    pos = S.pos();
-                    name = ident();
-                    if (S.token() == LPAREN) {
+                    pos = token.pos;
+                    Name name = ident();
+                    if (token.kind == LPAREN) {
                         return List.of(methodDeclaratorRest(
                             pos, mods, type, name, typarams,
                             isInterface, isVoid, dc));
@@ -2649,16 +2634,16 @@
                         List<JCTree> defs =
                             variableDeclaratorsRest(pos, mods, type, name, isInterface, dc,
                                                     new ListBuffer<JCTree>()).toList();
-                        storeEnd(defs.last(), S.endPos());
+                        storeEnd(defs.last(), token.endPos);
                         accept(SEMI);
                         return defs;
                     } else {
-                        pos = S.pos();
+                        pos = token.pos;
                         List<JCTree> err = isVoid
                             ? List.<JCTree>of(toP(F.at(pos).MethodDef(mods, name, type, typarams,
                                 List.<JCVariableDecl>nil(), List.<JCExpression>nil(), null, null)))
                             : null;
-                        return List.<JCTree>of(syntaxError(S.pos(), err, "expected", LPAREN));
+                        return List.<JCTree>of(syntaxError(token.pos, err, "expected", LPAREN));
                     }
                 }
             }
@@ -2686,27 +2671,27 @@
         List<JCVariableDecl> params = formalParameters();
         if (!isVoid) type = bracketsOpt(type);
         List<JCExpression> thrown = List.nil();
-        if (S.token() == THROWS) {
-            S.nextToken();
+        if (token.kind == THROWS) {
+            nextToken();
             thrown = qualidentList();
         }
         JCBlock body = null;
         JCExpression defaultValue;
-        if (S.token() == LBRACE) {
+        if (token.kind == LBRACE) {
             body = block();
             defaultValue = null;
         } else {
-            if (S.token() == DEFAULT) {
+            if (token.kind == DEFAULT) {
                 accept(DEFAULT);
                 defaultValue = annotationValue();
             } else {
                 defaultValue = null;
             }
             accept(SEMI);
-            if (S.pos() <= errorEndPos) {
+            if (token.pos <= errorEndPos) {
                 // error recovery
                 skip(false, true, false, false);
-                if (S.token() == LBRACE) {
+                if (token.kind == LBRACE) {
                     body = block();
                 }
             }
@@ -2725,8 +2710,8 @@
     List<JCExpression> qualidentList() {
         ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
         ts.append(qualident());
-        while (S.token() == COMMA) {
-            S.nextToken();
+        while (token.kind == COMMA) {
+            nextToken();
             ts.append(qualident());
         }
         return ts.toList();
@@ -2735,13 +2720,13 @@
     /** TypeParametersOpt = ["<" TypeParameter {"," TypeParameter} ">"]
      */
     List<JCTypeParameter> typeParametersOpt() {
-        if (S.token() == LT) {
+        if (token.kind == LT) {
             checkGenerics();
             ListBuffer<JCTypeParameter> typarams = new ListBuffer<JCTypeParameter>();
-            S.nextToken();
+            nextToken();
             typarams.append(typeParameter());
-            while (S.token() == COMMA) {
-                S.nextToken();
+            while (token.kind == COMMA) {
+                nextToken();
                 typarams.append(typeParameter());
             }
             accept(GT);
@@ -2756,14 +2741,14 @@
      *  TypeVariable = Ident
      */
     JCTypeParameter typeParameter() {
-        int pos = S.pos();
+        int pos = token.pos;
         Name name = ident();
         ListBuffer<JCExpression> bounds = new ListBuffer<JCExpression>();
-        if (S.token() == EXTENDS) {
-            S.nextToken();
+        if (token.kind == EXTENDS) {
+            nextToken();
             bounds.append(parseType());
-            while (S.token() == AMP) {
-                S.nextToken();
+            while (token.kind == AMP) {
+                nextToken();
                 bounds.append(parseType());
             }
         }
@@ -2778,10 +2763,10 @@
         ListBuffer<JCVariableDecl> params = new ListBuffer<JCVariableDecl>();
         JCVariableDecl lastParam = null;
         accept(LPAREN);
-        if (S.token() != RPAREN) {
+        if (token.kind != RPAREN) {
             params.append(lastParam = formalParameter());
-            while ((lastParam.mods.flags & Flags.VARARGS) == 0 && S.token() == COMMA) {
-                S.nextToken();
+            while ((lastParam.mods.flags & Flags.VARARGS) == 0 && token.kind == COMMA) {
+                nextToken();
                 params.append(lastParam = formalParameter());
             }
         }
@@ -2802,11 +2787,11 @@
     protected JCVariableDecl formalParameter() {
         JCModifiers mods = optFinal(Flags.PARAMETER);
         JCExpression type = parseType();
-        if (S.token() == ELLIPSIS) {
+        if (token.kind == ELLIPSIS) {
             checkVarargs();
             mods.flags |= Flags.VARARGS;
-            type = to(F.at(S.pos()).TypeArray(type));
-            S.nextToken();
+            type = to(F.at(token.pos).TypeArray(type));
+            nextToken();
         }
         return variableDeclaratorId(mods, type);
     }
@@ -2849,7 +2834,7 @@
     /** Return precedence of operator represented by token,
      *  -1 if token is not a binary operator. @see TreeInfo.opPrec
      */
-    static int prec(Token token) {
+    static int prec(TokenKind token) {
         int oc = optag(token);
         return (oc >= 0) ? TreeInfo.opPrec(oc) : -1;
     }
@@ -2869,7 +2854,7 @@
     /** Return operation tag of binary operator represented by token,
      *  -1 if token is not a binary operator.
      */
-    static int optag(Token token) {
+    static int optag(TokenKind token) {
         switch (token) {
         case BARBAR:
             return JCTree.OR;
@@ -2941,7 +2926,7 @@
     /** Return operation tag of unary operator represented by token,
      *  -1 if token is not a binary operator.
      */
-    static int unoptag(Token token) {
+    static int unoptag(TokenKind token) {
         switch (token) {
         case PLUS:
             return JCTree.POS;
@@ -2963,7 +2948,7 @@
     /** Return type tag of basic type represented by token,
      *  -1 if token is not a basic type identifier.
      */
-    static int typetag(Token token) {
+    static int typetag(TokenKind token) {
         switch (token) {
         case BYTE:
             return TypeTags.BYTE;
@@ -2988,49 +2973,49 @@
 
     void checkGenerics() {
         if (!allowGenerics) {
-            error(S.pos(), "generics.not.supported.in.source", source.name);
+            error(token.pos, "generics.not.supported.in.source", source.name);
             allowGenerics = true;
         }
     }
     void checkVarargs() {
         if (!allowVarargs) {
-            error(S.pos(), "varargs.not.supported.in.source", source.name);
+            error(token.pos, "varargs.not.supported.in.source", source.name);
             allowVarargs = true;
         }
     }
     void checkForeach() {
         if (!allowForeach) {
-            error(S.pos(), "foreach.not.supported.in.source", source.name);
+            error(token.pos, "foreach.not.supported.in.source", source.name);
             allowForeach = true;
         }
     }
     void checkStaticImports() {
         if (!allowStaticImport) {
-            error(S.pos(), "static.import.not.supported.in.source", source.name);
+            error(token.pos, "static.import.not.supported.in.source", source.name);
             allowStaticImport = true;
         }
     }
     void checkAnnotations() {
         if (!allowAnnotations) {
-            error(S.pos(), "annotations.not.supported.in.source", source.name);
+            error(token.pos, "annotations.not.supported.in.source", source.name);
             allowAnnotations = true;
         }
     }
     void checkDiamond() {
         if (!allowDiamond) {
-            error(S.pos(), "diamond.not.supported.in.source", source.name);
+            error(token.pos, "diamond.not.supported.in.source", source.name);
             allowDiamond = true;
         }
     }
     void checkMulticatch() {
         if (!allowMulticatch) {
-            error(S.pos(), "multicatch.not.supported.in.source", source.name);
+            error(token.pos, "multicatch.not.supported.in.source", source.name);
             allowMulticatch = true;
         }
     }
     void checkTryWithResources() {
         if (!allowTWR) {
-            error(S.pos(), "try.with.resources.not.supported.in.source", source.name);
+            error(token.pos, "try.with.resources.not.supported.in.source", source.name);
             allowTWR = true;
         }
     }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.parser.Tokens.Token;
+import com.sun.tools.javac.util.*;
+
+import java.nio.*;
+
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** An extension to the base lexical analyzer that captures
+ *  and processes the contents of doc comments.  It does so by
+ *  translating Unicode escape sequences and by stripping the
+ *  leading whitespace and starts from each line of the comment.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class JavadocTokenizer extends JavaTokenizer {
+
+    /** Create a scanner from the input buffer.  buffer must implement
+     *  array() and compact(), and remaining() must be less than limit().
+     */
+    protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
+        super(fac, buffer);
+    }
+
+    /** Create a scanner from the input array.  The array must have at
+     *  least a single character of extra space.
+     */
+    protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
+        super(fac, input, inputLength);
+    }
+
+    /** The comment input buffer, index of next chacter to be read,
+     *  index of one past last character in buffer.
+     */
+    private char[] buf;
+    private int bp;
+    private int buflen;
+
+    /** The current character.
+     */
+    private char ch;
+
+    /** The column number position of the current character.
+     */
+    private int col;
+
+    /** The buffer index of the last converted Unicode character
+     */
+    private int unicodeConversionBp = 0;
+
+    /**
+     * Buffer for doc comment.
+     */
+    private char[] docCommentBuffer = new char[1024];
+
+    /**
+     * Number of characters in doc comment buffer.
+     */
+    private int docCommentCount;
+
+    /**
+     * Translated and stripped contents of doc comment
+     */
+    private String docComment = null;
+
+
+    /** Unconditionally expand the comment buffer.
+     */
+    private void expandCommentBuffer() {
+        char[] newBuffer = new char[docCommentBuffer.length * 2];
+        System.arraycopy(docCommentBuffer, 0, newBuffer,
+                         0, docCommentBuffer.length);
+        docCommentBuffer = newBuffer;
+    }
+
+    /** Convert an ASCII digit from its base (8, 10, or 16)
+     *  to its value.
+     */
+    private int digit(int base) {
+        char c = ch;
+        int result = Character.digit(c, base);
+        if (result >= 0 && c > 0x7f) {
+            ch = "0123456789abcdef".charAt(result);
+        }
+        return result;
+    }
+
+    /** Convert Unicode escape; bp points to initial '\' character
+     *  (Spec 3.3).
+     */
+    private void convertUnicode() {
+        if (ch == '\\' && unicodeConversionBp != bp) {
+            bp++; ch = buf[bp]; col++;
+            if (ch == 'u') {
+                do {
+                    bp++; ch = buf[bp]; col++;
+                } while (ch == 'u');
+                int limit = bp + 3;
+                if (limit < buflen) {
+                    int d = digit(16);
+                    int code = d;
+                    while (bp < limit && d >= 0) {
+                        bp++; ch = buf[bp]; col++;
+                        d = digit(16);
+                        code = (code << 4) + d;
+                    }
+                    if (d >= 0) {
+                        ch = (char)code;
+                        unicodeConversionBp = bp;
+                        return;
+                    }
+                }
+                // "illegal.Unicode.esc", reported by base scanner
+            } else {
+                bp--;
+                ch = '\\';
+                col--;
+            }
+        }
+    }
+
+
+    /** Read next character.
+     */
+    private void scanChar() {
+        bp++;
+        ch = buf[bp];
+        switch (ch) {
+        case '\r': // return
+            col = 0;
+            break;
+        case '\n': // newline
+            if (bp == 0 || buf[bp-1] != '\r') {
+                col = 0;
+            }
+            break;
+        case '\t': // tab
+            col = (col / TabInc * TabInc) + TabInc;
+            break;
+        case '\\': // possible Unicode
+            col++;
+            convertUnicode();
+            break;
+        default:
+            col++;
+            break;
+        }
+    }
+
+    @Override
+    public Token readToken() {
+        docComment = null;
+        Token tk = super.readToken();
+        tk.docComment = docComment;
+        return tk;
+    }
+
+    /**
+     * Read next character in doc comment, skipping over double '\' characters.
+     * If a double '\' is skipped, put in the buffer and update buffer count.
+     */
+    private void scanDocCommentChar() {
+        scanChar();
+        if (ch == '\\') {
+            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
+                if (docCommentCount == docCommentBuffer.length)
+                    expandCommentBuffer();
+                docCommentBuffer[docCommentCount++] = ch;
+                bp++; col++;
+            } else {
+                convertUnicode();
+            }
+        }
+    }
+
+    /**
+     * Process a doc comment and make the string content available.
+     * Strips leading whitespace and stars.
+     */
+    @SuppressWarnings("fallthrough")
+    protected void processComment(int pos, int endPos, CommentStyle style) {
+        if (style != CommentStyle.JAVADOC) {
+            return;
+        }
+
+        buf = reader.getRawCharacters(pos, endPos);
+        buflen = buf.length;
+        bp = 0;
+        col = 0;
+
+        docCommentCount = 0;
+
+        boolean firstLine = true;
+
+        // Skip over first slash
+        scanDocCommentChar();
+        // Skip over first star
+        scanDocCommentChar();
+
+        // consume any number of stars
+        while (bp < buflen && ch == '*') {
+            scanDocCommentChar();
+        }
+        // is the comment in the form /**/, /***/, /****/, etc. ?
+        if (bp < buflen && ch == '/') {
+            docComment = "";
+            return;
+        }
+
+        // skip a newline on the first line of the comment.
+        if (bp < buflen) {
+            if (ch == LF) {
+                scanDocCommentChar();
+                firstLine = false;
+            } else if (ch == CR) {
+                scanDocCommentChar();
+                if (ch == LF) {
+                    scanDocCommentChar();
+                    firstLine = false;
+                }
+            }
+        }
+
+    outerLoop:
+
+        // The outerLoop processes the doc comment, looping once
+        // for each line.  For each line, it first strips off
+        // whitespace, then it consumes any stars, then it
+        // puts the rest of the line into our buffer.
+        while (bp < buflen) {
+
+            // The wsLoop consumes whitespace from the beginning
+            // of each line.
+        wsLoop:
+
+            while (bp < buflen) {
+                switch(ch) {
+                case ' ':
+                    scanDocCommentChar();
+                    break;
+                case '\t':
+                    col = ((col - 1) / TabInc * TabInc) + TabInc;
+                    scanDocCommentChar();
+                    break;
+                case FF:
+                    col = 0;
+                    scanDocCommentChar();
+                    break;
+// Treat newline at beginning of line (blank line, no star)
+// as comment text.  Old Javadoc compatibility requires this.
+/*---------------------------------*
+                case CR: // (Spec 3.4)
+                    scanDocCommentChar();
+                    if (ch == LF) {
+                        col = 0;
+                        scanDocCommentChar();
+                    }
+                    break;
+                case LF: // (Spec 3.4)
+                    scanDocCommentChar();
+                    break;
+*---------------------------------*/
+                default:
+                    // we've seen something that isn't whitespace;
+                    // jump out.
+                    break wsLoop;
+                }
+            }
+
+            // Are there stars here?  If so, consume them all
+            // and check for the end of comment.
+            if (ch == '*') {
+                // skip all of the stars
+                do {
+                    scanDocCommentChar();
+                } while (ch == '*');
+
+                // check for the closing slash.
+                if (ch == '/') {
+                    // We're done with the doc comment
+                    // scanChar() and breakout.
+                    break outerLoop;
+                }
+            } else if (! firstLine) {
+                //The current line does not begin with a '*' so we will indent it.
+                for (int i = 1; i < col; i++) {
+                    if (docCommentCount == docCommentBuffer.length)
+                        expandCommentBuffer();
+                    docCommentBuffer[docCommentCount++] = ' ';
+                }
+            }
+
+            // The textLoop processes the rest of the characters
+            // on the line, adding them to our buffer.
+        textLoop:
+            while (bp < buflen) {
+                switch (ch) {
+                case '*':
+                    // Is this just a star?  Or is this the
+                    // end of a comment?
+                    scanDocCommentChar();
+                    if (ch == '/') {
+                        // This is the end of the comment,
+                        // set ch and return our buffer.
+                        break outerLoop;
+                    }
+                    // This is just an ordinary star.  Add it to
+                    // the buffer.
+                    if (docCommentCount == docCommentBuffer.length)
+                        expandCommentBuffer();
+                    docCommentBuffer[docCommentCount++] = '*';
+                    break;
+                case ' ':
+                case '\t':
+                    if (docCommentCount == docCommentBuffer.length)
+                        expandCommentBuffer();
+                    docCommentBuffer[docCommentCount++] = ch;
+                    scanDocCommentChar();
+                    break;
+                case FF:
+                    scanDocCommentChar();
+                    break textLoop; // treat as end of line
+                case CR: // (Spec 3.4)
+                    scanDocCommentChar();
+                    if (ch != LF) {
+                        // Canonicalize CR-only line terminator to LF
+                        if (docCommentCount == docCommentBuffer.length)
+                            expandCommentBuffer();
+                        docCommentBuffer[docCommentCount++] = (char)LF;
+                        break textLoop;
+                    }
+                    /* fall through to LF case */
+                case LF: // (Spec 3.4)
+                    // We've seen a newline.  Add it to our
+                    // buffer and break out of this loop,
+                    // starting fresh on a new line.
+                    if (docCommentCount == docCommentBuffer.length)
+                        expandCommentBuffer();
+                    docCommentBuffer[docCommentCount++] = ch;
+                    scanDocCommentChar();
+                    break textLoop;
+                default:
+                    // Add the character to our buffer.
+                    if (docCommentCount == docCommentBuffer.length)
+                        expandCommentBuffer();
+                    docCommentBuffer[docCommentCount++] = ch;
+                    scanDocCommentChar();
+                }
+            } // end textLoop
+            firstLine = false;
+        } // end outerLoop
+
+        if (docCommentCount > 0) {
+            int i = docCommentCount - 1;
+        trailLoop:
+            while (i > -1) {
+                switch (docCommentBuffer[i]) {
+                case '*':
+                    i--;
+                    break;
+                default:
+                    break trailLoop;
+                }
+            }
+            docCommentCount = i + 1;
+
+            // Store the text of the doc comment
+            docComment = new String(docCommentBuffer, 0 , docCommentCount);
+        } else {
+            docComment = "";
+        }
+    }
+
+    /** Build a map for translating between line numbers and
+     * positions in the input.
+     *
+     * @return a LineMap */
+    public Position.LineMap getLineMap() {
+        char[] buf = reader.getRawCharacters();
+        return Position.makeLineMap(buf, buf.length, true);
+    }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Keywords.java	Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import com.sun.tools.javac.util.Context;
-import com.sun.tools.javac.util.Log;
-import com.sun.tools.javac.util.Name;
-import com.sun.tools.javac.util.Names;
-
-import static com.sun.tools.javac.parser.Token.*;
-
-/**
- * Map from Name to Token and Token to String.
- *
- * <p><b>This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.</b>
- */
-public class Keywords {
-    public static final Context.Key<Keywords> keywordsKey =
-        new Context.Key<Keywords>();
-
-    public static Keywords instance(Context context) {
-        Keywords instance = context.get(keywordsKey);
-        if (instance == null)
-            instance = new Keywords(context);
-        return instance;
-    }
-
-    private final Names names;
-
-    protected Keywords(Context context) {
-        context.put(keywordsKey, this);
-        names = Names.instance(context);
-
-        for (Token t : Token.values()) {
-            if (t.name != null)
-                enterKeyword(t.name, t);
-            else
-                tokenName[t.ordinal()] = null;
-        }
-
-        key = new Token[maxKey+1];
-        for (int i = 0; i <= maxKey; i++) key[i] = IDENTIFIER;
-        for (Token t : Token.values()) {
-            if (t.name != null)
-                key[tokenName[t.ordinal()].getIndex()] = t;
-        }
-    }
-
-
-    public Token key(Name name) {
-        return (name.getIndex() > maxKey) ? IDENTIFIER : key[name.getIndex()];
-    }
-
-    /**
-     * Keyword array. Maps name indices to Token.
-     */
-    private final Token[] key;
-
-    /**  The number of the last entered keyword.
-     */
-    private int maxKey = 0;
-
-    /** The names of all tokens.
-     */
-    private Name[] tokenName = new Name[Token.values().length];
-
-    private void enterKeyword(String s, Token token) {
-        Name n = names.fromString(s);
-        tokenName[token.ordinal()] = n;
-        if (n.getIndex() > maxKey) maxKey = n.getIndex();
-    }
-}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java	Mon Oct 24 13:00:20 2011 +0100
@@ -25,7 +25,7 @@
 
 package com.sun.tools.javac.parser;
 
-import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.parser.Tokens.*;
 import com.sun.tools.javac.util.Position.LineMap;
 
 /**
@@ -40,22 +40,26 @@
 public interface Lexer {
 
     /**
-     * Has a @deprecated been encountered in last doc comment?
-     * This needs to be reset by client with resetDeprecatedFlag.
+     * Consume the next token.
      */
-    boolean deprecatedFlag();
+    void nextToken();
 
-    void resetDeprecatedFlag();
+    /**
+     * Return current token.
+     */
+    Token token();
 
     /**
-     * Returns the documentation string of the current token.
+     * Return the last character position of the previous token.
      */
-    String docComment();
+    Token prevToken();
 
     /**
-     * Return the last character position of the current token.
+     * Splits the current token in two and return the first (splitted) token.
+     * For instance '<<<' is splitted into two tokens '<' and '<<' respectively,
+     * and the latter is returned.
      */
-    int endPos();
+    Token split();
 
     /**
      * Return the position where a lexical error occurred;
@@ -74,69 +78,4 @@
      * @return a LineMap
      */
     LineMap getLineMap();
-
-    /**
-     * Returns a copy of the input buffer, up to its inputLength.
-     * Unicode escape sequences are not translated.
-     */
-    char[] getRawCharacters();
-
-    /**
-     * Returns a copy of a character array subset of the input buffer.
-     * The returned array begins at the <code>beginIndex</code> and
-     * extends to the character at index <code>endIndex - 1</code>.
-     * Thus the length of the substring is <code>endIndex-beginIndex</code>.
-     * This behavior is like
-     * <code>String.substring(beginIndex, endIndex)</code>.
-     * Unicode escape sequences are not translated.
-     *
-     * @param beginIndex the beginning index, inclusive.
-     * @param endIndex the ending index, exclusive.
-     * @throws IndexOutOfBounds if either offset is outside of the
-     *         array bounds
-     */
-    char[] getRawCharacters(int beginIndex, int endIndex);
-
-    /**
-     * Return the name of an identifier or token for the current token.
-     */
-    Name name();
-
-    /**
-     * Read token.
-     */
-    void nextToken();
-
-    /**
-     * Return the current token's position: a 0-based
-     *  offset from beginning of the raw input stream
-     *  (before unicode translation)
-     */
-    int pos();
-
-    /**
-     * Return the last character position of the previous token.
-     */
-    int prevEndPos();
-
-    /**
-     * Return the radix of a numeric literal token.
-     */
-    int radix();
-
-    /**
-     * The value of a literal token, recorded as a string.
-     *  For integers, leading 0x and 'l' suffixes are suppressed.
-     */
-    String stringVal();
-
-    /**
-     * Return the current token, set by nextToken().
-     */
-    Token token();
-
-    /**
-     * Sets the current token.
-     */
-    void token(Token token);
 }
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java	Mon Oct 24 13:00:20 2011 +0100
@@ -55,7 +55,7 @@
 
     final TreeMaker F;
     final Log log;
-    final Keywords keywords;
+    final Tokens tokens;
     final Source source;
     final Names names;
     final Options options;
@@ -67,7 +67,7 @@
         this.F = TreeMaker.instance(context);
         this.log = Log.instance(context);
         this.names = Names.instance(context);
-        this.keywords = Keywords.instance(context);
+        this.tokens = Tokens.instance(context);
         this.source = Source.instance(context);
         this.options = Options.instance(context);
         this.scannerFactory = ScannerFactory.instance(context);
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java	Mon Oct 24 13:00:20 2011 +0100
@@ -27,13 +27,11 @@
 
 import java.nio.*;
 
-import com.sun.tools.javac.code.Source;
-import com.sun.tools.javac.file.JavacFileManager;
 import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.util.Position.LineMap;
+import com.sun.tools.javac.parser.JavaTokenizer.*;
 
-
-import static com.sun.tools.javac.parser.Token.*;
-import static com.sun.tools.javac.util.LayoutCharacters.*;
+import static com.sun.tools.javac.parser.Tokens.*;
 
 /** The lexical analyzer maps an input stream consisting of
  *  ASCII characters and Unicode escapes into a token sequence.
@@ -45,119 +43,17 @@
  */
 public class Scanner implements Lexer {
 
-    private static boolean scannerDebug = false;
-
-    /* Output variables; set by nextToken():
-     */
+    private Tokens tokens;
 
     /** The token, set by nextToken().
      */
     private Token token;
 
-    /** Allow hex floating-point literals.
-     */
-    private boolean allowHexFloats;
-
-    /** Allow binary literals.
-     */
-    private boolean allowBinaryLiterals;
-
-    /** Allow underscores in literals.
-     */
-    private boolean allowUnderscoresInLiterals;
-
-    /** The source language setting.
-     */
-    private Source source;
-
-    /** The token's position, 0-based offset from beginning of text.
-     */
-    private int pos;
-
-    /** Character position just after the last character of the token.
+    /** The previous token, set by nextToken().
      */
-    private int endPos;
-
-    /** The last character position of the previous token.
-     */
-    private int prevEndPos;
-
-    /** The position where a lexical error occurred;
-     */
-    private int errPos = Position.NOPOS;
-
-    /** The name of an identifier or token:
-     */
-    private Name name;
-
-    /** The radix of a numeric literal token.
-     */
-    private int radix;
-
-    /** Has a @deprecated been encountered in last doc comment?
-     *  this needs to be reset by client.
-     */
-    protected boolean deprecatedFlag = false;
-
-    /** A character buffer for literals.
-     */
-    private char[] sbuf = new char[128];
-    private int sp;
+    private Token prevToken;
 
-    /** The input buffer, index of next chacter to be read,
-     *  index of one past last character in buffer.
-     */
-    private char[] buf;
-    private int bp;
-    private int buflen;
-    private int eofPos;
-
-    /** The current character.
-     */
-    private char ch;
-
-    /** The buffer index of the last converted unicode character
-     */
-    private int unicodeConversionBp = -1;
-
-    /** The log to be used for error reporting.
-     */
-    private final Log log;
-
-    /** The name table. */
-    private final Names names;
-
-    /** The keyword table. */
-    private final Keywords keywords;
-
-    /** Common code for constructors. */
-    private Scanner(ScannerFactory fac) {
-        log = fac.log;
-        names = fac.names;
-        keywords = fac.keywords;
-        source = fac.source;
-        allowBinaryLiterals = source.allowBinaryLiterals();
-        allowHexFloats = source.allowHexFloats();
-        allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
-    }
-
-    private static final boolean hexFloatsWork = hexFloatsWork();
-    private static boolean hexFloatsWork() {
-        try {
-            Float.valueOf("0x1.0p1");
-            return true;
-        } catch (NumberFormatException ex) {
-            return false;
-        }
-    }
-
-    /** Create a scanner from the input buffer.  buffer must implement
-     *  array() and compact(), and remaining() must be less than limit().
-     */
-    protected Scanner(ScannerFactory fac, CharBuffer buffer) {
-        this(fac, JavacFileManager.toArray(buffer), buffer.limit());
-    }
-
+    private JavaTokenizer tokenizer;
     /**
      * Create a scanner from the input array.  This method might
      * modify the array.  To avoid copying the input array, ensure
@@ -169,972 +65,49 @@
      * @param inputLength the size of the input.
      * Must be positive and less than or equal to input.length.
      */
-    protected Scanner(ScannerFactory fac, char[] input, int inputLength) {
-        this(fac);
-        eofPos = inputLength;
-        if (inputLength == input.length) {
-            if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
-                inputLength--;
-            } else {
-                char[] newInput = new char[inputLength + 1];
-                System.arraycopy(input, 0, newInput, 0, input.length);
-                input = newInput;
-            }
-        }
-        buf = input;
-        buflen = inputLength;
-        buf[buflen] = EOI;
-        bp = -1;
-        scanChar();
-    }
-
-    /** Report an error at the given position using the provided arguments.
-     */
-    private void lexError(int pos, String key, Object... args) {
-        log.error(pos, key, args);
-        token = ERROR;
-        errPos = pos;
-    }
-
-    /** Report an error at the current token position using the provided
-     *  arguments.
-     */
-    private void lexError(String key, Object... args) {
-        lexError(pos, key, args);
-    }
-
-    /** Convert an ASCII digit from its base (8, 10, or 16)
-     *  to its value.
-     */
-    private int digit(int base) {
-        char c = ch;
-        int result = Character.digit(c, base);
-        if (result >= 0 && c > 0x7f) {
-            lexError(pos+1, "illegal.nonascii.digit");
-            ch = "0123456789abcdef".charAt(result);
-        }
-        return result;
-    }
-
-    /** Convert unicode escape; bp points to initial '\' character
-     *  (Spec 3.3).
-     */
-    private void convertUnicode() {
-        if (ch == '\\' && unicodeConversionBp != bp) {
-            bp++; ch = buf[bp];
-            if (ch == 'u') {
-                do {
-                    bp++; ch = buf[bp];
-                } while (ch == 'u');
-                int limit = bp + 3;
-                if (limit < buflen) {
-                    int d = digit(16);
-                    int code = d;
-                    while (bp < limit && d >= 0) {
-                        bp++; ch = buf[bp];
-                        d = digit(16);
-                        code = (code << 4) + d;
-                    }
-                    if (d >= 0) {
-                        ch = (char)code;
-                        unicodeConversionBp = bp;
-                        return;
-                    }
-                }
-                lexError(bp, "illegal.unicode.esc");
-            } else {
-                bp--;
-                ch = '\\';
-            }
-        }
-    }
-
-    /** Read next character.
-     */
-    private void scanChar() {
-        ch = buf[++bp];
-        if (ch == '\\') {
-            convertUnicode();
-        }
-    }
-
-    /** Read next character in comment, skipping over double '\' characters.
-     */
-    private void scanCommentChar() {
-        scanChar();
-        if (ch == '\\') {
-            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
-                bp++;
-            } else {
-                convertUnicode();
-            }
-        }
-    }
-
-    /** Append a character to sbuf.
-     */
-    private void putChar(char ch) {
-        if (sp == sbuf.length) {
-            char[] newsbuf = new char[sbuf.length * 2];
-            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
-            sbuf = newsbuf;
-        }
-        sbuf[sp++] = ch;
-    }
-
-    /** Read next character in character or string literal and copy into sbuf.
-     */
-    private void scanLitChar() {
-        if (ch == '\\') {
-            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
-                bp++;
-                putChar('\\');
-                scanChar();
-            } else {
-                scanChar();
-                switch (ch) {
-                case '0': case '1': case '2': case '3':
-                case '4': case '5': case '6': case '7':
-                    char leadch = ch;
-                    int oct = digit(8);
-                    scanChar();
-                    if ('0' <= ch && ch <= '7') {
-                        oct = oct * 8 + digit(8);
-                        scanChar();
-                        if (leadch <= '3' && '0' <= ch && ch <= '7') {
-                            oct = oct * 8 + digit(8);
-                            scanChar();
-                        }
-                    }
-                    putChar((char)oct);
-                    break;
-                case 'b':
-                    putChar('\b'); scanChar(); break;
-                case 't':
-                    putChar('\t'); scanChar(); break;
-                case 'n':
-                    putChar('\n'); scanChar(); break;
-                case 'f':
-                    putChar('\f'); scanChar(); break;
-                case 'r':
-                    putChar('\r'); scanChar(); break;
-                case '\'':
-                    putChar('\''); scanChar(); break;
-                case '\"':
-                    putChar('\"'); scanChar(); break;
-                case '\\':
-                    putChar('\\'); scanChar(); break;
-                default:
-                    lexError(bp, "illegal.esc.char");
-                }
-            }
-        } else if (bp != buflen) {
-            putChar(ch); scanChar();
-        }
-    }
-
-    private void scanDigits(int digitRadix) {
-        char saveCh;
-        int savePos;
-        do {
-            if (ch != '_') {
-                putChar(ch);
-            } else {
-                if (!allowUnderscoresInLiterals) {
-                    lexError("unsupported.underscore.lit", source.name);
-                    allowUnderscoresInLiterals = true;
-                }
-            }
-            saveCh = ch;
-            savePos = bp;
-            scanChar();
-        } while (digit(digitRadix) >= 0 || ch == '_');
-        if (saveCh == '_')
-            lexError(savePos, "illegal.underscore");
-    }
-
-    /** Read fractional part of hexadecimal floating point number.
-     */
-    private void scanHexExponentAndSuffix() {
-        if (ch == 'p' || ch == 'P') {
-            putChar(ch);
-            scanChar();
-            skipIllegalUnderscores();
-            if (ch == '+' || ch == '-') {
-                putChar(ch);
-                scanChar();
-            }
-            skipIllegalUnderscores();
-            if ('0' <= ch && ch <= '9') {
-                scanDigits(10);
-                if (!allowHexFloats) {
-                    lexError("unsupported.fp.lit", source.name);
-                    allowHexFloats = true;
-                }
-                else if (!hexFloatsWork)
-                    lexError("unsupported.cross.fp.lit");
-            } else
-                lexError("malformed.fp.lit");
-        } else {
-            lexError("malformed.fp.lit");
-        }
-        if (ch == 'f' || ch == 'F') {
-            putChar(ch);
-            scanChar();
-            token = FLOATLITERAL;
-        } else {
-            if (ch == 'd' || ch == 'D') {
-                putChar(ch);
-                scanChar();
-            }
-            token = DOUBLELITERAL;
-        }
-    }
-
-    /** Read fractional part of floating point number.
-     */
-    private void scanFraction() {
-        skipIllegalUnderscores();
-        if ('0' <= ch && ch <= '9') {
-            scanDigits(10);
-        }
-        int sp1 = sp;
-        if (ch == 'e' || ch == 'E') {
-            putChar(ch);
-            scanChar();
-            skipIllegalUnderscores();
-            if (ch == '+' || ch == '-') {
-                putChar(ch);
-                scanChar();
-            }
-            skipIllegalUnderscores();
-            if ('0' <= ch && ch <= '9') {
-                scanDigits(10);
-                return;
-            }
-            lexError("malformed.fp.lit");
-            sp = sp1;
-        }
-    }
-
-    /** Read fractional part and 'd' or 'f' suffix of floating point number.
-     */
-    private void scanFractionAndSuffix() {
-        this.radix = 10;
-        scanFraction();
-        if (ch == 'f' || ch == 'F') {
-            putChar(ch);
-            scanChar();
-            token = FLOATLITERAL;
-        } else {
-            if (ch == 'd' || ch == 'D') {
-                putChar(ch);
-                scanChar();
-            }
-            token = DOUBLELITERAL;
-        }
-    }
-
-    /** Read fractional part and 'd' or 'f' suffix of floating point number.
-     */
-    private void scanHexFractionAndSuffix(boolean seendigit) {
-        this.radix = 16;
-        Assert.check(ch == '.');
-        putChar(ch);
-        scanChar();
-        skipIllegalUnderscores();
-        if (digit(16) >= 0) {
-            seendigit = true;
-            scanDigits(16);
-        }
-        if (!seendigit)
-            lexError("invalid.hex.number");
-        else
-            scanHexExponentAndSuffix();
-    }
-
-    private void skipIllegalUnderscores() {
-        if (ch == '_') {
-            lexError(bp, "illegal.underscore");
-            while (ch == '_')
-                scanChar();
-        }
-    }
-
-    /** Read a number.
-     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
-     */
-    private void scanNumber(int radix) {
-        this.radix = radix;
-        // for octal, allow base-10 digit in case it's a float literal
-        int digitRadix = (radix == 8 ? 10 : radix);
-        boolean seendigit = false;
-        if (digit(digitRadix) >= 0) {
-            seendigit = true;
-            scanDigits(digitRadix);
-        }
-        if (radix == 16 && ch == '.') {
-            scanHexFractionAndSuffix(seendigit);
-        } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) {
-            scanHexExponentAndSuffix();
-        } else if (digitRadix == 10 && ch == '.') {
-            putChar(ch);
-            scanChar();
-            scanFractionAndSuffix();
-        } else if (digitRadix == 10 &&
-                   (ch == 'e' || ch == 'E' ||
-                    ch == 'f' || ch == 'F' ||
-                    ch == 'd' || ch == 'D')) {
-            scanFractionAndSuffix();
-        } else {
-            if (ch == 'l' || ch == 'L') {
-                scanChar();
-                token = LONGLITERAL;
-            } else {
-                token = INTLITERAL;
-            }
-        }
-    }
-
-    /** Read an identifier.
-     */
-    private void scanIdent() {
-        boolean isJavaIdentifierPart;
-        char high;
-        do {
-            if (sp == sbuf.length) putChar(ch); else sbuf[sp++] = ch;
-            // optimization, was: putChar(ch);
-
-            scanChar();
-            switch (ch) {
-            case 'A': case 'B': case 'C': case 'D': case 'E':
-            case 'F': case 'G': case 'H': case 'I': case 'J':
-            case 'K': case 'L': case 'M': case 'N': case 'O':
-            case 'P': case 'Q': case 'R': case 'S': case 'T':
-            case 'U': case 'V': case 'W': case 'X': case 'Y':
-            case 'Z':
-            case 'a': case 'b': case 'c': case 'd': case 'e':
-            case 'f': case 'g': case 'h': case 'i': case 'j':
-            case 'k': case 'l': case 'm': case 'n': case 'o':
-            case 'p': case 'q': case 'r': case 's': case 't':
-            case 'u': case 'v': case 'w': case 'x': case 'y':
-            case 'z':
-            case '$': case '_':
-            case '0': case '1': case '2': case '3': case '4':
-            case '5': case '6': case '7': case '8': case '9':
-            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
-            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
-            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
-            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
-            case '\u0015': case '\u0016': case '\u0017':
-            case '\u0018': case '\u0019': case '\u001B':
-            case '\u007F':
-                break;
-            case '\u001A': // EOI is also a legal identifier part
-                if (bp >= buflen) {
-                    name = names.fromChars(sbuf, 0, sp);
-                    token = keywords.key(name);
-                    return;
-                }
-                break;
-            default:
-                if (ch < '\u0080') {
-                    // all ASCII range chars already handled, above
-                    isJavaIdentifierPart = false;
-                } else {
-                    high = scanSurrogates();
-                    if (high != 0) {
-                        if (sp == sbuf.length) {
-                            putChar(high);
-                        } else {
-                            sbuf[sp++] = high;
-                        }
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
-                            Character.toCodePoint(high, ch));
-                    } else {
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(ch);
-                    }
-                }
-                if (!isJavaIdentifierPart) {
-                    name = names.fromChars(sbuf, 0, sp);
-                    token = keywords.key(name);
-                    return;
-                }
-            }
-        } while (true);
+    protected Scanner(ScannerFactory fac, CharBuffer buf) {
+        this(fac, new JavaTokenizer(fac, buf));
     }
 
-    /** Are surrogates supported?
-     */
-    final static boolean surrogatesSupported = surrogatesSupported();
-    private static boolean surrogatesSupported() {
-        try {
-            Character.isHighSurrogate('a');
-            return true;
-        } catch (NoSuchMethodError ex) {
-            return false;
-        }
-    }
-
-    /** Scan surrogate pairs.  If 'ch' is a high surrogate and
-     *  the next character is a low surrogate, then put the low
-     *  surrogate in 'ch', and return the high surrogate.
-     *  otherwise, just return 0.
-     */
-    private char scanSurrogates() {
-        if (surrogatesSupported && Character.isHighSurrogate(ch)) {
-            char high = ch;
-
-            scanChar();
-
-            if (Character.isLowSurrogate(ch)) {
-                return high;
-            }
-
-            ch = high;
-        }
-
-        return 0;
-    }
-
-    /** Return true if ch can be part of an operator.
-     */
-    private boolean isSpecial(char ch) {
-        switch (ch) {
-        case '!': case '%': case '&': case '*': case '?':
-        case '+': case '-': case ':': case '<': case '=':
-        case '>': case '^': case '|': case '~':
-        case '@':
-            return true;
-        default:
-            return false;
-        }
-    }
-
-    /** Read longest possible sequence of special characters and convert
-     *  to token.
-     */
-    private void scanOperator() {
-        while (true) {
-            putChar(ch);
-            Name newname = names.fromChars(sbuf, 0, sp);
-            if (keywords.key(newname) == IDENTIFIER) {
-                sp--;
-                break;
-            }
-            name = newname;
-            token = keywords.key(newname);
-            scanChar();
-            if (!isSpecial(ch)) break;
-        }
-    }
-
-    /**
-     * Scan a documention comment; determine if a deprecated tag is present.
-     * Called once the initial /, * have been skipped, positioned at the second *
-     * (which is treated as the beginning of the first line).
-     * Stops positioned at the closing '/'.
-     */
-    @SuppressWarnings("fallthrough")
-    private void scanDocComment() {
-        boolean deprecatedPrefix = false;
-
-        forEachLine:
-        while (bp < buflen) {
-
-            // Skip optional WhiteSpace at beginning of line
-            while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
-                scanCommentChar();
-            }
-
-            // Skip optional consecutive Stars
-            while (bp < buflen && ch == '*') {
-                scanCommentChar();
-                if (ch == '/') {
-                    return;
-                }
-            }
-
-            // Skip optional WhiteSpace after Stars
-            while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
-                scanCommentChar();
-            }
-
-            deprecatedPrefix = false;
-            // At beginning of line in the JavaDoc sense.
-            if (bp < buflen && ch == '@' && !deprecatedFlag) {
-                scanCommentChar();
-                if (bp < buflen && ch == 'd') {
-                    scanCommentChar();
-                    if (bp < buflen && ch == 'e') {
-                        scanCommentChar();
-                        if (bp < buflen && ch == 'p') {
-                            scanCommentChar();
-                            if (bp < buflen && ch == 'r') {
-                                scanCommentChar();
-                                if (bp < buflen && ch == 'e') {
-                                    scanCommentChar();
-                                    if (bp < buflen && ch == 'c') {
-                                        scanCommentChar();
-                                        if (bp < buflen && ch == 'a') {
-                                            scanCommentChar();
-                                            if (bp < buflen && ch == 't') {
-                                                scanCommentChar();
-                                                if (bp < buflen && ch == 'e') {
-                                                    scanCommentChar();
-                                                    if (bp < buflen && ch == 'd') {
-                                                        deprecatedPrefix = true;
-                                                        scanCommentChar();
-                                                    }}}}}}}}}}}
-            if (deprecatedPrefix && bp < buflen) {
-                if (Character.isWhitespace(ch)) {
-                    deprecatedFlag = true;
-                } else if (ch == '*') {
-                    scanCommentChar();
-                    if (ch == '/') {
-                        deprecatedFlag = true;
-                        return;
-                    }
-                }
-            }
-
-            // Skip rest of line
-            while (bp < buflen) {
-                switch (ch) {
-                case '*':
-                    scanCommentChar();
-                    if (ch == '/') {
-                        return;
-                    }
-                    break;
-                case CR: // (Spec 3.4)
-                    scanCommentChar();
-                    if (ch != LF) {
-                        continue forEachLine;
-                    }
-                    /* fall through to LF case */
-                case LF: // (Spec 3.4)
-                    scanCommentChar();
-                    continue forEachLine;
-                default:
-                    scanCommentChar();
-                }
-            } // rest of line
-        } // forEachLine
-        return;
-    }
-
-    /** The value of a literal token, recorded as a string.
-     *  For integers, leading 0x and 'l' suffixes are suppressed.
-     */
-    public String stringVal() {
-        return new String(sbuf, 0, sp);
+    protected Scanner(ScannerFactory fac, char[] buf, int inputLength) {
+        this(fac, new JavaTokenizer(fac, buf, inputLength));
     }
 
-    /** Read token.
-     */
-    public void nextToken() {
-
-        try {
-            prevEndPos = endPos;
-            sp = 0;
-
-            while (true) {
-                pos = bp;
-                switch (ch) {
-                case ' ': // (Spec 3.6)
-                case '\t': // (Spec 3.6)
-                case FF: // (Spec 3.6)
-                    do {
-                        scanChar();
-                    } while (ch == ' ' || ch == '\t' || ch == FF);
-                    endPos = bp;
-                    processWhiteSpace();
-                    break;
-                case LF: // (Spec 3.4)
-                    scanChar();
-                    endPos = bp;
-                    processLineTerminator();
-                    break;
-                case CR: // (Spec 3.4)
-                    scanChar();
-                    if (ch == LF) {
-                        scanChar();
-                    }
-                    endPos = bp;
-                    processLineTerminator();
-                    break;
-                case 'A': case 'B': case 'C': case 'D': case 'E':
-                case 'F': case 'G': case 'H': case 'I': case 'J':
-                case 'K': case 'L': case 'M': case 'N': case 'O':
-                case 'P': case 'Q': case 'R': case 'S': case 'T':
-                case 'U': case 'V': case 'W': case 'X': case 'Y':
-                case 'Z':
-                case 'a': case 'b': case 'c': case 'd': case 'e':
-                case 'f': case 'g': case 'h': case 'i': case 'j':
-                case 'k': case 'l': case 'm': case 'n': case 'o':
-                case 'p': case 'q': case 'r': case 's': case 't':
-                case 'u': case 'v': case 'w': case 'x': case 'y':
-                case 'z':
-                case '$': case '_':
-                    scanIdent();
-                    return;
-                case '0':
-                    scanChar();
-                    if (ch == 'x' || ch == 'X') {
-                        scanChar();
-                        skipIllegalUnderscores();
-                        if (ch == '.') {
-                            scanHexFractionAndSuffix(false);
-                        } else if (digit(16) < 0) {
-                            lexError("invalid.hex.number");
-                        } else {
-                            scanNumber(16);
-                        }
-                    } else if (ch == 'b' || ch == 'B') {
-                        if (!allowBinaryLiterals) {
-                            lexError("unsupported.binary.lit", source.name);
-                            allowBinaryLiterals = true;
-                        }
-                        scanChar();
-                        skipIllegalUnderscores();
-                        if (digit(2) < 0) {
-                            lexError("invalid.binary.number");
-                        } else {
-                            scanNumber(2);
-                        }
-                    } else {
-                        putChar('0');
-                        if (ch == '_') {
-                            int savePos = bp;
-                            do {
-                                scanChar();
-                            } while (ch == '_');
-                            if (digit(10) < 0) {
-                                lexError(savePos, "illegal.underscore");
-                            }
-                        }
-                        scanNumber(8);
-                    }
-                    return;
-                case '1': case '2': case '3': case '4':
-                case '5': case '6': case '7': case '8': case '9':
-                    scanNumber(10);
-                    return;
-                case '.':
-                    scanChar();
-                    if ('0' <= ch && ch <= '9') {
-                        putChar('.');
-                        scanFractionAndSuffix();
-                    } else if (ch == '.') {
-                        putChar('.'); putChar('.');
-                        scanChar();
-                        if (ch == '.') {
-                            scanChar();
-                            putChar('.');
-                            token = ELLIPSIS;
-                        } else {
-                            lexError("malformed.fp.lit");
-                        }
-                    } else {
-                        token = DOT;
-                    }
-                    return;
-                case ',':
-                    scanChar(); token = COMMA; return;
-                case ';':
-                    scanChar(); token = SEMI; return;
-                case '(':
-                    scanChar(); token = LPAREN; return;
-                case ')':
-                    scanChar(); token = RPAREN; return;
-                case '[':
-                    scanChar(); token = LBRACKET; return;
-                case ']':
-                    scanChar(); token = RBRACKET; return;
-                case '{':
-                    scanChar(); token = LBRACE; return;
-                case '}':
-                    scanChar(); token = RBRACE; return;
-                case '/':
-                    scanChar();
-                    if (ch == '/') {
-                        do {
-                            scanCommentChar();
-                        } while (ch != CR && ch != LF && bp < buflen);
-                        if (bp < buflen) {
-                            endPos = bp;
-                            processComment(CommentStyle.LINE);
-                        }
-                        break;
-                    } else if (ch == '*') {
-                        scanChar();
-                        CommentStyle style;
-                        if (ch == '*') {
-                            style = CommentStyle.JAVADOC;
-                            scanDocComment();
-                        } else {
-                            style = CommentStyle.BLOCK;
-                            while (bp < buflen) {
-                                if (ch == '*') {
-                                    scanChar();
-                                    if (ch == '/') break;
-                                } else {
-                                    scanCommentChar();
-                                }
-                            }
-                        }
-                        if (ch == '/') {
-                            scanChar();
-                            endPos = bp;
-                            processComment(style);
-                            break;
-                        } else {
-                            lexError("unclosed.comment");
-                            return;
-                        }
-                    } else if (ch == '=') {
-                        name = names.slashequals;
-                        token = SLASHEQ;
-                        scanChar();
-                    } else {
-                        name = names.slash;
-                        token = SLASH;
-                    }
-                    return;
-                case '\'':
-                    scanChar();
-                    if (ch == '\'') {
-                        lexError("empty.char.lit");
-                    } else {
-                        if (ch == CR || ch == LF)
-                            lexError(pos, "illegal.line.end.in.char.lit");
-                        scanLitChar();
-                        if (ch == '\'') {
-                            scanChar();
-                            token = CHARLITERAL;
-                        } else {
-                            lexError(pos, "unclosed.char.lit");
-                        }
-                    }
-                    return;
-                case '\"':
-                    scanChar();
-                    while (ch != '\"' && ch != CR && ch != LF && bp < buflen)
-                        scanLitChar();
-                    if (ch == '\"') {
-                        token = STRINGLITERAL;
-                        scanChar();
-                    } else {
-                        lexError(pos, "unclosed.str.lit");
-                    }
-                    return;
-                default:
-                    if (isSpecial(ch)) {
-                        scanOperator();
-                    } else {
-                        boolean isJavaIdentifierStart;
-                        if (ch < '\u0080') {
-                            // all ASCII range chars already handled, above
-                            isJavaIdentifierStart = false;
-                        } else {
-                            char high = scanSurrogates();
-                            if (high != 0) {
-                                if (sp == sbuf.length) {
-                                    putChar(high);
-                                } else {
-                                    sbuf[sp++] = high;
-                                }
-
-                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
-                                    Character.toCodePoint(high, ch));
-                            } else {
-                                isJavaIdentifierStart = Character.isJavaIdentifierStart(ch);
-                            }
-                        }
-                        if (isJavaIdentifierStart) {
-                            scanIdent();
-                        } else if (bp == buflen || ch == EOI && bp+1 == buflen) { // JLS 3.5
-                            token = EOF;
-                            pos = bp = eofPos;
-                        } else {
-                            lexError("illegal.char", String.valueOf((int)ch));
-                            scanChar();
-                        }
-                    }
-                    return;
-                }
-            }
-        } finally {
-            endPos = bp;
-            if (scannerDebug)
-                System.out.println("nextToken(" + pos
-                                   + "," + endPos + ")=|" +
-                                   new String(getRawCharacters(pos, endPos))
-                                   + "|");
-        }
+    protected Scanner(ScannerFactory fac, JavaTokenizer tokenizer) {
+        this.tokenizer = tokenizer;
+        tokens = fac.tokens;
+        token = prevToken = DUMMY;
     }
 
-    /** Return the current token, set by nextToken().
-     */
     public Token token() {
         return token;
     }
 
-    /** Sets the current token.
-     * This method is primarily used to update the token stream when the
-     * parser is handling the end of nested type arguments such as
-     * {@code List<List<String>>} and needs to disambiguate between
-     * repeated use of ">" and relation operators such as ">>" and ">>>". Noting
-     * that this does not handle arbitrary tokens containing Unicode escape
-     * sequences.
-     */
-    public void token(Token token) {
-        pos += this.token.name.length() - token.name.length();
-        prevEndPos = pos;
-        this.token = token;
-    }
-
-    /** Return the current token's position: a 0-based
-     *  offset from beginning of the raw input stream
-     *  (before unicode translation)
-     */
-    public int pos() {
-        return pos;
-    }
-
-    /** Return the last character position of the current token.
-     */
-    public int endPos() {
-        return endPos;
-    }
-
-    /** Return the last character position of the previous token.
-     */
-    public int prevEndPos() {
-        return prevEndPos;
+    public Token prevToken() {
+        return prevToken;
     }
 
-    /** Return the position where a lexical error occurred;
-     */
-    public int errPos() {
-        return errPos;
-    }
-
-    /** Set the position where a lexical error occurred;
-     */
-    public void errPos(int pos) {
-        errPos = pos;
-    }
-
-    /** Return the name of an identifier or token for the current token.
-     */
-    public Name name() {
-        return name;
-    }
-
-    /** Return the radix of a numeric literal token.
-     */
-    public int radix() {
-        return radix;
-    }
-
-    /** Has a @deprecated been encountered in last doc comment?
-     *  This needs to be reset by client with resetDeprecatedFlag.
-     */
-    public boolean deprecatedFlag() {
-        return deprecatedFlag;
-    }
-
-    public void resetDeprecatedFlag() {
-        deprecatedFlag = false;
-    }
-
-    /**
-     * Returns the documentation string of the current token.
-     */
-    public String docComment() {
-        return null;
+    public void nextToken() {
+        prevToken = token;
+        token = tokenizer.readToken();
     }
 
-    /**
-     * Returns a copy of the input buffer, up to its inputLength.
-     * Unicode escape sequences are not translated.
-     */
-    public char[] getRawCharacters() {
-        char[] chars = new char[buflen];
-        System.arraycopy(buf, 0, chars, 0, buflen);
-        return chars;
-    }
-
-    /**
-     * Returns a copy of a character array subset of the input buffer.
-     * The returned array begins at the <code>beginIndex</code> and
-     * extends to the character at index <code>endIndex - 1</code>.
-     * Thus the length of the substring is <code>endIndex-beginIndex</code>.
-     * This behavior is like
-     * <code>String.substring(beginIndex, endIndex)</code>.
-     * Unicode escape sequences are not translated.
-     *
-     * @param beginIndex the beginning index, inclusive.
-     * @param endIndex the ending index, exclusive.
-     * @throws IndexOutOfBounds if either offset is outside of the
-     *         array bounds
-     */
-    public char[] getRawCharacters(int beginIndex, int endIndex) {
-        int length = endIndex - beginIndex;
-        char[] chars = new char[length];
-        System.arraycopy(buf, beginIndex, chars, 0, length);
-        return chars;
-    }
-
-    public enum CommentStyle {
-        LINE,
-        BLOCK,
-        JAVADOC,
+    public Token split() {
+        Token[] splitTokens = token.split(tokens);
+        prevToken = splitTokens[0];
+        token = splitTokens[1];
+        return token;
     }
 
-    /**
-     * Called when a complete comment has been scanned. pos and endPos
-     * will mark the comment boundary.
-     */
-    protected void processComment(CommentStyle style) {
-        if (scannerDebug)
-            System.out.println("processComment(" + pos
-                               + "," + endPos + "," + style + ")=|"
-                               + new String(getRawCharacters(pos, endPos))
-                               + "|");
+    public LineMap getLineMap() {
+        return tokenizer.getLineMap();
     }
 
-    /**
-     * Called when a complete whitespace run has been scanned. pos and endPos
-     * will mark the whitespace boundary.
-     */
-    protected void processWhiteSpace() {
-        if (scannerDebug)
-            System.out.println("processWhitespace(" + pos
-                               + "," + endPos + ")=|" +
-                               new String(getRawCharacters(pos, endPos))
-                               + "|");
+    public int errPos() {
+        return tokenizer.errPos();
     }
 
-    /**
-     * Called when a line terminator has been processed.
-     */
-    protected void processLineTerminator() {
-        if (scannerDebug)
-            System.out.println("processTerminator(" + pos
-                               + "," + endPos + ")=|" +
-                               new String(getRawCharacters(pos, endPos))
-                               + "|");
+    public void errPos(int pos) {
+        tokenizer.errPos(pos);
     }
-
-    /** Build a map for translating between line numbers and
-     * positions in the input.
-     *
-     * @return a LineMap */
-    public Position.LineMap getLineMap() {
-        return Position.makeLineMap(buf, buflen, false);
-    }
-
 }
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java	Mon Oct 24 13:00:20 2011 +0100
@@ -57,7 +57,7 @@
     final Log log;
     final Names names;
     final Source source;
-    final Keywords keywords;
+    final Tokens tokens;
 
     /** Create a new scanner factory. */
     protected ScannerFactory(Context context) {
@@ -65,14 +65,14 @@
         this.log = Log.instance(context);
         this.names = Names.instance(context);
         this.source = Source.instance(context);
-        this.keywords = Keywords.instance(context);
+        this.tokens = Tokens.instance(context);
     }
 
     public Scanner newScanner(CharSequence input, boolean keepDocComments) {
         if (input instanceof CharBuffer) {
             CharBuffer buf = (CharBuffer) input;
             if (keepDocComments)
-                return new DocCommentScanner(this, buf);
+                return new Scanner(this, new JavadocTokenizer(this, buf));
             else
                 return new Scanner(this, buf);
         } else {
@@ -83,7 +83,7 @@
 
     public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
         if (keepDocComments)
-            return new DocCommentScanner(this, input, inputLength);
+            return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
         else
             return new Scanner(this, input, inputLength);
     }
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Token.java	Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,198 +0,0 @@
-/*
- * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.  Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.util.Locale;
-
-import com.sun.tools.javac.api.Formattable;
-import com.sun.tools.javac.api.Messages;
-
-/** An interface that defines codes for Java source tokens
- *  returned from lexical analysis.
- *
- *  <p><b>This is NOT part of any supported API.
- *  If you write code that depends on this, you do so at your own risk.
- *  This code and its internal interfaces are subject to change or
- *  deletion without notice.</b>
- */
-public enum Token implements Formattable {
-    EOF,
-    ERROR,
-    IDENTIFIER,
-    ABSTRACT("abstract"),
-    ASSERT("assert"),
-    BOOLEAN("boolean"),
-    BREAK("break"),
-    BYTE("byte"),
-    CASE("case"),
-    CATCH("catch"),
-    CHAR("char"),
-    CLASS("class"),
-    CONST("const"),
-    CONTINUE("continue"),
-    DEFAULT("default"),
-    DO("do"),
-    DOUBLE("double"),
-    ELSE("else"),
-    ENUM("enum"),
-    EXTENDS("extends"),
-    FINAL("final"),
-    FINALLY("finally"),
-    FLOAT("float"),
-    FOR("for"),
-    GOTO("goto"),
-    IF("if"),
-    IMPLEMENTS("implements"),
-    IMPORT("import"),
-    INSTANCEOF("instanceof"),
-    INT("int"),
-    INTERFACE("interface"),
-    LONG("long"),
-    NATIVE("native"),
-    NEW("new"),
-    PACKAGE("package"),
-    PRIVATE("private"),
-    PROTECTED("protected"),
-    PUBLIC("public"),
-    RETURN("return"),
-    SHORT("short"),
-    STATIC("static"),
-    STRICTFP("strictfp"),
-    SUPER("super"),
-    SWITCH("switch"),
-    SYNCHRONIZED("synchronized"),
-    THIS("this"),
-    THROW("throw"),
-    THROWS("throws"),
-    TRANSIENT("transient"),
-    TRY("try"),
-    VOID("void"),
-    VOLATILE("volatile"),
-    WHILE("while"),
-    INTLITERAL,
-    LONGLITERAL,
-    FLOATLITERAL,
-    DOUBLELITERAL,
-    CHARLITERAL,
-    STRINGLITERAL,
-    TRUE("true"),
-    FALSE("false"),
-    NULL("null"),
-    LPAREN("("),
-    RPAREN(")"),
-    LBRACE("{"),
-    RBRACE("}"),
-    LBRACKET("["),
-    RBRACKET("]"),
-    SEMI(";"),
-    COMMA(","),
-    DOT("."),
-    ELLIPSIS("..."),
-    EQ("="),
-    GT(">"),
-    LT("<"),
-    BANG("!"),
-    TILDE("~"),
-    QUES("?"),
-    COLON(":"),
-    EQEQ("=="),
-    LTEQ("<="),
-    GTEQ(">="),
-    BANGEQ("!="),
-    AMPAMP("&&"),
-    BARBAR("||"),
-    PLUSPLUS("++"),
-    SUBSUB("--"),
-    PLUS("+"),
-    SUB("-"),
-    STAR("*"),
-    SLASH("/"),
-    AMP("&"),
-    BAR("|"),
-    CARET("^"),
-    PERCENT("%"),
-    LTLT("<<"),
-    GTGT(">>"),
-    GTGTGT(">>>"),
-    PLUSEQ("+="),
-    SUBEQ("-="),
-    STAREQ("*="),
-    SLASHEQ("/="),
-    AMPEQ("&="),
-    BAREQ("|="),
-    CARETEQ("^="),
-    PERCENTEQ("%="),
-    LTLTEQ("<<="),
-    GTGTEQ(">>="),
-    GTGTGTEQ(">>>="),
-    MONKEYS_AT("@"),
-    CUSTOM;
-
-    Token() {
-        this(null);
-    }
-    Token(String name) {
-        this.name = name;
-    }
-
-    public final String name;
-
-    public String toString() {
-        switch (this) {
-        case IDENTIFIER:
-            return "token.identifier";
-        case CHARLITERAL:
-            return "token.character";
-        case STRINGLITERAL:
-            return "token.string";
-        case INTLITERAL:
-            return "token.integer";
-        case LONGLITERAL:
-            return "token.long-integer";
-        case FLOATLITERAL:
-            return "token.float";
-        case DOUBLELITERAL:
-            return "token.double";
-        case ERROR:
-            return "token.bad-symbol";
-        case EOF:
-            return "token.end-of-input";
-        case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
-        case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
-            return "'" + name + "'";
-        default:
-            return name;
-        }
-    }
-
-    public String getKind() {
-        return "Token";
-    }
-
-    public String toString(Locale locale, Messages messages) {
-        return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
-    }
-}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.Locale;
+
+import com.sun.tools.javac.api.Formattable;
+import com.sun.tools.javac.api.Messages;
+import com.sun.tools.javac.parser.Tokens.Token.Tag;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Names;
+
+/** A class that defines codes/utilities for Java source tokens
+ *  returned from lexical analysis.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class Tokens {
+
+    private final Names names;
+
+    /**
+     * Keyword array. Maps name indices to Token.
+     */
+    private final TokenKind[] key;
+
+    /**  The number of the last entered keyword.
+     */
+    private int maxKey = 0;
+
+    /** The names of all tokens.
+     */
+    private Name[] tokenName = new Name[TokenKind.values().length];
+
+    public static final Context.Key<Tokens> tokensKey =
+        new Context.Key<Tokens>();
+
+    public static Tokens instance(Context context) {
+        Tokens instance = context.get(tokensKey);
+        if (instance == null)
+            instance = new Tokens(context);
+        return instance;
+    }
+
+    protected Tokens(Context context) {
+        context.put(tokensKey, this);
+        names = Names.instance(context);
+
+        for (TokenKind t : TokenKind.values()) {
+            if (t.name != null)
+                enterKeyword(t.name, t);
+            else
+                tokenName[t.ordinal()] = null;
+        }
+
+        key = new TokenKind[maxKey+1];
+        for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
+        for (TokenKind t : TokenKind.values()) {
+            if (t.name != null)
+            key[tokenName[t.ordinal()].getIndex()] = t;
+        }
+    }
+
+    private void enterKeyword(String s, TokenKind token) {
+        Name n = names.fromString(s);
+        tokenName[token.ordinal()] = n;
+        if (n.getIndex() > maxKey) maxKey = n.getIndex();
+    }
+
+    /**
+     * Create a new token given a name; if the name corresponds to a token name,
+     * a new token of the corresponding kind is returned; otherwise, an
+     * identifier token is returned.
+     */
+    TokenKind lookupKind(Name name) {
+        return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
+    }
+
+    TokenKind lookupKind(String name) {
+        return lookupKind(names.fromString(name));
+    }
+
+    /**
+     * This enum defines all tokens used by the javac scanner. A token is
+     * optionally associated with a name.
+     */
+    public enum TokenKind implements Formattable {
+        EOF(),
+        ERROR(),
+        IDENTIFIER(Tag.NAMED),
+        ABSTRACT("abstract"),
+        ASSERT("assert", Tag.NAMED),
+        BOOLEAN("boolean", Tag.NAMED),
+        BREAK("break"),
+        BYTE("byte", Tag.NAMED),
+        CASE("case"),
+        CATCH("catch"),
+        CHAR("char", Tag.NAMED),
+        CLASS("class"),
+        CONST("const"),
+        CONTINUE("continue"),
+        DEFAULT("default"),
+        DO("do"),
+        DOUBLE("double", Tag.NAMED),
+        ELSE("else"),
+        ENUM("enum", Tag.NAMED),
+        EXTENDS("extends"),
+        FINAL("final"),
+        FINALLY("finally"),
+        FLOAT("float", Tag.NAMED),
+        FOR("for"),
+        GOTO("goto"),
+        IF("if"),
+        IMPLEMENTS("implements"),
+        IMPORT("import"),
+        INSTANCEOF("instanceof"),
+        INT("int", Tag.NAMED),
+        INTERFACE("interface"),
+        LONG("long", Tag.NAMED),
+        NATIVE("native"),
+        NEW("new"),
+        PACKAGE("package"),
+        PRIVATE("private"),
+        PROTECTED("protected"),
+        PUBLIC("public"),
+        RETURN("return"),
+        SHORT("short", Tag.NAMED),
+        STATIC("static"),
+        STRICTFP("strictfp"),
+        SUPER("super", Tag.NAMED),
+        SWITCH("switch"),
+        SYNCHRONIZED("synchronized"),
+        THIS("this", Tag.NAMED),
+        THROW("throw"),
+        THROWS("throws"),
+        TRANSIENT("transient"),
+        TRY("try"),
+        VOID("void", Tag.NAMED),
+        VOLATILE("volatile"),
+        WHILE("while"),
+        INTLITERAL(Tag.NUMERIC),
+        LONGLITERAL(Tag.NUMERIC),
+        FLOATLITERAL(Tag.NUMERIC),
+        DOUBLELITERAL(Tag.NUMERIC),
+        CHARLITERAL(Tag.NUMERIC),
+        STRINGLITERAL(Tag.STRING),
+        TRUE("true", Tag.NAMED),
+        FALSE("false", Tag.NAMED),
+        NULL("null", Tag.NAMED),
+        LPAREN("("),
+        RPAREN(")"),
+        LBRACE("{"),
+        RBRACE("}"),
+        LBRACKET("["),
+        RBRACKET("]"),
+        SEMI(";"),
+        COMMA(","),
+        DOT("."),
+        ELLIPSIS("..."),
+        EQ("="),
+        GT(">"),
+        LT("<"),
+        BANG("!"),
+        TILDE("~"),
+        QUES("?"),
+        COLON(":"),
+        EQEQ("=="),
+        LTEQ("<="),
+        GTEQ(">="),
+        BANGEQ("!="),
+        AMPAMP("&&"),
+        BARBAR("||"),
+        PLUSPLUS("++"),
+        SUBSUB("--"),
+        PLUS("+"),
+        SUB("-"),
+        STAR("*"),
+        SLASH("/"),
+        AMP("&"),
+        BAR("|"),
+        CARET("^"),
+        PERCENT("%"),
+        LTLT("<<"),
+        GTGT(">>"),
+        GTGTGT(">>>"),
+        PLUSEQ("+="),
+        SUBEQ("-="),
+        STAREQ("*="),
+        SLASHEQ("/="),
+        AMPEQ("&="),
+        BAREQ("|="),
+        CARETEQ("^="),
+        PERCENTEQ("%="),
+        LTLTEQ("<<="),
+        GTGTEQ(">>="),
+        GTGTGTEQ(">>>="),
+        MONKEYS_AT("@"),
+        CUSTOM;
+
+        public final String name;
+        public final Tag tag;
+
+        TokenKind() {
+            this(null, Tag.DEFAULT);
+        }
+
+        TokenKind(String name) {
+            this(name, Tag.DEFAULT);
+        }
+
+        TokenKind(Tag tag) {
+            this(null, tag);
+        }
+
+        TokenKind(String name, Tag tag) {
+            this.name = name;
+            this.tag = tag;
+        }
+
+        public String toString() {
+            switch (this) {
+            case IDENTIFIER:
+                return "token.identifier";
+            case CHARLITERAL:
+                return "token.character";
+            case STRINGLITERAL:
+                return "token.string";
+            case INTLITERAL:
+                return "token.integer";
+            case LONGLITERAL:
+                return "token.long-integer";
+            case FLOATLITERAL:
+                return "token.float";
+            case DOUBLELITERAL:
+                return "token.double";
+            case ERROR:
+                return "token.bad-symbol";
+            case EOF:
+                return "token.end-of-input";
+            case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
+            case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
+                return "'" + name + "'";
+            default:
+                return name;
+            }
+        }
+
+        public String getKind() {
+            return "Token";
+        }
+
+        public String toString(Locale locale, Messages messages) {
+            return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
+        }
+    }
+
+    /**
+     * This is the class representing a javac token. Each token has several fields
+     * that are set by the javac lexer (i.e. start/end position, string value, etc).
+     */
+    public static class Token {
+
+        /** tags constants **/
+        enum Tag {
+            DEFAULT,
+            NAMED,
+            STRING,
+            NUMERIC;
+        }
+
+        /** The token kind */
+        public final TokenKind kind;
+
+        /** The start position of this token */
+        public final int pos;
+
+        /** The end position of this token */
+        public final int endPos;
+
+        /** Is this token preceeded by a deprecated comment? */
+        public final boolean deprecatedFlag;
+
+        /** Is this token preceeded by a deprecated comment? */
+        public String docComment;
+
+        Token(TokenKind kind, int pos, int endPos,
+                boolean deprecatedFlag) {
+            this.kind = kind;
+            this.pos = pos;
+            this.endPos = endPos;
+            this.deprecatedFlag = deprecatedFlag;
+            checkKind();
+        }
+
+        Token[] split(Tokens tokens) {
+            if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
+                throw new AssertionError("Cant split" + kind);
+            }
+
+            TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
+            TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
+
+            if (t1 == null || t2 == null) {
+                throw new AssertionError("Cant split - bad subtokens");
+            }
+            return new Token[] {
+                new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
+                new Token(t2, pos + t1.name.length(), endPos, false)
+            };
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.DEFAULT) {
+                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+            }
+        }
+
+        public Name name() {
+            throw new UnsupportedOperationException();
+        }
+
+        public String stringVal() {
+            throw new UnsupportedOperationException();
+        }
+
+        public int radix() {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    final static class NamedToken extends Token {
+        /** The name of this token */
+        public final Name name;
+
+        public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
+            super(kind, pos, endPos, deprecatedFlag);
+            this.name = name;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.NAMED) {
+                throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
+            }
+        }
+
+        @Override
+        public Name name() {
+            return name;
+        }
+    }
+
+    static class StringToken extends Token {
+        /** The string value of this token */
+        public final String stringVal;
+
+        public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
+            super(kind, pos, endPos, deprecatedFlag);
+            this.stringVal = stringVal;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.STRING) {
+                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+            }
+        }
+
+        @Override
+        public String stringVal() {
+            return stringVal;
+        }
+    }
+
+    final static class NumericToken extends StringToken {
+        /** The 'radix' value of this token */
+        public final int radix;
+
+        public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
+            super(kind, pos, endPos, stringVal, deprecatedFlag);
+            this.radix = radix;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.NUMERIC) {
+                throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
+            }
+        }
+
+        @Override
+        public int radix() {
+            return radix;
+        }
+    }
+
+    public static final Token DUMMY =
+                new Token(TokenKind.ERROR, 0, 0, false);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.file.JavacFileManager;
+import java.nio.CharBuffer;
+import com.sun.tools.javac.util.Log;
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
+ * characters contained in the input stream, handling unicode escape accordingly.
+ * Additionally, it provide features for saving chars into a buffer and to retrieve
+ * them at a later stage.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class UnicodeReader {
+
+    /** The input buffer, index of next character to be read,
+     *  index of one past last character in buffer.
+     */
+    protected char[] buf;
+    protected int bp;
+    protected final int buflen;
+
+    /** The current character.
+     */
+    protected char ch;
+
+    /** The buffer index of the last converted unicode character
+     */
+    protected int unicodeConversionBp = -1;
+
+    protected Log log;
+
+    /**
+     * Create a scanner from the input array.  This method might
+     * modify the array.  To avoid copying the input array, ensure
+     * that {@code inputLength < input.length} or
+     * {@code input[input.length -1]} is a white space character.
+     *
+     * @param fac the factory which created this Scanner
+     * @param input the input, might be modified
+     * @param inputLength the size of the input.
+     * Must be positive and less than or equal to input.length.
+     */
+    protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
+        this(sf, JavacFileManager.toArray(buffer), buffer.limit());
+    }
+
+    protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
+        log = sf.log;
+        if (inputLength == input.length) {
+            if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
+                inputLength--;
+            } else {
+                char[] newInput = new char[inputLength + 1];
+                System.arraycopy(input, 0, newInput, 0, input.length);
+                input = newInput;
+            }
+        }
+        buf = input;
+        buflen = inputLength;
+        buf[buflen] = EOI;
+        bp = -1;
+        scanChar();
+    }
+
+    /** Read next character.
+     */
+    protected void scanChar() {
+        if (bp < buflen) {
+            ch = buf[++bp];
+            if (ch == '\\') {
+                convertUnicode();
+            }
+        }
+    }
+
+    /** Convert unicode escape; bp points to initial '\' character
+     *  (Spec 3.3).
+     */
+    protected void convertUnicode() {
+        if (ch == '\\' && unicodeConversionBp != bp) {
+            bp++; ch = buf[bp];
+            if (ch == 'u') {
+                do {
+                    bp++; ch = buf[bp];
+                } while (ch == 'u');
+                int limit = bp + 3;
+                if (limit < buflen) {
+                    int d = digit(bp, 16);
+                    int code = d;
+                    while (bp < limit && d >= 0) {
+                        bp++; ch = buf[bp];
+                        d = digit(bp, 16);
+                        code = (code << 4) + d;
+                    }
+                    if (d >= 0) {
+                        ch = (char)code;
+                        unicodeConversionBp = bp;
+                        return;
+                    }
+                }
+                log.error(bp, "illegal.unicode.esc");
+            } else {
+                bp--;
+                ch = '\\';
+            }
+        }
+    }
+
+    /** Are surrogates supported?
+     */
+    final static boolean surrogatesSupported = surrogatesSupported();
+    private static boolean surrogatesSupported() {
+        try {
+            Character.isHighSurrogate('a');
+            return true;
+        } catch (NoSuchMethodError ex) {
+            return false;
+        }
+    }
+
+    /** Scan surrogate pairs.  If 'ch' is a high surrogate and
+     *  the next character is a low surrogate, then put the low
+     *  surrogate in 'ch', and return the high surrogate.
+     *  otherwise, just return 0.
+     */
+    protected char scanSurrogates() {
+        if (surrogatesSupported && Character.isHighSurrogate(ch)) {
+            char high = ch;
+
+            scanChar();
+
+            if (Character.isLowSurrogate(ch)) {
+                return high;
+            }
+
+            ch = high;
+        }
+
+        return 0;
+    }
+
+    /** Convert an ASCII digit from its base (8, 10, or 16)
+     *  to its value.
+     */
+    protected int digit(int pos, int base) {
+        char c = ch;
+        int result = Character.digit(c, base);
+        if (result >= 0 && c > 0x7f) {
+            log.error(pos + 1, "illegal.nonascii.digit");
+            ch = "0123456789abcdef".charAt(result);
+        }
+        return result;
+    }
+
+    protected boolean isUnicode() {
+        return unicodeConversionBp == bp;
+    }
+
+    protected void skipChar() {
+        bp++;
+    }
+
+    protected char peekChar() {
+        return buf[bp + 1];
+    }
+
+    /**
+     * Returns a copy of the input buffer, up to its inputLength.
+     * Unicode escape sequences are not translated.
+     */
+    public char[] getRawCharacters() {
+        char[] chars = new char[buflen];
+        System.arraycopy(buf, 0, chars, 0, buflen);
+        return chars;
+    }
+
+    /**
+     * Returns a copy of a character array subset of the input buffer.
+     * The returned array begins at the <code>beginIndex</code> and
+     * extends to the character at index <code>endIndex - 1</code>.
+     * Thus the length of the substring is <code>endIndex-beginIndex</code>.
+     * This behavior is like
+     * <code>String.substring(beginIndex, endIndex)</code>.
+     * Unicode escape sequences are not translated.
+     *
+     * @param beginIndex the beginning index, inclusive.
+     * @param endIndex the ending index, exclusive.
+     * @throws IndexOutOfBounds if either offset is outside of the
+     *         array bounds
+     */
+    public char[] getRawCharacters(int beginIndex, int endIndex) {
+        int length = endIndex - beginIndex;
+        char[] chars = new char[length];
+        System.arraycopy(buf, beginIndex, chars, 0, length);
+        return chars;
+    }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java	Mon Oct 24 13:00:20 2011 +0100
@@ -1072,9 +1072,9 @@
             Assert.checkNonNull(names);
             next.put(Names.namesKey, names);
 
-            Keywords keywords = Keywords.instance(context);
-            Assert.checkNonNull(keywords);
-            next.put(Keywords.keywordsKey, keywords);
+            Tokens tokens = Tokens.instance(context);
+            Assert.checkNonNull(tokens);
+            next.put(Tokens.tokensKey, tokens);
 
             JavaCompiler oldCompiler = JavaCompiler.instance(context);
             JavaCompiler nextCompiler = JavaCompiler.instance(next);
--- a/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java	Mon Oct 24 13:00:20 2011 +0100
@@ -39,7 +39,6 @@
 
 import com.sun.tools.javac.code.Symbol.CompletionFailure;
 import com.sun.tools.javac.comp.Annotate;
-import com.sun.tools.javac.parser.DocCommentScanner;
 import com.sun.tools.javac.tree.JCTree;
 import com.sun.tools.javac.tree.JCTree.JCClassDecl;
 import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
--- a/langtools/test/tools/javac/api/TestJavacTaskScanner.java	Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/test/tools/javac/api/TestJavacTaskScanner.java	Mon Oct 24 13:00:20 2011 +0100
@@ -32,6 +32,7 @@
 
 import com.sun.tools.javac.api.JavacTaskImpl;
 import com.sun.tools.javac.parser.*;
+import com.sun.tools.javac.parser.Tokens.Token;
 import com.sun.tools.javac.util.*;
 import java.io.*;
 import java.net.*;
@@ -93,7 +94,7 @@
 
         check(numTokens, "#Tokens", 1222);
         check(numParseTypeElements, "#parseTypeElements", 136);
-        check(numAllMembers, "#allMembers", 67);
+        check(numAllMembers, "#allMembers", 52);
     }
 
     void check(int value, String name, int expected) {
@@ -206,7 +207,8 @@
 
     public void nextToken() {
         super.nextToken();
-        System.err.format("Saw token %s (%s)%n", token(), name());
+        Token tk = token();
+        System.err.format("Saw token %s %n", tk.kind);
         test.numTokens++;
     }
 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ * @compile -Xlint -Werror DeprecatedDocComment3.java
+ */
+
+class DeprecatedDocComment3 {
+    static class Foo { }
+
+    ; /** @deprecated */ ;
+
+    static class A {}
+
+    static class B {
+       A a; //not deprecated!
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/test/tools/javac/tree/DocCommentToplevelTest.java	Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ */
+
+import com.sun.source.tree.*;
+import com.sun.source.util.*;
+import com.sun.tools.javac.tree.JCTree;
+
+import java.net.URI;
+import java.util.*;
+import javax.tools.*;
+
+
+public class DocCommentToplevelTest {
+
+    enum PackageKind {
+        HAS_PKG("package pkg;"),
+        NO_PKG("");
+
+        String pkgStr;
+
+        PackageKind(String pkgStr) {
+            this.pkgStr = pkgStr;
+        }
+    }
+
+    enum ImportKind {
+        ZERO(""),
+        ONE("import java.lang.*;"),
+        TWO("import java.lang.*; import java.util.*;");
+
+        String importStr;
+
+        ImportKind(String importStr) {
+            this.importStr = importStr;
+        }
+    }
+
+    enum ModifierKind {
+        DEFAULT(""),
+        PUBLIC("public");
+
+        String modStr;
+
+        ModifierKind(String modStr) {
+            this.modStr = modStr;
+        }
+    }
+
+    enum ToplevelDocKind {
+        HAS_DOC("/** Toplevel! */"),
+        NO_DOC("");
+
+        String docStr;
+
+        ToplevelDocKind(String docStr) {
+            this.docStr = docStr;
+        }
+    }
+
+    static int errors;
+    static int checks;
+
+    public static void main(String... args) throws Exception {
+        //create default shared JavaCompiler - reused across multiple compilations
+        JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
+        StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
+
+        for (PackageKind pk : PackageKind.values()) {
+            for (ImportKind ik : ImportKind.values()) {
+                for (ModifierKind mk1 : ModifierKind.values()) {
+                    for (ModifierKind mk2 : ModifierKind.values()) {
+                        for (ToplevelDocKind tdk : ToplevelDocKind.values()) {
+                            new DocCommentToplevelTest(pk, ik, mk1, mk2, tdk).run(comp, fm);
+                        }
+                    }
+                }
+            }
+        }
+
+        if (errors > 0)
+            throw new AssertionError(errors + " errors found");
+
+        System.out.println(checks + " checks were made");
+    }
+
+    PackageKind pk;
+    ImportKind ik;
+    ModifierKind mk1;
+    ModifierKind mk2;
+    ToplevelDocKind tdk;
+    JavaSource source;
+
+    DocCommentToplevelTest(PackageKind pk, ImportKind ik, ModifierKind mk1, ModifierKind mk2, ToplevelDocKind tdk) {
+        this.pk = pk;
+        this.ik = ik;
+        this.mk1 = mk1;
+        this.mk2 = mk2;
+        this.tdk = tdk;
+        source = new JavaSource();
+    }
+
+    void run(JavaCompiler comp, JavaFileManager fm) throws Exception {
+        JavacTask task = (JavacTask)comp.getTask(null, fm, null, Arrays.asList("-printsource"), null, Arrays.asList(source));
+        for (CompilationUnitTree cu: task.parse()) {
+            check(cu);
+        }
+    }
+
+    void check(CompilationUnitTree cu) {
+        checks++;
+
+        new TreeScanner<ClassTree,Void>() {
+
+            Map<JCTree, String> docComments;
+
+            @Override
+            public ClassTree visitCompilationUnit(CompilationUnitTree node, Void unused) {
+                docComments = ((JCTree.JCCompilationUnit)node).docComments;
+                boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
+                        (pk != PackageKind.NO_PKG || ik != ImportKind.ZERO);
+                boolean foundComment = docComments.get(node) != null;
+                if (expectedComment != foundComment) {
+                    error("Unexpected comment " + docComments.get(node) + " on toplevel");
+                }
+                return super.visitCompilationUnit(node, null);
+            }
+
+            @Override
+            public ClassTree visitClass(ClassTree node, Void unused) {
+                boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
+                        pk == PackageKind.NO_PKG && ik == ImportKind.ZERO &&
+                        node.getSimpleName().toString().equals("First");
+                boolean foundComment = docComments.get(node) != null;
+                if (expectedComment != foundComment) {
+                    error("Unexpected comment " + docComments.get(node) + " on class " + node.getSimpleName());
+                }
+                return super.visitClass(node, unused);
+            }
+        }.scan(cu, null);
+    }
+
+    void error(String msg) {
+        System.err.println("Error: " + msg);
+        System.err.println("Source: " + source.source);
+        errors++;
+    }
+
+    class JavaSource extends SimpleJavaFileObject {
+
+        String template = "#D\n#P\n#I\n" +
+                          "#M1 class First { }\n" +
+                          "#M2 class Second { }\n";
+
+        String source;
+
+        public JavaSource() {
+            super(URI.create("myfo:/Test.java"), JavaFileObject.Kind.SOURCE);
+            source = template.replace("#P", pk.pkgStr)
+                             .replace("#I", ik.importStr)
+                             .replace("#M1", mk1.modStr)
+                             .replace("#M2", mk2.modStr)
+                             .replace("#D", tdk.docStr);
+        }
+
+        @Override
+        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+            return source;
+        }
+    }
+}