7096014: Javac tokens should retain state
Summary: Refactor javac tokens from enum constants to stateful instances (to keep track of position, comments, etc.)
Reviewed-by: jjg
--- a/langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/apt/main/AptJavaCompiler.java Mon Oct 24 13:00:20 2011 +0100
@@ -42,7 +42,6 @@
import com.sun.tools.apt.comp.*;
import com.sun.tools.apt.util.Bark;
import com.sun.mirror.apt.AnnotationProcessorFactory;
-import com.sun.tools.javac.parser.DocCommentScanner;
/**
* <p><b>This is NOT part of any supported API.
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/DocCommentScanner.java Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,420 +0,0 @@
-/*
- * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation. Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.nio.*;
-
-import com.sun.tools.javac.util.*;
-import static com.sun.tools.javac.util.LayoutCharacters.*;
-
-/** An extension to the base lexical analyzer that captures
- * and processes the contents of doc comments. It does so by
- * translating Unicode escape sequences and by stripping the
- * leading whitespace and starts from each line of the comment.
- *
- * <p><b>This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.</b>
- */
-public class DocCommentScanner extends Scanner {
-
- /** Create a scanner from the input buffer. buffer must implement
- * array() and compact(), and remaining() must be less than limit().
- */
- protected DocCommentScanner(ScannerFactory fac, CharBuffer buffer) {
- super(fac, buffer);
- }
-
- /** Create a scanner from the input array. The array must have at
- * least a single character of extra space.
- */
- protected DocCommentScanner(ScannerFactory fac, char[] input, int inputLength) {
- super(fac, input, inputLength);
- }
-
- /** Starting position of the comment in original source
- */
- private int pos;
-
- /** The comment input buffer, index of next chacter to be read,
- * index of one past last character in buffer.
- */
- private char[] buf;
- private int bp;
- private int buflen;
-
- /** The current character.
- */
- private char ch;
-
- /** The column number position of the current character.
- */
- private int col;
-
- /** The buffer index of the last converted Unicode character
- */
- private int unicodeConversionBp = 0;
-
- /**
- * Buffer for doc comment.
- */
- private char[] docCommentBuffer = new char[1024];
-
- /**
- * Number of characters in doc comment buffer.
- */
- private int docCommentCount;
-
- /**
- * Translated and stripped contents of doc comment
- */
- private String docComment = null;
-
-
- /** Unconditionally expand the comment buffer.
- */
- private void expandCommentBuffer() {
- char[] newBuffer = new char[docCommentBuffer.length * 2];
- System.arraycopy(docCommentBuffer, 0, newBuffer,
- 0, docCommentBuffer.length);
- docCommentBuffer = newBuffer;
- }
-
- /** Convert an ASCII digit from its base (8, 10, or 16)
- * to its value.
- */
- private int digit(int base) {
- char c = ch;
- int result = Character.digit(c, base);
- if (result >= 0 && c > 0x7f) {
- ch = "0123456789abcdef".charAt(result);
- }
- return result;
- }
-
- /** Convert Unicode escape; bp points to initial '\' character
- * (Spec 3.3).
- */
- private void convertUnicode() {
- if (ch == '\\' && unicodeConversionBp != bp) {
- bp++; ch = buf[bp]; col++;
- if (ch == 'u') {
- do {
- bp++; ch = buf[bp]; col++;
- } while (ch == 'u');
- int limit = bp + 3;
- if (limit < buflen) {
- int d = digit(16);
- int code = d;
- while (bp < limit && d >= 0) {
- bp++; ch = buf[bp]; col++;
- d = digit(16);
- code = (code << 4) + d;
- }
- if (d >= 0) {
- ch = (char)code;
- unicodeConversionBp = bp;
- return;
- }
- }
- // "illegal.Unicode.esc", reported by base scanner
- } else {
- bp--;
- ch = '\\';
- col--;
- }
- }
- }
-
-
- /** Read next character.
- */
- private void scanChar() {
- bp++;
- ch = buf[bp];
- switch (ch) {
- case '\r': // return
- col = 0;
- break;
- case '\n': // newline
- if (bp == 0 || buf[bp-1] != '\r') {
- col = 0;
- }
- break;
- case '\t': // tab
- col = (col / TabInc * TabInc) + TabInc;
- break;
- case '\\': // possible Unicode
- col++;
- convertUnicode();
- break;
- default:
- col++;
- break;
- }
- }
-
- /**
- * Read next character in doc comment, skipping over double '\' characters.
- * If a double '\' is skipped, put in the buffer and update buffer count.
- */
- private void scanDocCommentChar() {
- scanChar();
- if (ch == '\\') {
- if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = ch;
- bp++; col++;
- } else {
- convertUnicode();
- }
- }
- }
-
- /* Reset doc comment before reading each new token
- */
- public void nextToken() {
- docComment = null;
- super.nextToken();
- }
-
- /**
- * Returns the documentation string of the current token.
- */
- public String docComment() {
- return docComment;
- }
-
- /**
- * Process a doc comment and make the string content available.
- * Strips leading whitespace and stars.
- */
- @SuppressWarnings("fallthrough")
- protected void processComment(CommentStyle style) {
- if (style != CommentStyle.JAVADOC) {
- return;
- }
-
- pos = pos();
- buf = getRawCharacters(pos, endPos());
- buflen = buf.length;
- bp = 0;
- col = 0;
-
- docCommentCount = 0;
-
- boolean firstLine = true;
-
- // Skip over first slash
- scanDocCommentChar();
- // Skip over first star
- scanDocCommentChar();
-
- // consume any number of stars
- while (bp < buflen && ch == '*') {
- scanDocCommentChar();
- }
- // is the comment in the form /**/, /***/, /****/, etc. ?
- if (bp < buflen && ch == '/') {
- docComment = "";
- return;
- }
-
- // skip a newline on the first line of the comment.
- if (bp < buflen) {
- if (ch == LF) {
- scanDocCommentChar();
- firstLine = false;
- } else if (ch == CR) {
- scanDocCommentChar();
- if (ch == LF) {
- scanDocCommentChar();
- firstLine = false;
- }
- }
- }
-
- outerLoop:
-
- // The outerLoop processes the doc comment, looping once
- // for each line. For each line, it first strips off
- // whitespace, then it consumes any stars, then it
- // puts the rest of the line into our buffer.
- while (bp < buflen) {
-
- // The wsLoop consumes whitespace from the beginning
- // of each line.
- wsLoop:
-
- while (bp < buflen) {
- switch(ch) {
- case ' ':
- scanDocCommentChar();
- break;
- case '\t':
- col = ((col - 1) / TabInc * TabInc) + TabInc;
- scanDocCommentChar();
- break;
- case FF:
- col = 0;
- scanDocCommentChar();
- break;
-// Treat newline at beginning of line (blank line, no star)
-// as comment text. Old Javadoc compatibility requires this.
-/*---------------------------------*
- case CR: // (Spec 3.4)
- scanDocCommentChar();
- if (ch == LF) {
- col = 0;
- scanDocCommentChar();
- }
- break;
- case LF: // (Spec 3.4)
- scanDocCommentChar();
- break;
-*---------------------------------*/
- default:
- // we've seen something that isn't whitespace;
- // jump out.
- break wsLoop;
- }
- }
-
- // Are there stars here? If so, consume them all
- // and check for the end of comment.
- if (ch == '*') {
- // skip all of the stars
- do {
- scanDocCommentChar();
- } while (ch == '*');
-
- // check for the closing slash.
- if (ch == '/') {
- // We're done with the doc comment
- // scanChar() and breakout.
- break outerLoop;
- }
- } else if (! firstLine) {
- //The current line does not begin with a '*' so we will indent it.
- for (int i = 1; i < col; i++) {
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = ' ';
- }
- }
-
- // The textLoop processes the rest of the characters
- // on the line, adding them to our buffer.
- textLoop:
- while (bp < buflen) {
- switch (ch) {
- case '*':
- // Is this just a star? Or is this the
- // end of a comment?
- scanDocCommentChar();
- if (ch == '/') {
- // This is the end of the comment,
- // set ch and return our buffer.
- break outerLoop;
- }
- // This is just an ordinary star. Add it to
- // the buffer.
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = '*';
- break;
- case ' ':
- case '\t':
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = ch;
- scanDocCommentChar();
- break;
- case FF:
- scanDocCommentChar();
- break textLoop; // treat as end of line
- case CR: // (Spec 3.4)
- scanDocCommentChar();
- if (ch != LF) {
- // Canonicalize CR-only line terminator to LF
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = (char)LF;
- break textLoop;
- }
- /* fall through to LF case */
- case LF: // (Spec 3.4)
- // We've seen a newline. Add it to our
- // buffer and break out of this loop,
- // starting fresh on a new line.
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = ch;
- scanDocCommentChar();
- break textLoop;
- default:
- // Add the character to our buffer.
- if (docCommentCount == docCommentBuffer.length)
- expandCommentBuffer();
- docCommentBuffer[docCommentCount++] = ch;
- scanDocCommentChar();
- }
- } // end textLoop
- firstLine = false;
- } // end outerLoop
-
- if (docCommentCount > 0) {
- int i = docCommentCount - 1;
- trailLoop:
- while (i > -1) {
- switch (docCommentBuffer[i]) {
- case '*':
- i--;
- break;
- default:
- break trailLoop;
- }
- }
- docCommentCount = i + 1;
-
- // Store the text of the doc comment
- docComment = new String(docCommentBuffer, 0 , docCommentCount);
- } else {
- docComment = "";
- }
- }
-
- /** Build a map for translating between line numbers and
- * positions in the input.
- *
- * @return a LineMap */
- public Position.LineMap getLineMap() {
- char[] buf = getRawCharacters();
- return Position.makeLineMap(buf, buf.length, true);
- }
-}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/EndPosParser.java Mon Oct 24 13:00:20 2011 +0100
@@ -67,14 +67,14 @@
/** {@inheritDoc} */
@Override
protected <T extends JCTree> T to(T t) {
- storeEnd(t, S.endPos());
+ storeEnd(t, token.endPos);
return t;
}
/** {@inheritDoc} */
@Override
protected <T extends JCTree> T toP(T t) {
- storeEnd(t, S.prevEndPos());
+ storeEnd(t, S.prevToken().endPos);
return t;
}
@@ -88,7 +88,7 @@
/** {@inheritDoc} */
@Override
JCExpression parExpression() {
- int pos = S.pos();
+ int pos = token.pos;
JCExpression t = super.parExpression();
return toP(F.at(pos).Parens(t));
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,896 @@
+/*
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.nio.CharBuffer;
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.util.*;
+
+
+import static com.sun.tools.javac.parser.Tokens.*;
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The lexical analyzer maps an input stream consisting of
+ * ASCII characters and Unicode escapes into a token sequence.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public class JavaTokenizer {
+
+ private static boolean scannerDebug = false;
+
+ /** Allow hex floating-point literals.
+ */
+ private boolean allowHexFloats;
+
+ /** Allow binary literals.
+ */
+ private boolean allowBinaryLiterals;
+
+ /** Allow underscores in literals.
+ */
+ private boolean allowUnderscoresInLiterals;
+
+ /** The source language setting.
+ */
+ private Source source;
+
+ /** The log to be used for error reporting.
+ */
+ private final Log log;
+
+ /** The name table. */
+ private final Names names;
+
+ /** The token factory. */
+ private final Tokens tokens;
+
+ /** The token kind, set by nextToken().
+ */
+ protected TokenKind tk;
+
+ /** The token's radix, set by nextToken().
+ */
+ protected int radix;
+
+ /** The token's name, set by nextToken().
+ */
+ protected Name name;
+
+ /** The position where a lexical error occurred;
+ */
+ protected int errPos = Position.NOPOS;
+
+ /** Has a @deprecated been encountered in last doc comment?
+ * this needs to be reset by client.
+ */
+ protected boolean deprecatedFlag = false;
+
+ /** A character buffer for saved chars.
+ */
+ protected char[] sbuf = new char[128];
+ protected int sp;
+
+ protected UnicodeReader reader;
+
+ private static final boolean hexFloatsWork = hexFloatsWork();
+ private static boolean hexFloatsWork() {
+ try {
+ Float.valueOf("0x1.0p1");
+ return true;
+ } catch (NumberFormatException ex) {
+ return false;
+ }
+ }
+
+ /**
+ * Create a scanner from the input array. This method might
+ * modify the array. To avoid copying the input array, ensure
+ * that {@code inputLength < input.length} or
+ * {@code input[input.length -1]} is a white space character.
+ *
+ * @param fac the factory which created this Scanner
+ * @param input the input, might be modified
+ * @param inputLength the size of the input.
+ * Must be positive and less than or equal to input.length.
+ */
+ protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
+ this(fac, new UnicodeReader(fac, buf));
+ }
+
+ protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
+ this(fac, new UnicodeReader(fac, buf, inputLength));
+ }
+
+ protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
+ log = fac.log;
+ names = fac.names;
+ tokens = fac.tokens;
+ source = fac.source;
+ this.reader = reader;
+ allowBinaryLiterals = source.allowBinaryLiterals();
+ allowHexFloats = source.allowHexFloats();
+ allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+ }
+
+ /** Report an error at the given position using the provided arguments.
+ */
+ protected void lexError(int pos, String key, Object... args) {
+ log.error(pos, key, args);
+ tk = TokenKind.ERROR;
+ errPos = pos;
+ }
+
+ /** Read next character in comment, skipping over double '\' characters.
+ */
+ protected void scanCommentChar() {
+ reader.scanChar();
+ if (reader.ch == '\\') {
+ if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+ reader.skipChar();
+ } else {
+ reader.convertUnicode();
+ }
+ }
+ }
+
+ /** Append a character to sbuf.
+ */
+ private void putChar(char ch) {
+ if (sp == sbuf.length) {
+ char[] newsbuf = new char[sbuf.length * 2];
+ System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
+ sbuf = newsbuf;
+ }
+ sbuf[sp++] = ch;
+ }
+
+ /** Read next character in character or string literal and copy into sbuf.
+ */
+ private void scanLitChar(int pos) {
+ if (reader.ch == '\\') {
+ if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+ reader.skipChar();
+ putChar('\\');
+ reader.scanChar();
+ } else {
+ reader.scanChar();
+ switch (reader.ch) {
+ case '0': case '1': case '2': case '3':
+ case '4': case '5': case '6': case '7':
+ char leadch = reader.ch;
+ int oct = reader.digit(pos, 8);
+ reader.scanChar();
+ if ('0' <= reader.ch && reader.ch <= '7') {
+ oct = oct * 8 + reader.digit(pos, 8);
+ reader.scanChar();
+ if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
+ oct = oct * 8 + reader.digit(pos, 8);
+ reader.scanChar();
+ }
+ }
+ putChar((char)oct);
+ break;
+ case 'b':
+ putChar('\b'); reader.scanChar(); break;
+ case 't':
+ putChar('\t'); reader.scanChar(); break;
+ case 'n':
+ putChar('\n'); reader.scanChar(); break;
+ case 'f':
+ putChar('\f'); reader.scanChar(); break;
+ case 'r':
+ putChar('\r'); reader.scanChar(); break;
+ case '\'':
+ putChar('\''); reader.scanChar(); break;
+ case '\"':
+ putChar('\"'); reader.scanChar(); break;
+ case '\\':
+ putChar('\\'); reader.scanChar(); break;
+ default:
+ lexError(reader.bp, "illegal.esc.char");
+ }
+ }
+ } else if (reader.bp != reader.buflen) {
+ putChar(reader.ch); reader.scanChar();
+ }
+ }
+
+ private void scanDigits(int pos, int digitRadix) {
+ char saveCh;
+ int savePos;
+ do {
+ if (reader.ch != '_') {
+ putChar(reader.ch);
+ } else {
+ if (!allowUnderscoresInLiterals) {
+ lexError(pos, "unsupported.underscore.lit", source.name);
+ allowUnderscoresInLiterals = true;
+ }
+ }
+ saveCh = reader.ch;
+ savePos = reader.bp;
+ reader.scanChar();
+ } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
+ if (saveCh == '_')
+ lexError(savePos, "illegal.underscore");
+ }
+
+ /** Read fractional part of hexadecimal floating point number.
+ */
+ private void scanHexExponentAndSuffix(int pos) {
+ if (reader.ch == 'p' || reader.ch == 'P') {
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '+' || reader.ch == '-') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ if (!allowHexFloats) {
+ lexError(pos, "unsupported.fp.lit", source.name);
+ allowHexFloats = true;
+ }
+ else if (!hexFloatsWork)
+ lexError(pos, "unsupported.cross.fp.lit");
+ } else
+ lexError(pos, "malformed.fp.lit");
+ } else {
+ lexError(pos, "malformed.fp.lit");
+ }
+ if (reader.ch == 'f' || reader.ch == 'F') {
+ putChar(reader.ch);
+ reader.scanChar();
+ tk = TokenKind.FLOATLITERAL;
+ radix = 16;
+ } else {
+ if (reader.ch == 'd' || reader.ch == 'D') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ tk = TokenKind.DOUBLELITERAL;
+ radix = 16;
+ }
+ }
+
+ /** Read fractional part of floating point number.
+ */
+ private void scanFraction(int pos) {
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ }
+ int sp1 = sp;
+ if (reader.ch == 'e' || reader.ch == 'E') {
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '+' || reader.ch == '-') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ skipIllegalUnderscores();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ scanDigits(pos, 10);
+ return;
+ }
+ lexError(pos, "malformed.fp.lit");
+ sp = sp1;
+ }
+ }
+
+ /** Read fractional part and 'd' or 'f' suffix of floating point number.
+ */
+ private void scanFractionAndSuffix(int pos) {
+ radix = 10;
+ scanFraction(pos);
+ if (reader.ch == 'f' || reader.ch == 'F') {
+ putChar(reader.ch);
+ reader.scanChar();
+ tk = TokenKind.FLOATLITERAL;
+ } else {
+ if (reader.ch == 'd' || reader.ch == 'D') {
+ putChar(reader.ch);
+ reader.scanChar();
+ }
+ tk = TokenKind.DOUBLELITERAL;
+ }
+ }
+
+ /** Read fractional part and 'd' or 'f' suffix of floating point number.
+ */
+ private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
+ radix = 16;
+ Assert.check(reader.ch == '.');
+ putChar(reader.ch);
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.digit(pos, 16) >= 0) {
+ seendigit = true;
+ scanDigits(pos, 16);
+ }
+ if (!seendigit)
+ lexError(pos, "invalid.hex.number");
+ else
+ scanHexExponentAndSuffix(pos);
+ }
+
+ private void skipIllegalUnderscores() {
+ if (reader.ch == '_') {
+ lexError(reader.bp, "illegal.underscore");
+ while (reader.ch == '_')
+ reader.scanChar();
+ }
+ }
+
+ /** Read a number.
+ * @param radix The radix of the number; one of 2, j8, 10, 16.
+ */
+ private void scanNumber(int pos, int radix) {
+ // for octal, allow base-10 digit in case it's a float literal
+ this.radix = radix;
+ int digitRadix = (radix == 8 ? 10 : radix);
+ boolean seendigit = false;
+ if (reader.digit(pos, digitRadix) >= 0) {
+ seendigit = true;
+ scanDigits(pos, digitRadix);
+ }
+ if (radix == 16 && reader.ch == '.') {
+ scanHexFractionAndSuffix(pos, seendigit);
+ } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
+ scanHexExponentAndSuffix(pos);
+ } else if (digitRadix == 10 && reader.ch == '.') {
+ putChar(reader.ch);
+ reader.scanChar();
+ scanFractionAndSuffix(pos);
+ } else if (digitRadix == 10 &&
+ (reader.ch == 'e' || reader.ch == 'E' ||
+ reader.ch == 'f' || reader.ch == 'F' ||
+ reader.ch == 'd' || reader.ch == 'D')) {
+ scanFractionAndSuffix(pos);
+ } else {
+ if (reader.ch == 'l' || reader.ch == 'L') {
+ reader.scanChar();
+ tk = TokenKind.LONGLITERAL;
+ } else {
+ tk = TokenKind.INTLITERAL;
+ }
+ }
+ }
+
+ /** Read an identifier.
+ */
+ private void scanIdent() {
+ boolean isJavaIdentifierPart;
+ char high;
+ do {
+ if (sp == sbuf.length) putChar(reader.ch); else sbuf[sp++] = reader.ch;
+ // optimization, was: putChar(reader.ch);
+
+ reader.scanChar();
+ switch (reader.ch) {
+ case 'A': case 'B': case 'C': case 'D': case 'E':
+ case 'F': case 'G': case 'H': case 'I': case 'J':
+ case 'K': case 'L': case 'M': case 'N': case 'O':
+ case 'P': case 'Q': case 'R': case 'S': case 'T':
+ case 'U': case 'V': case 'W': case 'X': case 'Y':
+ case 'Z':
+ case 'a': case 'b': case 'c': case 'd': case 'e':
+ case 'f': case 'g': case 'h': case 'i': case 'j':
+ case 'k': case 'l': case 'm': case 'n': case 'o':
+ case 'p': case 'q': case 'r': case 's': case 't':
+ case 'u': case 'v': case 'w': case 'x': case 'y':
+ case 'z':
+ case '$': case '_':
+ case '0': case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ case '\u0000': case '\u0001': case '\u0002': case '\u0003':
+ case '\u0004': case '\u0005': case '\u0006': case '\u0007':
+ case '\u0008': case '\u000E': case '\u000F': case '\u0010':
+ case '\u0011': case '\u0012': case '\u0013': case '\u0014':
+ case '\u0015': case '\u0016': case '\u0017':
+ case '\u0018': case '\u0019': case '\u001B':
+ case '\u007F':
+ break;
+ case '\u001A': // EOI is also a legal identifier part
+ if (reader.bp >= reader.buflen) {
+ name = names.fromChars(sbuf, 0, sp);
+ tk = tokens.lookupKind(name);
+ return;
+ }
+ break;
+ default:
+ if (reader.ch < '\u0080') {
+ // all ASCII range chars already handled, above
+ isJavaIdentifierPart = false;
+ } else {
+ high = reader.scanSurrogates();
+ if (high != 0) {
+ if (sp == sbuf.length) {
+ putChar(high);
+ } else {
+ sbuf[sp++] = high;
+ }
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(
+ Character.toCodePoint(high, reader.ch));
+ } else {
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+ }
+ }
+ if (!isJavaIdentifierPart) {
+ name = names.fromChars(sbuf, 0, sp);
+ tk = tokens.lookupKind(name);
+ return;
+ }
+ }
+ } while (true);
+ }
+
+ /** Return true if reader.ch can be part of an operator.
+ */
+ private boolean isSpecial(char ch) {
+ switch (ch) {
+ case '!': case '%': case '&': case '*': case '?':
+ case '+': case '-': case ':': case '<': case '=':
+ case '>': case '^': case '|': case '~':
+ case '@':
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ /** Read longest possible sequence of special characters and convert
+ * to token.
+ */
+ private void scanOperator() {
+ while (true) {
+ putChar(reader.ch);
+ Name newname = names.fromChars(sbuf, 0, sp);
+ TokenKind tk1 = tokens.lookupKind(newname);
+ if (tk1 == TokenKind.IDENTIFIER) {
+ sp--;
+ break;
+ }
+ tk = tk1;
+ reader.scanChar();
+ if (!isSpecial(reader.ch)) break;
+ }
+ }
+
+ /**
+ * Scan a documentation comment; determine if a deprecated tag is present.
+ * Called once the initial /, * have been skipped, positioned at the second *
+ * (which is treated as the beginning of the first line).
+ * Stops positioned at the closing '/'.
+ */
+ @SuppressWarnings("fallthrough")
+ private void scanDocComment() {
+ boolean deprecatedPrefix = false;
+
+ forEachLine:
+ while (reader.bp < reader.buflen) {
+
+ // Skip optional WhiteSpace at beginning of line
+ while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+ scanCommentChar();
+ }
+
+ // Skip optional consecutive Stars
+ while (reader.bp < reader.buflen && reader.ch == '*') {
+ scanCommentChar();
+ if (reader.ch == '/') {
+ return;
+ }
+ }
+
+ // Skip optional WhiteSpace after Stars
+ while (reader.bp < reader.buflen && (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF)) {
+ scanCommentChar();
+ }
+
+ deprecatedPrefix = false;
+ // At beginning of line in the JavaDoc sense.
+ if (reader.bp < reader.buflen && reader.ch == '@' && !deprecatedFlag) {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'd') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'p') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'r') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'c') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'a') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 't') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'e') {
+ scanCommentChar();
+ if (reader.bp < reader.buflen && reader.ch == 'd') {
+ deprecatedPrefix = true;
+ scanCommentChar();
+ }}}}}}}}}}}
+ if (deprecatedPrefix && reader.bp < reader.buflen) {
+ if (Character.isWhitespace(reader.ch)) {
+ deprecatedFlag = true;
+ } else if (reader.ch == '*') {
+ scanCommentChar();
+ if (reader.ch == '/') {
+ deprecatedFlag = true;
+ return;
+ }
+ }
+ }
+
+ // Skip rest of line
+ while (reader.bp < reader.buflen) {
+ switch (reader.ch) {
+ case '*':
+ scanCommentChar();
+ if (reader.ch == '/') {
+ return;
+ }
+ break;
+ case CR: // (Spec 3.4)
+ scanCommentChar();
+ if (reader.ch != LF) {
+ continue forEachLine;
+ }
+ /* fall through to LF case */
+ case LF: // (Spec 3.4)
+ scanCommentChar();
+ continue forEachLine;
+ default:
+ scanCommentChar();
+ }
+ } // rest of line
+ } // forEachLine
+ return;
+ }
+
+ /** Read token.
+ */
+ public Token readToken() {
+
+ sp = 0;
+ name = null;
+ deprecatedFlag = false;
+ radix = 0;
+ int pos = 0;
+ int endPos = 0;
+
+ try {
+ loop: while (true) {
+ pos = reader.bp;
+ switch (reader.ch) {
+ case ' ': // (Spec 3.6)
+ case '\t': // (Spec 3.6)
+ case FF: // (Spec 3.6)
+ do {
+ reader.scanChar();
+ } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
+ processWhiteSpace(pos, reader.bp);
+ break;
+ case LF: // (Spec 3.4)
+ reader.scanChar();
+ processLineTerminator(pos, reader.bp);
+ break;
+ case CR: // (Spec 3.4)
+ reader.scanChar();
+ if (reader.ch == LF) {
+ reader.scanChar();
+ }
+ processLineTerminator(pos, reader.bp);
+ break;
+ case 'A': case 'B': case 'C': case 'D': case 'E':
+ case 'F': case 'G': case 'H': case 'I': case 'J':
+ case 'K': case 'L': case 'M': case 'N': case 'O':
+ case 'P': case 'Q': case 'R': case 'S': case 'T':
+ case 'U': case 'V': case 'W': case 'X': case 'Y':
+ case 'Z':
+ case 'a': case 'b': case 'c': case 'd': case 'e':
+ case 'f': case 'g': case 'h': case 'i': case 'j':
+ case 'k': case 'l': case 'm': case 'n': case 'o':
+ case 'p': case 'q': case 'r': case 's': case 't':
+ case 'u': case 'v': case 'w': case 'x': case 'y':
+ case 'z':
+ case '$': case '_':
+ scanIdent();
+ break loop;
+ case '0':
+ reader.scanChar();
+ if (reader.ch == 'x' || reader.ch == 'X') {
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.ch == '.') {
+ scanHexFractionAndSuffix(pos, false);
+ } else if (reader.digit(pos, 16) < 0) {
+ lexError(pos, "invalid.hex.number");
+ } else {
+ scanNumber(pos, 16);
+ }
+ } else if (reader.ch == 'b' || reader.ch == 'B') {
+ if (!allowBinaryLiterals) {
+ lexError(pos, "unsupported.binary.lit", source.name);
+ allowBinaryLiterals = true;
+ }
+ reader.scanChar();
+ skipIllegalUnderscores();
+ if (reader.digit(pos, 2) < 0) {
+ lexError(pos, "invalid.binary.number");
+ } else {
+ scanNumber(pos, 2);
+ }
+ } else {
+ putChar('0');
+ if (reader.ch == '_') {
+ int savePos = reader.bp;
+ do {
+ reader.scanChar();
+ } while (reader.ch == '_');
+ if (reader.digit(pos, 10) < 0) {
+ lexError(savePos, "illegal.underscore");
+ }
+ }
+ scanNumber(pos, 8);
+ }
+ break loop;
+ case '1': case '2': case '3': case '4':
+ case '5': case '6': case '7': case '8': case '9':
+ scanNumber(pos, 10);
+ break loop;
+ case '.':
+ reader.scanChar();
+ if ('0' <= reader.ch && reader.ch <= '9') {
+ putChar('.');
+ scanFractionAndSuffix(pos);
+ } else if (reader.ch == '.') {
+ putChar('.'); putChar('.');
+ reader.scanChar();
+ if (reader.ch == '.') {
+ reader.scanChar();
+ putChar('.');
+ tk = TokenKind.ELLIPSIS;
+ } else {
+ lexError(pos, "malformed.fp.lit");
+ }
+ } else {
+ tk = TokenKind.DOT;
+ }
+ break loop;
+ case ',':
+ reader.scanChar(); tk = TokenKind.COMMA; break loop;
+ case ';':
+ reader.scanChar(); tk = TokenKind.SEMI; break loop;
+ case '(':
+ reader.scanChar(); tk = TokenKind.LPAREN; break loop;
+ case ')':
+ reader.scanChar(); tk = TokenKind.RPAREN; break loop;
+ case '[':
+ reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
+ case ']':
+ reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
+ case '{':
+ reader.scanChar(); tk = TokenKind.LBRACE; break loop;
+ case '}':
+ reader.scanChar(); tk = TokenKind.RBRACE; break loop;
+ case '/':
+ reader.scanChar();
+ if (reader.ch == '/') {
+ do {
+ scanCommentChar();
+ } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
+ if (reader.bp < reader.buflen) {
+ processComment(pos, reader.bp, CommentStyle.LINE);
+ }
+ break;
+ } else if (reader.ch == '*') {
+ reader.scanChar();
+ CommentStyle style;
+ if (reader.ch == '*') {
+ style = CommentStyle.JAVADOC;
+ scanDocComment();
+ } else {
+ style = CommentStyle.BLOCK;
+ while (reader.bp < reader.buflen) {
+ if (reader.ch == '*') {
+ reader.scanChar();
+ if (reader.ch == '/') break;
+ } else {
+ scanCommentChar();
+ }
+ }
+ }
+ if (reader.ch == '/') {
+ reader.scanChar();
+ processComment(pos, reader.bp, style);
+ break;
+ } else {
+ lexError(pos, "unclosed.comment");
+ break loop;
+ }
+ } else if (reader.ch == '=') {
+ tk = TokenKind.SLASHEQ;
+ reader.scanChar();
+ } else {
+ tk = TokenKind.SLASH;
+ }
+ break loop;
+ case '\'':
+ reader.scanChar();
+ if (reader.ch == '\'') {
+ lexError(pos, "empty.char.lit");
+ } else {
+ if (reader.ch == CR || reader.ch == LF)
+ lexError(pos, "illegal.line.end.in.char.lit");
+ scanLitChar(pos);
+ char ch2 = reader.ch;
+ if (reader.ch == '\'') {
+ reader.scanChar();
+ tk = TokenKind.CHARLITERAL;
+ } else {
+ lexError(pos, "unclosed.char.lit");
+ }
+ }
+ break loop;
+ case '\"':
+ reader.scanChar();
+ while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
+ scanLitChar(pos);
+ if (reader.ch == '\"') {
+ tk = TokenKind.STRINGLITERAL;
+ reader.scanChar();
+ } else {
+ lexError(pos, "unclosed.str.lit");
+ }
+ break loop;
+ default:
+ if (isSpecial(reader.ch)) {
+ scanOperator();
+ } else {
+ boolean isJavaIdentifierStart;
+ if (reader.ch < '\u0080') {
+ // all ASCII range chars already handled, above
+ isJavaIdentifierStart = false;
+ } else {
+ char high = reader.scanSurrogates();
+ if (high != 0) {
+ if (sp == sbuf.length) {
+ putChar(high);
+ } else {
+ sbuf[sp++] = high;
+ }
+
+ isJavaIdentifierStart = Character.isJavaIdentifierStart(
+ Character.toCodePoint(high, reader.ch));
+ } else {
+ isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
+ }
+ }
+ if (isJavaIdentifierStart) {
+ scanIdent();
+ } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
+ tk = TokenKind.EOF;
+ pos = reader.buflen;
+ } else {
+ lexError(pos, "illegal.char", String.valueOf((int)reader.ch));
+ reader.scanChar();
+ }
+ }
+ break loop;
+ }
+ }
+ endPos = reader.bp;
+ switch (tk.tag) {
+ case DEFAULT: return new Token(tk, pos, endPos, deprecatedFlag);
+ case NAMED: return new NamedToken(tk, pos, endPos, name, deprecatedFlag);
+ case STRING: return new StringToken(tk, pos, endPos, new String(sbuf, 0, sp), deprecatedFlag);
+ case NUMERIC: return new NumericToken(tk, pos, endPos, new String(sbuf, 0, sp), radix, deprecatedFlag);
+ default: throw new AssertionError();
+ }
+ }
+ finally {
+ if (scannerDebug) {
+ System.out.println("nextToken(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+ }
+ }
+
+ /** Return the position where a lexical error occurred;
+ */
+ public int errPos() {
+ return errPos;
+ }
+
+ /** Set the position where a lexical error occurred;
+ */
+ public void errPos(int pos) {
+ errPos = pos;
+ }
+
+ public enum CommentStyle {
+ LINE,
+ BLOCK,
+ JAVADOC,
+ }
+
+ /**
+ * Called when a complete comment has been scanned. pos and endPos
+ * will mark the comment boundary.
+ */
+ protected void processComment(int pos, int endPos, CommentStyle style) {
+ if (scannerDebug)
+ System.out.println("processComment(" + pos
+ + "," + endPos + "," + style + ")=|"
+ + new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /**
+ * Called when a complete whitespace run has been scanned. pos and endPos
+ * will mark the whitespace boundary.
+ */
+ protected void processWhiteSpace(int pos, int endPos) {
+ if (scannerDebug)
+ System.out.println("processWhitespace(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /**
+ * Called when a line terminator has been processed.
+ */
+ protected void processLineTerminator(int pos, int endPos) {
+ if (scannerDebug)
+ System.out.println("processTerminator(" + pos
+ + "," + endPos + ")=|" +
+ new String(reader.getRawCharacters(pos, endPos))
+ + "|");
+ }
+
+ /** Build a map for translating between line numbers and
+ * positions in the input.
+ *
+ * @return a LineMap */
+ public Position.LineMap getLineMap() {
+ return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
+ }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavacParser.java Mon Oct 24 13:00:20 2011 +0100
@@ -28,6 +28,7 @@
import java.util.*;
import com.sun.tools.javac.code.*;
+import com.sun.tools.javac.parser.Tokens.*;
import com.sun.tools.javac.tree.*;
import com.sun.tools.javac.tree.JCTree.*;
import com.sun.tools.javac.util.*;
@@ -36,7 +37,7 @@
import com.sun.tools.javac.util.List;
import static com.sun.tools.javac.util.ListBuffer.lb;
-import static com.sun.tools.javac.parser.Token.*;
+import static com.sun.tools.javac.parser.Tokens.TokenKind.*;
/** The parser maps a token sequence into an abstract syntax
* tree. It operates by recursive descent, with code derived
@@ -67,9 +68,6 @@
*/
private Log log;
- /** The keyword table. */
- private Keywords keywords;
-
/** The Source language setting. */
private Source source;
@@ -83,11 +81,10 @@
boolean keepDocComments,
boolean keepLineMap) {
this.S = S;
- S.nextToken(); // prime the pump
+ nextToken(); // prime the pump
this.F = fac.F;
this.log = fac.log;
this.names = fac.names;
- this.keywords = fac.keywords;
this.source = fac.source;
this.allowGenerics = source.allowGenerics();
this.allowVarargs = source.allowVarargs();
@@ -178,7 +175,16 @@
*/
private int lastmode = 0;
-/* ---------- error recovery -------------- */
+ /* ---------- token management -------------- */
+
+ protected Token token;
+
+ protected void nextToken() {
+ S.nextToken();
+ token = S.token();
+ }
+
+ /* ---------- error recovery -------------- */
private JCErroneous errorTree;
@@ -186,9 +192,9 @@
*/
private void skip(boolean stopAtImport, boolean stopAtMemberDecl, boolean stopAtIdentifier, boolean stopAtStatement) {
while (true) {
- switch (S.token()) {
+ switch (token.kind) {
case SEMI:
- S.nextToken();
+ nextToken();
return;
case PUBLIC:
case FINAL:
@@ -249,15 +255,15 @@
return;
break;
}
- S.nextToken();
+ nextToken();
}
}
- private JCErroneous syntaxError(int pos, String key, Token... args) {
+ private JCErroneous syntaxError(int pos, String key, TokenKind... args) {
return syntaxError(pos, List.<JCTree>nil(), key, args);
}
- private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, Token... args) {
+ private JCErroneous syntaxError(int pos, List<JCTree> errs, String key, TokenKind... args) {
setErrorEndPos(pos);
JCErroneous err = F.at(pos).Erroneous(errs);
reportSyntaxError(err, key, (Object[])args);
@@ -287,16 +293,16 @@
private void reportSyntaxError(JCDiagnostic.DiagnosticPosition diagPos, String key, Object... args) {
int pos = diagPos.getPreferredPosition();
if (pos > S.errPos() || pos == Position.NOPOS) {
- if (S.token() == EOF) {
+ if (token.kind == EOF) {
error(diagPos, "premature.eof");
} else {
error(diagPos, key, args);
}
}
S.errPos(pos);
- if (S.pos() == errorPos)
- S.nextToken(); // guarantee progress
- errorPos = S.pos();
+ if (token.pos == errorPos)
+ nextToken(); // guarantee progress
+ errorPos = token.pos;
}
@@ -304,25 +310,25 @@
* reported at the same position.
*/
private JCErroneous syntaxError(String key) {
- return syntaxError(S.pos(), key);
+ return syntaxError(token.pos, key);
}
/** Generate a syntax error at current position unless one was
* already reported at the same position.
*/
- private JCErroneous syntaxError(String key, Token arg) {
- return syntaxError(S.pos(), key, arg);
+ private JCErroneous syntaxError(String key, TokenKind arg) {
+ return syntaxError(token.pos, key, arg);
}
/** If next input token matches given token, skip it, otherwise report
* an error.
*/
- public void accept(Token token) {
- if (S.token() == token) {
- S.nextToken();
+ public void accept(TokenKind tk) {
+ if (token.kind == tk) {
+ nextToken();
} else {
- setErrorEndPos(S.pos());
- reportSyntaxError(S.prevEndPos(), "expected", token);
+ setErrorEndPos(token.pos);
+ reportSyntaxError(S.prevToken().endPos, "expected", tk);
}
}
@@ -340,14 +346,14 @@
/** Report an illegal start of expression/type error at current position.
*/
JCExpression illegal() {
- return illegal(S.pos());
+ return illegal(token.pos);
}
/** Diagnose a modifier flag from the set, if any. */
void checkNoMods(long mods) {
if (mods != 0) {
long lowestMod = mods & -mods;
- error(S.pos(), "mod.not.allowed.here",
+ error(token.pos, "mod.not.allowed.here",
Flags.asFlagSet(lowestMod));
}
}
@@ -435,30 +441,30 @@
* Ident = IDENTIFIER
*/
Name ident() {
- if (S.token() == IDENTIFIER) {
- Name name = S.name();
- S.nextToken();
+ if (token.kind == IDENTIFIER) {
+ Name name = token.name();
+ nextToken();
return name;
- } else if (S.token() == ASSERT) {
+ } else if (token.kind == ASSERT) {
if (allowAsserts) {
- error(S.pos(), "assert.as.identifier");
- S.nextToken();
+ error(token.pos, "assert.as.identifier");
+ nextToken();
return names.error;
} else {
- warning(S.pos(), "assert.as.identifier");
- Name name = S.name();
- S.nextToken();
+ warning(token.pos, "assert.as.identifier");
+ Name name = token.name();
+ nextToken();
return name;
}
- } else if (S.token() == ENUM) {
+ } else if (token.kind == ENUM) {
if (allowEnums) {
- error(S.pos(), "enum.as.identifier");
- S.nextToken();
+ error(token.pos, "enum.as.identifier");
+ nextToken();
return names.error;
} else {
- warning(S.pos(), "enum.as.identifier");
- Name name = S.name();
- S.nextToken();
+ warning(token.pos, "enum.as.identifier");
+ Name name = token.name();
+ nextToken();
return name;
}
} else {
@@ -471,17 +477,17 @@
* Qualident = Ident { DOT Ident }
*/
public JCExpression qualident() {
- JCExpression t = toP(F.at(S.pos()).Ident(ident()));
- while (S.token() == DOT) {
- int pos = S.pos();
- S.nextToken();
+ JCExpression t = toP(F.at(token.pos).Ident(ident()));
+ while (token.kind == DOT) {
+ int pos = token.pos;
+ nextToken();
t = toP(F.at(pos).Select(t, ident()));
}
return t;
}
JCExpression literal(Name prefix) {
- return literal(prefix, S.pos());
+ return literal(prefix, token.pos);
}
/**
@@ -498,27 +504,29 @@
*/
JCExpression literal(Name prefix, int pos) {
JCExpression t = errorTree;
- switch (S.token()) {
+ switch (token.kind) {
case INTLITERAL:
try {
t = F.at(pos).Literal(
TypeTags.INT,
- Convert.string2int(strval(prefix), S.radix()));
+ Convert.string2int(strval(prefix), token.radix()));
} catch (NumberFormatException ex) {
- error(S.pos(), "int.number.too.large", strval(prefix));
+ error(token.pos, "int.number.too.large", strval(prefix));
}
break;
case LONGLITERAL:
try {
t = F.at(pos).Literal(
TypeTags.LONG,
- new Long(Convert.string2long(strval(prefix), S.radix())));
+ new Long(Convert.string2long(strval(prefix), token.radix())));
} catch (NumberFormatException ex) {
- error(S.pos(), "int.number.too.large", strval(prefix));
+ error(token.pos, "int.number.too.large", strval(prefix));
}
break;
case FLOATLITERAL: {
- String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
+ String proper = token.radix() == 16 ?
+ ("0x"+ token.stringVal()) :
+ token.stringVal();
Float n;
try {
n = Float.valueOf(proper);
@@ -527,15 +535,17 @@
n = Float.NaN;
}
if (n.floatValue() == 0.0f && !isZero(proper))
- error(S.pos(), "fp.number.too.small");
+ error(token.pos, "fp.number.too.small");
else if (n.floatValue() == Float.POSITIVE_INFINITY)
- error(S.pos(), "fp.number.too.large");
+ error(token.pos, "fp.number.too.large");
else
t = F.at(pos).Literal(TypeTags.FLOAT, n);
break;
}
case DOUBLELITERAL: {
- String proper = (S.radix() == 16 ? ("0x"+ S.stringVal()) : S.stringVal());
+ String proper = token.radix() == 16 ?
+ ("0x"+ token.stringVal()) :
+ token.stringVal();
Double n;
try {
n = Double.valueOf(proper);
@@ -544,9 +554,9 @@
n = Double.NaN;
}
if (n.doubleValue() == 0.0d && !isZero(proper))
- error(S.pos(), "fp.number.too.small");
+ error(token.pos, "fp.number.too.small");
else if (n.doubleValue() == Double.POSITIVE_INFINITY)
- error(S.pos(), "fp.number.too.large");
+ error(token.pos, "fp.number.too.large");
else
t = F.at(pos).Literal(TypeTags.DOUBLE, n);
break;
@@ -554,17 +564,17 @@
case CHARLITERAL:
t = F.at(pos).Literal(
TypeTags.CHAR,
- S.stringVal().charAt(0) + 0);
+ token.stringVal().charAt(0) + 0);
break;
case STRINGLITERAL:
t = F.at(pos).Literal(
TypeTags.CLASS,
- S.stringVal());
+ token.stringVal());
break;
case TRUE: case FALSE:
t = F.at(pos).Literal(
TypeTags.BOOLEAN,
- (S.token() == TRUE ? 1 : 0));
+ (token.kind == TRUE ? 1 : 0));
break;
case NULL:
t = F.at(pos).Literal(
@@ -576,8 +586,8 @@
}
if (t == errorTree)
t = F.at(pos).Erroneous();
- storeEnd(t, S.endPos());
- S.nextToken();
+ storeEnd(t, token.endPos);
+ nextToken();
return t;
}
//where
@@ -590,7 +600,7 @@
}
String strval(Name prefix) {
- String s = S.stringVal();
+ String s = token.stringVal();
return prefix.isEmpty() ? s : prefix + s;
}
@@ -627,17 +637,17 @@
JCExpression term() {
JCExpression t = term1();
if ((mode & EXPR) != 0 &&
- S.token() == EQ || PLUSEQ.compareTo(S.token()) <= 0 && S.token().compareTo(GTGTGTEQ) <= 0)
+ token.kind == EQ || PLUSEQ.compareTo(token.kind) <= 0 && token.kind.compareTo(GTGTGTEQ) <= 0)
return termRest(t);
else
return t;
}
JCExpression termRest(JCExpression t) {
- switch (S.token()) {
+ switch (token.kind) {
case EQ: {
- int pos = S.pos();
- S.nextToken();
+ int pos = token.pos;
+ nextToken();
mode = EXPR;
JCExpression t1 = term();
return toP(F.at(pos).Assign(t, t1));
@@ -653,12 +663,12 @@
case LTLTEQ:
case GTGTEQ:
case GTGTGTEQ:
- int pos = S.pos();
- Token token = S.token();
- S.nextToken();
+ int pos = token.pos;
+ TokenKind tk = token.kind;
+ nextToken();
mode = EXPR;
JCExpression t1 = term();
- return F.at(pos).Assignop(optag(token), t, t1);
+ return F.at(pos).Assignop(optag(tk), t, t1);
default:
return t;
}
@@ -670,7 +680,7 @@
*/
JCExpression term1() {
JCExpression t = term2();
- if ((mode & EXPR) != 0 && S.token() == QUES) {
+ if ((mode & EXPR) != 0 && token.kind == QUES) {
mode = EXPR;
return term1Rest(t);
} else {
@@ -681,9 +691,9 @@
/** Expression1Rest = ["?" Expression ":" Expression1]
*/
JCExpression term1Rest(JCExpression t) {
- if (S.token() == QUES) {
- int pos = S.pos();
- S.nextToken();
+ if (token.kind == QUES) {
+ int pos = token.pos;
+ nextToken();
JCExpression t1 = term();
accept(COLON);
JCExpression t2 = term1();
@@ -699,7 +709,7 @@
*/
JCExpression term2() {
JCExpression t = term3();
- if ((mode & EXPR) != 0 && prec(S.token()) >= TreeInfo.orPrec) {
+ if ((mode & EXPR) != 0 && prec(token.kind) >= TreeInfo.orPrec) {
mode = EXPR;
return term2Rest(t, TreeInfo.orPrec);
} else {
@@ -725,28 +735,23 @@
JCExpression[] odStack = newOdStack();
List<Token[]> savedOp = opStackSupply.elems;
Token[] opStack = newOpStack();
- List<int[]> savedPos = posStackSupply.elems;
- int[] posStack = newPosStack();
+
// optimization, was odStack = new Tree[...]; opStack = new Tree[...];
int top = 0;
odStack[0] = t;
- int startPos = S.pos();
- Token topOp = ERROR;
- int topOpPos = Position.NOPOS;
- while (prec(S.token()) >= minprec) {
- posStack[top] = topOpPos;
+ int startPos = token.pos;
+ Token topOp = Tokens.DUMMY;
+ while (prec(token.kind) >= minprec) {
opStack[top] = topOp;
top++;
- topOp = S.token();
- topOpPos = S.pos();
- S.nextToken();
- odStack[top] = (topOp == INSTANCEOF) ? parseType() : term3();
- while (top > 0 && prec(topOp) >= prec(S.token())) {
- odStack[top-1] = makeOp(topOpPos, topOp, odStack[top-1],
+ topOp = token;
+ nextToken();
+ odStack[top] = (topOp.kind == INSTANCEOF) ? parseType() : term3();
+ while (top > 0 && prec(topOp.kind) >= prec(token.kind)) {
+ odStack[top-1] = makeOp(topOp.pos, topOp.kind, odStack[top-1],
odStack[top]);
top--;
topOp = opStack[top];
- topOpPos = posStack[top];
}
}
Assert.check(top == 0);
@@ -761,14 +766,13 @@
odStackSupply.elems = savedOd; // optimization
opStackSupply.elems = savedOp; // optimization
- posStackSupply.elems = savedPos; // optimization
return t;
}
//where
/** Construct a binary or type test node.
*/
private JCExpression makeOp(int pos,
- Token topOp,
+ TokenKind topOp,
JCExpression od1,
JCExpression od2)
{
@@ -817,7 +821,6 @@
*/
ListBuffer<JCExpression[]> odStackSupply = new ListBuffer<JCExpression[]>();
ListBuffer<Token[]> opStackSupply = new ListBuffer<Token[]>();
- ListBuffer<int[]> posStackSupply = new ListBuffer<int[]>();
private JCExpression[] newOdStack() {
if (odStackSupply.elems == odStackSupply.last)
@@ -835,14 +838,6 @@
return opStack;
}
- private int[] newPosStack() {
- if (posStackSupply.elems == posStackSupply.last)
- posStackSupply.append(new int[infixPrecedenceLevels + 1]);
- int[] posStack = posStackSupply.elems.head;
- posStackSupply.elems = posStackSupply.elems.tail;
- return posStack;
- }
-
/** Expression3 = PrefixOp Expression3
* | "(" Expr | TypeNoParams ")" Expression3
* | Primary {Selector} {PostfixOp}
@@ -871,10 +866,10 @@
* SuperSuffix = Arguments | "." Ident [Arguments]
*/
protected JCExpression term3() {
- int pos = S.pos();
+ int pos = token.pos;
JCExpression t;
List<JCExpression> typeArgs = typeArgumentsOpt(EXPR);
- switch (S.token()) {
+ switch (token.kind) {
case QUES:
if ((mode & TYPE) != 0 && (mode & (TYPEARG|NOPARAMS)) == TYPEARG) {
mode = TYPE;
@@ -883,49 +878,49 @@
return illegal();
case PLUSPLUS: case SUBSUB: case BANG: case TILDE: case PLUS: case SUB:
if (typeArgs == null && (mode & EXPR) != 0) {
- Token token = S.token();
- S.nextToken();
+ TokenKind tk = token.kind;
+ nextToken();
mode = EXPR;
- if (token == SUB &&
- (S.token() == INTLITERAL || S.token() == LONGLITERAL) &&
- S.radix() == 10) {
+ if (tk == SUB &&
+ (token.kind == INTLITERAL || token.kind == LONGLITERAL) &&
+ token.radix() == 10) {
mode = EXPR;
t = literal(names.hyphen, pos);
} else {
t = term3();
- return F.at(pos).Unary(unoptag(token), t);
+ return F.at(pos).Unary(unoptag(tk), t);
}
} else return illegal();
break;
case LPAREN:
if (typeArgs == null && (mode & EXPR) != 0) {
- S.nextToken();
+ nextToken();
mode = EXPR | TYPE | NOPARAMS;
t = term3();
- if ((mode & TYPE) != 0 && S.token() == LT) {
+ if ((mode & TYPE) != 0 && token.kind == LT) {
// Could be a cast to a parameterized type
int op = JCTree.LT;
- int pos1 = S.pos();
- S.nextToken();
+ int pos1 = token.pos;
+ nextToken();
mode &= (EXPR | TYPE);
mode |= TYPEARG;
JCExpression t1 = term3();
if ((mode & TYPE) != 0 &&
- (S.token() == COMMA || S.token() == GT)) {
+ (token.kind == COMMA || token.kind == GT)) {
mode = TYPE;
ListBuffer<JCExpression> args = new ListBuffer<JCExpression>();
args.append(t1);
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
args.append(typeArgument());
}
accept(GT);
t = toP(F.at(pos1).TypeApply(t, args.toList()));
checkGenerics();
- while (S.token() == DOT) {
- S.nextToken();
+ while (token.kind == DOT) {
+ nextToken();
mode = TYPE;
- t = toP(F.at(S.pos()).Select(t, ident()));
+ t = toP(F.at(token.pos).Select(t, ident()));
t = typeArgumentsOpt(t);
}
t = bracketsOpt(toP(t));
@@ -948,7 +943,7 @@
JCExpression t1 = term3();
return F.at(pos).TypeCast(t, t1);
} else if ((lastmode & TYPE) != 0) {
- switch (S.token()) {
+ switch (token.kind) {
/*case PLUSPLUS: case SUBSUB: */
case BANG: case TILDE:
case LPAREN: case THIS: case SUPER:
@@ -969,7 +964,7 @@
if ((mode & EXPR) != 0) {
mode = EXPR;
t = to(F.at(pos).Ident(names._this));
- S.nextToken();
+ nextToken();
if (typeArgs == null)
t = argumentsOpt(null, t);
else
@@ -997,22 +992,22 @@
if (typeArgs != null) return illegal();
if ((mode & EXPR) != 0) {
mode = EXPR;
- S.nextToken();
- if (S.token() == LT) typeArgs = typeArguments(false);
+ nextToken();
+ if (token.kind == LT) typeArgs = typeArguments(false);
t = creator(pos, typeArgs);
typeArgs = null;
} else return illegal();
break;
case IDENTIFIER: case ASSERT: case ENUM:
if (typeArgs != null) return illegal();
- t = toP(F.at(S.pos()).Ident(ident()));
+ t = toP(F.at(token.pos).Ident(ident()));
loop: while (true) {
- pos = S.pos();
- switch (S.token()) {
+ pos = token.pos;
+ switch (token.kind) {
case LBRACKET:
- S.nextToken();
- if (S.token() == RBRACKET) {
- S.nextToken();
+ nextToken();
+ if (token.kind == RBRACKET) {
+ nextToken();
t = bracketsOpt(t);
t = toP(F.at(pos).TypeArray(t));
t = bracketsSuffix(t);
@@ -1033,24 +1028,24 @@
}
break loop;
case DOT:
- S.nextToken();
+ nextToken();
int oldmode = mode;
mode &= ~NOPARAMS;
typeArgs = typeArgumentsOpt(EXPR);
mode = oldmode;
if ((mode & EXPR) != 0) {
- switch (S.token()) {
+ switch (token.kind) {
case CLASS:
if (typeArgs != null) return illegal();
mode = EXPR;
t = to(F.at(pos).Select(t, names._class));
- S.nextToken();
+ nextToken();
break loop;
case THIS:
if (typeArgs != null) return illegal();
mode = EXPR;
t = to(F.at(pos).Select(t, names._this));
- S.nextToken();
+ nextToken();
break loop;
case SUPER:
mode = EXPR;
@@ -1061,9 +1056,9 @@
case NEW:
if (typeArgs != null) return illegal();
mode = EXPR;
- int pos1 = S.pos();
- S.nextToken();
- if (S.token() == LT) typeArgs = typeArguments(false);
+ int pos1 = token.pos;
+ nextToken();
+ if (token.kind == LT) typeArgs = typeArguments(false);
t = innerCreator(pos1, typeArgs, t);
typeArgs = null;
break loop;
@@ -1087,8 +1082,8 @@
case VOID:
if (typeArgs != null) illegal();
if ((mode & EXPR) != 0) {
- S.nextToken();
- if (S.token() == DOT) {
+ nextToken();
+ if (token.kind == DOT) {
JCPrimitiveTypeTree ti = toP(F.at(pos).TypeIdent(TypeTags.VOID));
t = bracketsSuffix(ti);
} else {
@@ -1099,7 +1094,7 @@
// a void type (like other primitive types) to the next phase.
// The error will be reported in Attr.attribTypes or Attr.visitApply.
JCPrimitiveTypeTree ti = to(F.at(pos).TypeIdent(TypeTags.VOID));
- S.nextToken();
+ nextToken();
return ti;
//return illegal();
}
@@ -1109,14 +1104,14 @@
}
if (typeArgs != null) illegal();
while (true) {
- int pos1 = S.pos();
- if (S.token() == LBRACKET) {
- S.nextToken();
+ int pos1 = token.pos;
+ if (token.kind == LBRACKET) {
+ nextToken();
if ((mode & TYPE) != 0) {
int oldmode = mode;
mode = TYPE;
- if (S.token() == RBRACKET) {
- S.nextToken();
+ if (token.kind == RBRACKET) {
+ nextToken();
t = bracketsOpt(t);
t = toP(F.at(pos1).TypeArray(t));
return t;
@@ -1129,21 +1124,21 @@
t = to(F.at(pos1).Indexed(t, t1));
}
accept(RBRACKET);
- } else if (S.token() == DOT) {
- S.nextToken();
+ } else if (token.kind == DOT) {
+ nextToken();
typeArgs = typeArgumentsOpt(EXPR);
- if (S.token() == SUPER && (mode & EXPR) != 0) {
+ if (token.kind == SUPER && (mode & EXPR) != 0) {
mode = EXPR;
t = to(F.at(pos1).Select(t, names._super));
- S.nextToken();
+ nextToken();
t = arguments(typeArgs, t);
typeArgs = null;
- } else if (S.token() == NEW && (mode & EXPR) != 0) {
+ } else if (token.kind == NEW && (mode & EXPR) != 0) {
if (typeArgs != null) return illegal();
mode = EXPR;
- int pos2 = S.pos();
- S.nextToken();
- if (S.token() == LT) typeArgs = typeArguments(false);
+ int pos2 = token.pos;
+ nextToken();
+ if (token.kind == LT) typeArgs = typeArguments(false);
t = innerCreator(pos2, typeArgs, t);
typeArgs = null;
} else {
@@ -1155,11 +1150,11 @@
break;
}
}
- while ((S.token() == PLUSPLUS || S.token() == SUBSUB) && (mode & EXPR) != 0) {
+ while ((token.kind == PLUSPLUS || token.kind == SUBSUB) && (mode & EXPR) != 0) {
mode = EXPR;
- t = to(F.at(S.pos()).Unary(
- S.token() == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
- S.nextToken();
+ t = to(F.at(token.pos).Unary(
+ token.kind == PLUSPLUS ? JCTree.POSTINC : JCTree.POSTDEC, t));
+ nextToken();
}
return toP(t);
}
@@ -1167,13 +1162,13 @@
/** SuperSuffix = Arguments | "." [TypeArguments] Ident [Arguments]
*/
JCExpression superSuffix(List<JCExpression> typeArgs, JCExpression t) {
- S.nextToken();
- if (S.token() == LPAREN || typeArgs != null) {
+ nextToken();
+ if (token.kind == LPAREN || typeArgs != null) {
t = arguments(typeArgs, t);
} else {
- int pos = S.pos();
+ int pos = token.pos;
accept(DOT);
- typeArgs = (S.token() == LT) ? typeArguments(false) : null;
+ typeArgs = (token.kind == LT) ? typeArguments(false) : null;
t = toP(F.at(pos).Select(t, ident()));
t = argumentsOpt(typeArgs, t);
}
@@ -1183,15 +1178,15 @@
/** BasicType = BYTE | SHORT | CHAR | INT | LONG | FLOAT | DOUBLE | BOOLEAN
*/
JCPrimitiveTypeTree basicType() {
- JCPrimitiveTypeTree t = to(F.at(S.pos()).TypeIdent(typetag(S.token())));
- S.nextToken();
+ JCPrimitiveTypeTree t = to(F.at(token.pos).TypeIdent(typetag(token.kind)));
+ nextToken();
return t;
}
/** ArgumentsOpt = [ Arguments ]
*/
JCExpression argumentsOpt(List<JCExpression> typeArgs, JCExpression t) {
- if ((mode & EXPR) != 0 && S.token() == LPAREN || typeArgs != null) {
+ if ((mode & EXPR) != 0 && token.kind == LPAREN || typeArgs != null) {
mode = EXPR;
return arguments(typeArgs, t);
} else {
@@ -1203,24 +1198,24 @@
*/
List<JCExpression> arguments() {
ListBuffer<JCExpression> args = lb();
- if (S.token() == LPAREN) {
- S.nextToken();
- if (S.token() != RPAREN) {
+ if (token.kind == LPAREN) {
+ nextToken();
+ if (token.kind != RPAREN) {
args.append(parseExpression());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
args.append(parseExpression());
}
}
accept(RPAREN);
} else {
- syntaxError(S.pos(), "expected", LPAREN);
+ syntaxError(token.pos, "expected", LPAREN);
}
return args.toList();
}
JCMethodInvocation arguments(List<JCExpression> typeArgs, JCExpression t) {
- int pos = S.pos();
+ int pos = token.pos;
List<JCExpression> args = arguments();
return toP(F.at(pos).Apply(typeArgs, t, args));
}
@@ -1228,7 +1223,7 @@
/** TypeArgumentsOpt = [ TypeArguments ]
*/
JCExpression typeArgumentsOpt(JCExpression t) {
- if (S.token() == LT &&
+ if (token.kind == LT &&
(mode & TYPE) != 0 &&
(mode & NOPARAMS) == 0) {
mode = TYPE;
@@ -1243,7 +1238,7 @@
}
List<JCExpression> typeArgumentsOpt(int useMode) {
- if (S.token() == LT) {
+ if (token.kind == LT) {
checkGenerics();
if ((mode & useMode) == 0 ||
(mode & NOPARAMS) != 0) {
@@ -1258,47 +1253,37 @@
/** TypeArguments = "<" TypeArgument {"," TypeArgument} ">"
*/
List<JCExpression> typeArguments(boolean diamondAllowed) {
- if (S.token() == LT) {
- S.nextToken();
- if (S.token() == GT && diamondAllowed) {
+ if (token.kind == LT) {
+ nextToken();
+ if (token.kind == GT && diamondAllowed) {
checkDiamond();
mode |= DIAMOND;
- S.nextToken();
+ nextToken();
return List.nil();
} else {
ListBuffer<JCExpression> args = ListBuffer.lb();
args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
args.append(((mode & EXPR) == 0) ? typeArgument() : parseType());
}
- switch (S.token()) {
- case GTGTGTEQ:
- S.token(GTGTEQ);
- break;
- case GTGTEQ:
- S.token(GTEQ);
- break;
- case GTEQ:
- S.token(EQ);
- break;
- case GTGTGT:
- S.token(GTGT);
- break;
- case GTGT:
- S.token(GT);
+ switch (token.kind) {
+
+ case GTGTGTEQ: case GTGTEQ: case GTEQ:
+ case GTGTGT: case GTGT:
+ token = S.split();
break;
case GT:
- S.nextToken();
+ nextToken();
break;
default:
- args.append(syntaxError(S.pos(), "expected", GT));
+ args.append(syntaxError(token.pos, "expected", GT));
break;
}
return args.toList();
}
} else {
- return List.<JCExpression>of(syntaxError(S.pos(), "expected", LT));
+ return List.<JCExpression>of(syntaxError(token.pos, "expected", LT));
}
}
@@ -1308,24 +1293,24 @@
* | "?" SUPER Type
*/
JCExpression typeArgument() {
- if (S.token() != QUES) return parseType();
- int pos = S.pos();
- S.nextToken();
- if (S.token() == EXTENDS) {
+ if (token.kind != QUES) return parseType();
+ int pos = token.pos;
+ nextToken();
+ if (token.kind == EXTENDS) {
TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.EXTENDS));
- S.nextToken();
+ nextToken();
JCExpression bound = parseType();
return F.at(pos).Wildcard(t, bound);
- } else if (S.token() == SUPER) {
+ } else if (token.kind == SUPER) {
TypeBoundKind t = to(F.at(pos).TypeBoundKind(BoundKind.SUPER));
- S.nextToken();
+ nextToken();
JCExpression bound = parseType();
return F.at(pos).Wildcard(t, bound);
- } else if (S.token() == IDENTIFIER) {
+ } else if (token.kind == IDENTIFIER) {
//error recovery
TypeBoundKind t = F.at(Position.NOPOS).TypeBoundKind(BoundKind.UNBOUND);
JCExpression wc = toP(F.at(pos).Wildcard(t, null));
- JCIdent id = toP(F.at(S.pos()).Ident(ident()));
+ JCIdent id = toP(F.at(token.pos).Ident(ident()));
JCErroneous err = F.at(pos).Erroneous(List.<JCTree>of(wc, id));
reportSyntaxError(err, "expected3", GT, EXTENDS, SUPER);
return err;
@@ -1336,7 +1321,7 @@
}
JCTypeApply typeArguments(JCExpression t, boolean diamondAllowed) {
- int pos = S.pos();
+ int pos = token.pos;
List<JCExpression> args = typeArguments(diamondAllowed);
return toP(F.at(pos).TypeApply(t, args));
}
@@ -1344,9 +1329,9 @@
/** BracketsOpt = {"[" "]"}
*/
private JCExpression bracketsOpt(JCExpression t) {
- if (S.token() == LBRACKET) {
- int pos = S.pos();
- S.nextToken();
+ if (token.kind == LBRACKET) {
+ int pos = token.pos;
+ nextToken();
t = bracketsOptCont(t, pos);
F.at(pos);
}
@@ -1363,17 +1348,17 @@
* BracketsSuffixType =
*/
JCExpression bracketsSuffix(JCExpression t) {
- if ((mode & EXPR) != 0 && S.token() == DOT) {
+ if ((mode & EXPR) != 0 && token.kind == DOT) {
mode = EXPR;
- int pos = S.pos();
- S.nextToken();
+ int pos = token.pos;
+ nextToken();
accept(CLASS);
- if (S.pos() == errorEndPos) {
+ if (token.pos == errorEndPos) {
// error recovery
Name name = null;
- if (S.token() == IDENTIFIER) {
- name = S.name();
- S.nextToken();
+ if (token.kind == IDENTIFIER) {
+ name = token.name();
+ nextToken();
} else {
name = names.error;
}
@@ -1384,7 +1369,7 @@
} else if ((mode & TYPE) != 0) {
mode = TYPE;
} else {
- syntaxError(S.pos(), "dot.class.expected");
+ syntaxError(token.pos, "dot.class.expected");
}
return t;
}
@@ -1392,7 +1377,7 @@
/** Creator = Qualident [TypeArguments] ( ArrayCreatorRest | ClassCreatorRest )
*/
JCExpression creator(int newpos, List<JCExpression> typeArgs) {
- switch (S.token()) {
+ switch (token.kind) {
case BYTE: case SHORT: case CHAR: case INT: case LONG: case FLOAT:
case DOUBLE: case BOOLEAN:
if (typeArgs == null)
@@ -1405,29 +1390,29 @@
mode = TYPE;
boolean diamondFound = false;
int lastTypeargsPos = -1;
- if (S.token() == LT) {
+ if (token.kind == LT) {
checkGenerics();
- lastTypeargsPos = S.pos();
+ lastTypeargsPos = token.pos;
t = typeArguments(t, true);
diamondFound = (mode & DIAMOND) != 0;
}
- while (S.token() == DOT) {
+ while (token.kind == DOT) {
if (diamondFound) {
//cannot select after a diamond
illegal();
}
- int pos = S.pos();
- S.nextToken();
+ int pos = token.pos;
+ nextToken();
t = toP(F.at(pos).Select(t, ident()));
- if (S.token() == LT) {
- lastTypeargsPos = S.pos();
+ if (token.kind == LT) {
+ lastTypeargsPos = token.pos;
checkGenerics();
t = typeArguments(t, true);
diamondFound = (mode & DIAMOND) != 0;
}
}
mode = oldmode;
- if (S.token() == LBRACKET) {
+ if (token.kind == LBRACKET) {
JCExpression e = arrayCreatorRest(newpos, t);
if (diamondFound) {
reportSyntaxError(lastTypeargsPos, "cannot.create.array.with.diamond");
@@ -1441,17 +1426,17 @@
// modified to improve error recovery.
pos = typeArgs.head.pos;
}
- setErrorEndPos(S.prevEndPos());
+ setErrorEndPos(S.prevToken().endPos);
JCErroneous err = F.at(pos).Erroneous(typeArgs.prepend(e));
reportSyntaxError(err, "cannot.create.array.with.type.arguments");
return toP(err);
}
return e;
- } else if (S.token() == LPAREN) {
+ } else if (token.kind == LPAREN) {
return classCreatorRest(newpos, null, typeArgs, t);
} else {
- setErrorEndPos(S.pos());
- reportSyntaxError(S.pos(), "expected2", LPAREN, LBRACKET);
+ setErrorEndPos(token.pos);
+ reportSyntaxError(token.pos, "expected2", LPAREN, LBRACKET);
t = toP(F.at(newpos).NewClass(null, typeArgs, t, List.<JCExpression>nil(), null));
return toP(F.at(newpos).Erroneous(List.<JCTree>of(t)));
}
@@ -1460,8 +1445,8 @@
/** InnerCreator = Ident [TypeArguments] ClassCreatorRest
*/
JCExpression innerCreator(int newpos, List<JCExpression> typeArgs, JCExpression encl) {
- JCExpression t = toP(F.at(S.pos()).Ident(ident()));
- if (S.token() == LT) {
+ JCExpression t = toP(F.at(token.pos).Ident(ident()));
+ if (token.kind == LT) {
int oldmode = mode;
checkGenerics();
t = typeArguments(t, true);
@@ -1475,23 +1460,23 @@
*/
JCExpression arrayCreatorRest(int newpos, JCExpression elemtype) {
accept(LBRACKET);
- if (S.token() == RBRACKET) {
+ if (token.kind == RBRACKET) {
accept(RBRACKET);
elemtype = bracketsOpt(elemtype);
- if (S.token() == LBRACE) {
+ if (token.kind == LBRACE) {
return arrayInitializer(newpos, elemtype);
} else {
JCExpression t = toP(F.at(newpos).NewArray(elemtype, List.<JCExpression>nil(), null));
- return syntaxError(S.pos(), List.<JCTree>of(t), "array.dimension.missing");
+ return syntaxError(token.pos, List.<JCTree>of(t), "array.dimension.missing");
}
} else {
ListBuffer<JCExpression> dims = new ListBuffer<JCExpression>();
dims.append(parseExpression());
accept(RBRACKET);
- while (S.token() == LBRACKET) {
- int pos = S.pos();
- S.nextToken();
- if (S.token() == RBRACKET) {
+ while (token.kind == LBRACKET) {
+ int pos = token.pos;
+ nextToken();
+ if (token.kind == RBRACKET) {
elemtype = bracketsOptCont(elemtype, pos);
} else {
dims.append(parseExpression());
@@ -1511,8 +1496,8 @@
{
List<JCExpression> args = arguments();
JCClassDecl body = null;
- if (S.token() == LBRACE) {
- int pos = S.pos();
+ if (token.kind == LBRACE) {
+ int pos = token.pos;
List<JCTree> defs = classOrInterfaceBody(names.empty, false);
JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
body = toP(F.at(pos).AnonymousClassDef(mods, defs));
@@ -1525,13 +1510,13 @@
JCExpression arrayInitializer(int newpos, JCExpression t) {
accept(LBRACE);
ListBuffer<JCExpression> elems = new ListBuffer<JCExpression>();
- if (S.token() == COMMA) {
- S.nextToken();
- } else if (S.token() != RBRACE) {
+ if (token.kind == COMMA) {
+ nextToken();
+ } else if (token.kind != RBRACE) {
elems.append(variableInitializer());
- while (S.token() == COMMA) {
- S.nextToken();
- if (S.token() == RBRACE) break;
+ while (token.kind == COMMA) {
+ nextToken();
+ if (token.kind == RBRACE) break;
elems.append(variableInitializer());
}
}
@@ -1542,7 +1527,7 @@
/** VariableInitializer = ArrayInitializer | Expression
*/
public JCExpression variableInitializer() {
- return S.token() == LBRACE ? arrayInitializer(S.pos(), null) : parseExpression();
+ return token.kind == LBRACE ? arrayInitializer(token.pos, null) : parseExpression();
}
/** ParExpression = "(" Expression ")"
@@ -1560,19 +1545,19 @@
accept(LBRACE);
List<JCStatement> stats = blockStatements();
JCBlock t = F.at(pos).Block(flags, stats);
- while (S.token() == CASE || S.token() == DEFAULT) {
- syntaxError("orphaned", S.token());
+ while (token.kind == CASE || token.kind == DEFAULT) {
+ syntaxError("orphaned", token.kind);
switchBlockStatementGroups();
}
// the Block node has a field "endpos" for first char of last token, which is
// usually but not necessarily the last char of the last token.
- t.endpos = S.pos();
+ t.endpos = token.pos;
accept(RBRACE);
return toP(t);
}
public JCBlock block() {
- return block(S.pos(), 0);
+ return block(token.pos, 0);
}
/** BlockStatements = { BlockStatement }
@@ -1588,8 +1573,8 @@
int lastErrPos = -1;
ListBuffer<JCStatement> stats = new ListBuffer<JCStatement>();
while (true) {
- int pos = S.pos();
- switch (S.token()) {
+ int pos = token.pos;
+ switch (token.kind) {
case RBRACE: case CASE: case DEFAULT: case EOF:
return stats.toList();
case LBRACE: case IF: case FOR: case WHILE: case DO: case TRY:
@@ -1599,64 +1584,63 @@
break;
case MONKEYS_AT:
case FINAL: {
- String dc = S.docComment();
+ String dc = token.docComment;
JCModifiers mods = modifiersOpt();
- if (S.token() == INTERFACE ||
- S.token() == CLASS ||
- allowEnums && S.token() == ENUM) {
+ if (token.kind == INTERFACE ||
+ token.kind == CLASS ||
+ allowEnums && token.kind == ENUM) {
stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
} else {
JCExpression t = parseType();
stats.appendList(variableDeclarators(mods, t,
new ListBuffer<JCStatement>()));
// A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
- storeEnd(stats.elems.last(), S.endPos());
+ storeEnd(stats.elems.last(), token.endPos);
accept(SEMI);
}
break;
}
case ABSTRACT: case STRICTFP: {
- String dc = S.docComment();
+ String dc = token.docComment;
JCModifiers mods = modifiersOpt();
stats.append(classOrInterfaceOrEnumDeclaration(mods, dc));
break;
}
case INTERFACE:
case CLASS:
- stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
- S.docComment()));
+ String dc = token.docComment;
+ stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
break;
case ENUM:
case ASSERT:
- if (allowEnums && S.token() == ENUM) {
- error(S.pos(), "local.enum");
- stats.
- append(classOrInterfaceOrEnumDeclaration(modifiersOpt(),
- S.docComment()));
+ if (allowEnums && token.kind == ENUM) {
+ error(token.pos, "local.enum");
+ dc = token.docComment;
+ stats.append(classOrInterfaceOrEnumDeclaration(modifiersOpt(), dc));
break;
- } else if (allowAsserts && S.token() == ASSERT) {
+ } else if (allowAsserts && token.kind == ASSERT) {
stats.append(parseStatement());
break;
}
/* fall through to default */
default:
- Name name = S.name();
+ Token prevToken = token;
JCExpression t = term(EXPR | TYPE);
- if (S.token() == COLON && t.getTag() == JCTree.IDENT) {
- S.nextToken();
+ if (token.kind == COLON && t.getTag() == JCTree.IDENT) {
+ nextToken();
JCStatement stat = parseStatement();
- stats.append(F.at(pos).Labelled(name, stat));
+ stats.append(F.at(pos).Labelled(prevToken.name(), stat));
} else if ((lastmode & TYPE) != 0 &&
- (S.token() == IDENTIFIER ||
- S.token() == ASSERT ||
- S.token() == ENUM)) {
- pos = S.pos();
+ (token.kind == IDENTIFIER ||
+ token.kind == ASSERT ||
+ token.kind == ENUM)) {
+ pos = token.pos;
JCModifiers mods = F.at(Position.NOPOS).Modifiers(0);
F.at(pos);
stats.appendList(variableDeclarators(mods, t,
new ListBuffer<JCStatement>()));
// A "LocalVariableDeclarationStatement" subsumes the terminating semicolon
- storeEnd(stats.elems.last(), S.endPos());
+ storeEnd(stats.elems.last(), token.endPos);
accept(SEMI);
} else {
// This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
@@ -1666,15 +1650,12 @@
}
// error recovery
- if (S.pos() == lastErrPos)
+ if (token.pos == lastErrPos)
return stats.toList();
- if (S.pos() <= errorEndPos) {
+ if (token.pos <= errorEndPos) {
skip(false, true, true, true);
- lastErrPos = S.pos();
+ lastErrPos = token.pos;
}
-
- // ensure no dangling /** @deprecated */ active
- S.resetDeprecatedFlag();
}
}
@@ -1700,29 +1681,29 @@
*/
@SuppressWarnings("fallthrough")
public JCStatement parseStatement() {
- int pos = S.pos();
- switch (S.token()) {
+ int pos = token.pos;
+ switch (token.kind) {
case LBRACE:
return block();
case IF: {
- S.nextToken();
+ nextToken();
JCExpression cond = parExpression();
JCStatement thenpart = parseStatement();
JCStatement elsepart = null;
- if (S.token() == ELSE) {
- S.nextToken();
+ if (token.kind == ELSE) {
+ nextToken();
elsepart = parseStatement();
}
return F.at(pos).If(cond, thenpart, elsepart);
}
case FOR: {
- S.nextToken();
+ nextToken();
accept(LPAREN);
- List<JCStatement> inits = S.token() == SEMI ? List.<JCStatement>nil() : forInit();
+ List<JCStatement> inits = token.kind == SEMI ? List.<JCStatement>nil() : forInit();
if (inits.length() == 1 &&
inits.head.getTag() == JCTree.VARDEF &&
((JCVariableDecl) inits.head).init == null &&
- S.token() == COLON) {
+ token.kind == COLON) {
checkForeach();
JCVariableDecl var = (JCVariableDecl)inits.head;
accept(COLON);
@@ -1732,22 +1713,22 @@
return F.at(pos).ForeachLoop(var, expr, body);
} else {
accept(SEMI);
- JCExpression cond = S.token() == SEMI ? null : parseExpression();
+ JCExpression cond = token.kind == SEMI ? null : parseExpression();
accept(SEMI);
- List<JCExpressionStatement> steps = S.token() == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
+ List<JCExpressionStatement> steps = token.kind == RPAREN ? List.<JCExpressionStatement>nil() : forUpdate();
accept(RPAREN);
JCStatement body = parseStatement();
return F.at(pos).ForLoop(inits, cond, steps, body);
}
}
case WHILE: {
- S.nextToken();
+ nextToken();
JCExpression cond = parExpression();
JCStatement body = parseStatement();
return F.at(pos).WhileLoop(cond, body);
}
case DO: {
- S.nextToken();
+ nextToken();
JCStatement body = parseStatement();
accept(WHILE);
JCExpression cond = parExpression();
@@ -1756,21 +1737,21 @@
return t;
}
case TRY: {
- S.nextToken();
+ nextToken();
List<JCTree> resources = List.<JCTree>nil();
- if (S.token() == LPAREN) {
+ if (token.kind == LPAREN) {
checkTryWithResources();
- S.nextToken();
+ nextToken();
resources = resources();
accept(RPAREN);
}
JCBlock body = block();
ListBuffer<JCCatch> catchers = new ListBuffer<JCCatch>();
JCBlock finalizer = null;
- if (S.token() == CATCH || S.token() == FINALLY) {
- while (S.token() == CATCH) catchers.append(catchClause());
- if (S.token() == FINALLY) {
- S.nextToken();
+ if (token.kind == CATCH || token.kind == FINALLY) {
+ while (token.kind == CATCH) catchers.append(catchClause());
+ if (token.kind == FINALLY) {
+ nextToken();
finalizer = block();
}
} else {
@@ -1783,7 +1764,7 @@
return F.at(pos).Try(resources, body, catchers.toList(), finalizer);
}
case SWITCH: {
- S.nextToken();
+ nextToken();
JCExpression selector = parExpression();
accept(LBRACE);
List<JCCase> cases = switchBlockStatementGroups();
@@ -1792,41 +1773,41 @@
return t;
}
case SYNCHRONIZED: {
- S.nextToken();
+ nextToken();
JCExpression lock = parExpression();
JCBlock body = block();
return F.at(pos).Synchronized(lock, body);
}
case RETURN: {
- S.nextToken();
- JCExpression result = S.token() == SEMI ? null : parseExpression();
+ nextToken();
+ JCExpression result = token.kind == SEMI ? null : parseExpression();
JCReturn t = to(F.at(pos).Return(result));
accept(SEMI);
return t;
}
case THROW: {
- S.nextToken();
+ nextToken();
JCExpression exc = parseExpression();
JCThrow t = to(F.at(pos).Throw(exc));
accept(SEMI);
return t;
}
case BREAK: {
- S.nextToken();
- Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
+ nextToken();
+ Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
JCBreak t = to(F.at(pos).Break(label));
accept(SEMI);
return t;
}
case CONTINUE: {
- S.nextToken();
- Name label = (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM) ? ident() : null;
+ nextToken();
+ Name label = (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM) ? ident() : null;
JCContinue t = to(F.at(pos).Continue(label));
accept(SEMI);
return t;
}
case SEMI:
- S.nextToken();
+ nextToken();
return toP(F.at(pos).Skip());
case ELSE:
return toP(F.Exec(syntaxError("else.without.if")));
@@ -1835,12 +1816,12 @@
case CATCH:
return toP(F.Exec(syntaxError("catch.without.try")));
case ASSERT: {
- if (allowAsserts && S.token() == ASSERT) {
- S.nextToken();
+ if (allowAsserts && token.kind == ASSERT) {
+ nextToken();
JCExpression assertion = parseExpression();
JCExpression message = null;
- if (S.token() == COLON) {
- S.nextToken();
+ if (token.kind == COLON) {
+ nextToken();
message = parseExpression();
}
JCAssert t = to(F.at(pos).Assert(assertion, message));
@@ -1851,12 +1832,12 @@
}
case ENUM:
default:
- Name name = S.name();
+ Token prevToken = token;
JCExpression expr = parseExpression();
- if (S.token() == COLON && expr.getTag() == JCTree.IDENT) {
- S.nextToken();
+ if (token.kind == COLON && expr.getTag() == JCTree.IDENT) {
+ nextToken();
JCStatement stat = parseStatement();
- return F.at(pos).Labelled(name, stat);
+ return F.at(pos).Labelled(prevToken.name(), stat);
} else {
// This Exec is an "ExpressionStatement"; it subsumes the terminating semicolon
JCExpressionStatement stat = to(F.at(pos).Exec(checkExprStat(expr)));
@@ -1869,7 +1850,7 @@
/** CatchClause = CATCH "(" FormalParameter ")" Block
*/
protected JCCatch catchClause() {
- int pos = S.pos();
+ int pos = token.pos;
accept(CATCH);
accept(LPAREN);
JCModifiers mods = optFinal(Flags.PARAMETER);
@@ -1886,9 +1867,9 @@
List<JCExpression> catchTypes() {
ListBuffer<JCExpression> catchTypes = ListBuffer.lb();
catchTypes.add(parseType());
- while (S.token() == BAR) {
+ while (token.kind == BAR) {
checkMulticatch();
- S.nextToken();
+ nextToken();
catchTypes.add(qualident());
}
return catchTypes.toList();
@@ -1901,33 +1882,33 @@
List<JCCase> switchBlockStatementGroups() {
ListBuffer<JCCase> cases = new ListBuffer<JCCase>();
while (true) {
- int pos = S.pos();
- switch (S.token()) {
+ int pos = token.pos;
+ switch (token.kind) {
case CASE: {
- S.nextToken();
+ nextToken();
JCExpression pat = parseExpression();
accept(COLON);
List<JCStatement> stats = blockStatements();
JCCase c = F.at(pos).Case(pat, stats);
if (stats.isEmpty())
- storeEnd(c, S.prevEndPos());
+ storeEnd(c, S.prevToken().endPos);
cases.append(c);
break;
}
case DEFAULT: {
- S.nextToken();
+ nextToken();
accept(COLON);
List<JCStatement> stats = blockStatements();
JCCase c = F.at(pos).Case(null, stats);
if (stats.isEmpty())
- storeEnd(c, S.prevEndPos());
+ storeEnd(c, S.prevToken().endPos);
cases.append(c);
break;
}
case RBRACE: case EOF:
return cases.toList();
default:
- S.nextToken(); // to ensure progress
+ nextToken(); // to ensure progress
syntaxError(pos, "expected3",
CASE, DEFAULT, RBRACE);
}
@@ -1941,9 +1922,9 @@
T stats) {
// This Exec is a "StatementExpression"; it subsumes no terminating token
stats.append(toP(F.at(pos).Exec(checkExprStat(first))));
- while (S.token() == COMMA) {
- S.nextToken();
- pos = S.pos();
+ while (token.kind == COMMA) {
+ nextToken();
+ pos = token.pos;
JCExpression t = parseExpression();
// This Exec is a "StatementExpression"; it subsumes no terminating token
stats.append(toP(F.at(pos).Exec(checkExprStat(t))));
@@ -1956,13 +1937,13 @@
*/
List<JCStatement> forInit() {
ListBuffer<JCStatement> stats = lb();
- int pos = S.pos();
- if (S.token() == FINAL || S.token() == MONKEYS_AT) {
+ int pos = token.pos;
+ if (token.kind == FINAL || token.kind == MONKEYS_AT) {
return variableDeclarators(optFinal(0), parseType(), stats).toList();
} else {
JCExpression t = term(EXPR | TYPE);
if ((lastmode & TYPE) != 0 &&
- (S.token() == IDENTIFIER || S.token() == ASSERT || S.token() == ENUM))
+ (token.kind == IDENTIFIER || token.kind == ASSERT || token.kind == ENUM))
return variableDeclarators(modifiersOpt(), t, stats).toList();
else
return moreStatementExpressions(pos, t, stats).toList();
@@ -1972,7 +1953,7 @@
/** ForUpdate = StatementExpression MoreStatementExpressions
*/
List<JCExpressionStatement> forUpdate() {
- return moreStatementExpressions(S.pos(),
+ return moreStatementExpressions(token.pos,
parseExpression(),
new ListBuffer<JCExpressionStatement>()).toList();
}
@@ -1980,11 +1961,11 @@
/** AnnotationsOpt = { '@' Annotation }
*/
List<JCAnnotation> annotationsOpt() {
- if (S.token() != MONKEYS_AT) return List.nil(); // optimization
+ if (token.kind != MONKEYS_AT) return List.nil(); // optimization
ListBuffer<JCAnnotation> buf = new ListBuffer<JCAnnotation>();
- while (S.token() == MONKEYS_AT) {
- int pos = S.pos();
- S.nextToken();
+ while (token.kind == MONKEYS_AT) {
+ int pos = token.pos;
+ nextToken();
buf.append(annotation(pos));
}
return buf.toList();
@@ -2004,21 +1985,20 @@
int pos;
if (partial == null) {
flags = 0;
- pos = S.pos();
+ pos = token.pos;
} else {
flags = partial.flags;
annotations.appendList(partial.annotations);
pos = partial.pos;
}
- if (S.deprecatedFlag()) {
+ if (token.deprecatedFlag) {
flags |= Flags.DEPRECATED;
- S.resetDeprecatedFlag();
}
int lastPos = Position.NOPOS;
loop:
while (true) {
long flag;
- switch (S.token()) {
+ switch (token.kind) {
case PRIVATE : flag = Flags.PRIVATE; break;
case PROTECTED : flag = Flags.PROTECTED; break;
case PUBLIC : flag = Flags.PUBLIC; break;
@@ -2031,15 +2011,15 @@
case SYNCHRONIZED: flag = Flags.SYNCHRONIZED; break;
case STRICTFP : flag = Flags.STRICTFP; break;
case MONKEYS_AT : flag = Flags.ANNOTATION; break;
- case ERROR : flag = 0; S.nextToken(); break;
+ case ERROR : flag = 0; nextToken(); break;
default: break loop;
}
- if ((flags & flag) != 0) error(S.pos(), "repeated.modifier");
- lastPos = S.pos();
- S.nextToken();
+ if ((flags & flag) != 0) error(token.pos, "repeated.modifier");
+ lastPos = token.pos;
+ nextToken();
if (flag == Flags.ANNOTATION) {
checkAnnotations();
- if (S.token() != INTERFACE) {
+ if (token.kind != INTERFACE) {
JCAnnotation ann = annotation(lastPos);
// if first modifier is an annotation, set pos to annotation's.
if (flags == 0 && annotations.isEmpty())
@@ -2051,7 +2031,7 @@
}
flags |= flag;
}
- switch (S.token()) {
+ switch (token.kind) {
case ENUM: flags |= Flags.ENUM; break;
case INTERFACE: flags |= Flags.INTERFACE; break;
default: break;
@@ -2064,7 +2044,7 @@
JCModifiers mods = F.at(pos).Modifiers(flags, annotations.toList());
if (pos != Position.NOPOS)
- storeEnd(mods, S.prevEndPos());
+ storeEnd(mods, S.prevToken().endPos);
return mods;
}
@@ -2077,22 +2057,22 @@
JCTree ident = qualident();
List<JCExpression> fieldValues = annotationFieldValuesOpt();
JCAnnotation ann = F.at(pos).Annotation(ident, fieldValues);
- storeEnd(ann, S.prevEndPos());
+ storeEnd(ann, S.prevToken().endPos);
return ann;
}
List<JCExpression> annotationFieldValuesOpt() {
- return (S.token() == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
+ return (token.kind == LPAREN) ? annotationFieldValues() : List.<JCExpression>nil();
}
/** AnnotationFieldValues = "(" [ AnnotationFieldValue { "," AnnotationFieldValue } ] ")" */
List<JCExpression> annotationFieldValues() {
accept(LPAREN);
ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
- if (S.token() != RPAREN) {
+ if (token.kind != RPAREN) {
buf.append(annotationFieldValue());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
buf.append(annotationFieldValue());
}
}
@@ -2104,11 +2084,11 @@
* | Identifier "=" AnnotationValue
*/
JCExpression annotationFieldValue() {
- if (S.token() == IDENTIFIER) {
+ if (token.kind == IDENTIFIER) {
mode = EXPR;
JCExpression t1 = term1();
- if (t1.getTag() == JCTree.IDENT && S.token() == EQ) {
- int pos = S.pos();
+ if (t1.getTag() == JCTree.IDENT && token.kind == EQ) {
+ int pos = token.pos;
accept(EQ);
JCExpression v = annotationValue();
return toP(F.at(pos).Assign(t1, v));
@@ -2125,20 +2105,20 @@
*/
JCExpression annotationValue() {
int pos;
- switch (S.token()) {
+ switch (token.kind) {
case MONKEYS_AT:
- pos = S.pos();
- S.nextToken();
+ pos = token.pos;
+ nextToken();
return annotation(pos);
case LBRACE:
- pos = S.pos();
+ pos = token.pos;
accept(LBRACE);
ListBuffer<JCExpression> buf = new ListBuffer<JCExpression>();
- if (S.token() != RBRACE) {
+ if (token.kind != RBRACE) {
buf.append(annotationValue());
- while (S.token() == COMMA) {
- S.nextToken();
- if (S.token() == RBRACE) break;
+ while (token.kind == COMMA) {
+ nextToken();
+ if (token.kind == RBRACE) break;
buf.append(annotationValue());
}
}
@@ -2156,7 +2136,7 @@
JCExpression type,
T vdefs)
{
- return variableDeclaratorsRest(S.pos(), mods, type, ident(), false, null, vdefs);
+ return variableDeclaratorsRest(token.pos, mods, type, ident(), false, null, vdefs);
}
/** VariableDeclaratorsRest = VariableDeclaratorRest { "," VariableDeclarator }
@@ -2174,10 +2154,10 @@
T vdefs)
{
vdefs.append(variableDeclaratorRest(pos, mods, type, name, reqInit, dc));
- while (S.token() == COMMA) {
+ while (token.kind == COMMA) {
// All but last of multiple declarators subsume a comma
- storeEnd((JCTree)vdefs.elems.last(), S.endPos());
- S.nextToken();
+ storeEnd((JCTree)vdefs.elems.last(), token.endPos);
+ nextToken();
vdefs.append(variableDeclarator(mods, type, reqInit, dc));
}
return vdefs;
@@ -2187,7 +2167,7 @@
* ConstantDeclarator = Ident ConstantDeclaratorRest
*/
JCVariableDecl variableDeclarator(JCModifiers mods, JCExpression type, boolean reqInit, String dc) {
- return variableDeclaratorRest(S.pos(), mods, type, ident(), reqInit, dc);
+ return variableDeclaratorRest(token.pos, mods, type, ident(), reqInit, dc);
}
/** VariableDeclaratorRest = BracketsOpt ["=" VariableInitializer]
@@ -2200,11 +2180,11 @@
boolean reqInit, String dc) {
type = bracketsOpt(type);
JCExpression init = null;
- if (S.token() == EQ) {
- S.nextToken();
+ if (token.kind == EQ) {
+ nextToken();
init = variableInitializer();
}
- else if (reqInit) syntaxError(S.pos(), "expected", EQ);
+ else if (reqInit) syntaxError(token.pos, "expected", EQ);
JCVariableDecl result =
toP(F.at(pos).VarDef(mods, name, type, init));
attach(result, dc);
@@ -2214,11 +2194,11 @@
/** VariableDeclaratorId = Ident BracketsOpt
*/
JCVariableDecl variableDeclaratorId(JCModifiers mods, JCExpression type) {
- int pos = S.pos();
+ int pos = token.pos;
Name name = ident();
if ((mods.flags & Flags.VARARGS) != 0 &&
- S.token() == LBRACKET) {
- log.error(S.pos(), "varargs.and.old.array.syntax");
+ token.kind == LBRACKET) {
+ log.error(token.pos, "varargs.and.old.array.syntax");
}
type = bracketsOpt(type);
return toP(F.at(pos).VarDef(mods, name, type, null));
@@ -2229,12 +2209,12 @@
List<JCTree> resources() {
ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
defs.append(resource());
- while (S.token() == SEMI) {
+ while (token.kind == SEMI) {
// All but last of multiple declarators must subsume a semicolon
- storeEnd(defs.elems.last(), S.endPos());
- int semiColonPos = S.pos();
- S.nextToken();
- if (S.token() == RPAREN) { // Optional trailing semicolon
+ storeEnd(defs.elems.last(), token.endPos);
+ int semiColonPos = token.pos;
+ nextToken();
+ if (token.kind == RPAREN) { // Optional trailing semicolon
// after last resource
break;
}
@@ -2248,7 +2228,7 @@
protected JCTree resource() {
JCModifiers optFinal = optFinal(Flags.FINAL);
JCExpression type = parseType();
- int pos = S.pos();
+ int pos = token.pos;
Name ident = ident();
return variableDeclaratorRest(pos, optFinal, type, ident, true, null);
}
@@ -2256,54 +2236,61 @@
/** CompilationUnit = [ { "@" Annotation } PACKAGE Qualident ";"] {ImportDeclaration} {TypeDeclaration}
*/
public JCTree.JCCompilationUnit parseCompilationUnit() {
- int pos = S.pos();
+ Token firstToken = token;
JCExpression pid = null;
- String dc = S.docComment();
JCModifiers mods = null;
+ boolean consumedToplevelDoc = false;
+ boolean seenImport = false;
+ boolean seenPackage = false;
List<JCAnnotation> packageAnnotations = List.nil();
- if (S.token() == MONKEYS_AT)
+ if (token.kind == MONKEYS_AT)
mods = modifiersOpt();
- if (S.token() == PACKAGE) {
+ if (token.kind == PACKAGE) {
+ seenPackage = true;
if (mods != null) {
checkNoMods(mods.flags);
packageAnnotations = mods.annotations;
mods = null;
}
- S.nextToken();
+ nextToken();
pid = qualident();
accept(SEMI);
}
ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
boolean checkForImports = true;
- while (S.token() != EOF) {
- if (S.pos() <= errorEndPos) {
+ boolean firstTypeDecl = true;
+ while (token.kind != EOF) {
+ if (token.pos <= errorEndPos) {
// error recovery
skip(checkForImports, false, false, false);
- if (S.token() == EOF)
+ if (token.kind == EOF)
break;
}
- if (checkForImports && mods == null && S.token() == IMPORT) {
+ if (checkForImports && mods == null && token.kind == IMPORT) {
+ seenImport = true;
defs.append(importDeclaration());
} else {
- JCTree def = typeDeclaration(mods);
- if (keepDocComments && dc != null && docComments.get(def) == dc) {
- // If the first type declaration has consumed the first doc
- // comment, then don't use it for the top level comment as well.
- dc = null;
+ String docComment = token.docComment;
+ if (firstTypeDecl && !seenImport && !seenPackage) {
+ docComment = firstToken.docComment;
+ consumedToplevelDoc = true;
}
+ JCTree def = typeDeclaration(mods, docComment);
if (def instanceof JCExpressionStatement)
def = ((JCExpressionStatement)def).expr;
defs.append(def);
if (def instanceof JCClassDecl)
checkForImports = false;
mods = null;
+ firstTypeDecl = false;
}
}
- JCTree.JCCompilationUnit toplevel = F.at(pos).TopLevel(packageAnnotations, pid, defs.toList());
- attach(toplevel, dc);
+ JCTree.JCCompilationUnit toplevel = F.at(firstToken.pos).TopLevel(packageAnnotations, pid, defs.toList());
+ if (!consumedToplevelDoc)
+ attach(toplevel, firstToken.docComment);
if (defs.elems.isEmpty())
- storeEnd(toplevel, S.prevEndPos());
+ storeEnd(toplevel, S.prevToken().endPos);
if (keepDocComments)
toplevel.docComments = docComments;
if (keepLineMap)
@@ -2314,26 +2301,26 @@
/** ImportDeclaration = IMPORT [ STATIC ] Ident { "." Ident } [ "." "*" ] ";"
*/
JCTree importDeclaration() {
- int pos = S.pos();
- S.nextToken();
+ int pos = token.pos;
+ nextToken();
boolean importStatic = false;
- if (S.token() == STATIC) {
+ if (token.kind == STATIC) {
checkStaticImports();
importStatic = true;
- S.nextToken();
+ nextToken();
}
- JCExpression pid = toP(F.at(S.pos()).Ident(ident()));
+ JCExpression pid = toP(F.at(token.pos).Ident(ident()));
do {
- int pos1 = S.pos();
+ int pos1 = token.pos;
accept(DOT);
- if (S.token() == STAR) {
+ if (token.kind == STAR) {
pid = to(F.at(pos1).Select(pid, names.asterisk));
- S.nextToken();
+ nextToken();
break;
} else {
pid = toP(F.at(pos1).Select(pid, ident()));
}
- } while (S.token() == DOT);
+ } while (token.kind == DOT);
accept(SEMI);
return toP(F.at(pos).Import(pid, importStatic));
}
@@ -2341,14 +2328,13 @@
/** TypeDeclaration = ClassOrInterfaceOrEnumDeclaration
* | ";"
*/
- JCTree typeDeclaration(JCModifiers mods) {
- int pos = S.pos();
- if (mods == null && S.token() == SEMI) {
- S.nextToken();
+ JCTree typeDeclaration(JCModifiers mods, String docComment) {
+ int pos = token.pos;
+ if (mods == null && token.kind == SEMI) {
+ nextToken();
return toP(F.at(pos).Skip());
} else {
- String dc = S.docComment();
- return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), dc);
+ return classOrInterfaceOrEnumDeclaration(modifiersOpt(mods), docComment);
}
}
@@ -2358,19 +2344,19 @@
* @param dc The documentation comment for the class, or null.
*/
JCStatement classOrInterfaceOrEnumDeclaration(JCModifiers mods, String dc) {
- if (S.token() == CLASS) {
+ if (token.kind == CLASS) {
return classDeclaration(mods, dc);
- } else if (S.token() == INTERFACE) {
+ } else if (token.kind == INTERFACE) {
return interfaceDeclaration(mods, dc);
} else if (allowEnums) {
- if (S.token() == ENUM) {
+ if (token.kind == ENUM) {
return enumDeclaration(mods, dc);
} else {
- int pos = S.pos();
+ int pos = token.pos;
List<JCTree> errs;
- if (S.token() == IDENTIFIER) {
+ if (token.kind == IDENTIFIER) {
errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
- setErrorEndPos(S.pos());
+ setErrorEndPos(token.pos);
} else {
errs = List.<JCTree>of(mods);
}
@@ -2378,16 +2364,16 @@
CLASS, INTERFACE, ENUM)));
}
} else {
- if (S.token() == ENUM) {
- error(S.pos(), "enums.not.supported.in.source", source.name);
+ if (token.kind == ENUM) {
+ error(token.pos, "enums.not.supported.in.source", source.name);
allowEnums = true;
return enumDeclaration(mods, dc);
}
- int pos = S.pos();
+ int pos = token.pos;
List<JCTree> errs;
- if (S.token() == IDENTIFIER) {
+ if (token.kind == IDENTIFIER) {
errs = List.<JCTree>of(mods, toP(F.at(pos).Ident(ident())));
- setErrorEndPos(S.pos());
+ setErrorEndPos(token.pos);
} else {
errs = List.<JCTree>of(mods);
}
@@ -2402,20 +2388,20 @@
* @param dc The documentation comment for the class, or null.
*/
JCClassDecl classDeclaration(JCModifiers mods, String dc) {
- int pos = S.pos();
+ int pos = token.pos;
accept(CLASS);
Name name = ident();
List<JCTypeParameter> typarams = typeParametersOpt();
JCExpression extending = null;
- if (S.token() == EXTENDS) {
- S.nextToken();
+ if (token.kind == EXTENDS) {
+ nextToken();
extending = parseType();
}
List<JCExpression> implementing = List.nil();
- if (S.token() == IMPLEMENTS) {
- S.nextToken();
+ if (token.kind == IMPLEMENTS) {
+ nextToken();
implementing = typeList();
}
List<JCTree> defs = classOrInterfaceBody(name, false);
@@ -2431,15 +2417,15 @@
* @param dc The documentation comment for the interface, or null.
*/
JCClassDecl interfaceDeclaration(JCModifiers mods, String dc) {
- int pos = S.pos();
+ int pos = token.pos;
accept(INTERFACE);
Name name = ident();
List<JCTypeParameter> typarams = typeParametersOpt();
List<JCExpression> extending = List.nil();
- if (S.token() == EXTENDS) {
- S.nextToken();
+ if (token.kind == EXTENDS) {
+ nextToken();
extending = typeList();
}
List<JCTree> defs = classOrInterfaceBody(name, true);
@@ -2454,13 +2440,13 @@
* @param dc The documentation comment for the enum, or null.
*/
JCClassDecl enumDeclaration(JCModifiers mods, String dc) {
- int pos = S.pos();
+ int pos = token.pos;
accept(ENUM);
Name name = ident();
List<JCExpression> implementing = List.nil();
- if (S.token() == IMPLEMENTS) {
- S.nextToken();
+ if (token.kind == IMPLEMENTS) {
+ nextToken();
implementing = typeList();
}
@@ -2479,27 +2465,27 @@
List<JCTree> enumBody(Name enumName) {
accept(LBRACE);
ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
- if (S.token() == COMMA) {
- S.nextToken();
- } else if (S.token() != RBRACE && S.token() != SEMI) {
+ if (token.kind == COMMA) {
+ nextToken();
+ } else if (token.kind != RBRACE && token.kind != SEMI) {
defs.append(enumeratorDeclaration(enumName));
- while (S.token() == COMMA) {
- S.nextToken();
- if (S.token() == RBRACE || S.token() == SEMI) break;
+ while (token.kind == COMMA) {
+ nextToken();
+ if (token.kind == RBRACE || token.kind == SEMI) break;
defs.append(enumeratorDeclaration(enumName));
}
- if (S.token() != SEMI && S.token() != RBRACE) {
- defs.append(syntaxError(S.pos(), "expected3",
+ if (token.kind != SEMI && token.kind != RBRACE) {
+ defs.append(syntaxError(token.pos, "expected3",
COMMA, RBRACE, SEMI));
- S.nextToken();
+ nextToken();
}
}
- if (S.token() == SEMI) {
- S.nextToken();
- while (S.token() != RBRACE && S.token() != EOF) {
+ if (token.kind == SEMI) {
+ nextToken();
+ while (token.kind != RBRACE && token.kind != EOF) {
defs.appendList(classOrInterfaceBodyDeclaration(enumName,
false));
- if (S.pos() <= errorEndPos) {
+ if (token.pos <= errorEndPos) {
// error recovery
skip(false, true, true, false);
}
@@ -2512,23 +2498,22 @@
/** EnumeratorDeclaration = AnnotationsOpt [TypeArguments] IDENTIFIER [ Arguments ] [ "{" ClassBody "}" ]
*/
JCTree enumeratorDeclaration(Name enumName) {
- String dc = S.docComment();
+ String dc = token.docComment;
int flags = Flags.PUBLIC|Flags.STATIC|Flags.FINAL|Flags.ENUM;
- if (S.deprecatedFlag()) {
+ if (token.deprecatedFlag) {
flags |= Flags.DEPRECATED;
- S.resetDeprecatedFlag();
}
- int pos = S.pos();
+ int pos = token.pos;
List<JCAnnotation> annotations = annotationsOpt();
JCModifiers mods = F.at(annotations.isEmpty() ? Position.NOPOS : pos).Modifiers(flags, annotations);
List<JCExpression> typeArgs = typeArgumentsOpt();
- int identPos = S.pos();
+ int identPos = token.pos;
Name name = ident();
- int createPos = S.pos();
- List<JCExpression> args = (S.token() == LPAREN)
+ int createPos = token.pos;
+ List<JCExpression> args = (token.kind == LPAREN)
? arguments() : List.<JCExpression>nil();
JCClassDecl body = null;
- if (S.token() == LBRACE) {
+ if (token.kind == LBRACE) {
JCModifiers mods1 = F.at(Position.NOPOS).Modifiers(Flags.ENUM | Flags.STATIC);
List<JCTree> defs = classOrInterfaceBody(names.empty, false);
body = toP(F.at(identPos).AnonymousClassDef(mods1, defs));
@@ -2538,7 +2523,7 @@
JCIdent ident = F.at(identPos).Ident(enumName);
JCNewClass create = F.at(createPos).NewClass(null, typeArgs, ident, args, body);
if (createPos != identPos)
- storeEnd(create, S.prevEndPos());
+ storeEnd(create, S.prevToken().endPos);
ident = F.at(identPos).Ident(enumName);
JCTree result = toP(F.at(pos).VarDef(mods, name, ident, create));
attach(result, dc);
@@ -2550,8 +2535,8 @@
List<JCExpression> typeList() {
ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
ts.append(parseType());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
ts.append(parseType());
}
return ts.toList();
@@ -2562,16 +2547,16 @@
*/
List<JCTree> classOrInterfaceBody(Name className, boolean isInterface) {
accept(LBRACE);
- if (S.pos() <= errorEndPos) {
+ if (token.pos <= errorEndPos) {
// error recovery
skip(false, true, false, false);
- if (S.token() == LBRACE)
- S.nextToken();
+ if (token.kind == LBRACE)
+ nextToken();
}
ListBuffer<JCTree> defs = new ListBuffer<JCTree>();
- while (S.token() != RBRACE && S.token() != EOF) {
+ while (token.kind != RBRACE && token.kind != EOF) {
defs.appendList(classOrInterfaceBodyDeclaration(className, isInterface));
- if (S.pos() <= errorEndPos) {
+ if (token.pos <= errorEndPos) {
// error recovery
skip(false, true, true, false);
}
@@ -2598,23 +2583,23 @@
* ( ConstantDeclaratorsRest | InterfaceMethodDeclaratorRest ";" )
*/
protected List<JCTree> classOrInterfaceBodyDeclaration(Name className, boolean isInterface) {
- if (S.token() == SEMI) {
- S.nextToken();
+ if (token.kind == SEMI) {
+ nextToken();
return List.<JCTree>nil();
} else {
- String dc = S.docComment();
- int pos = S.pos();
+ String dc = token.docComment;
+ int pos = token.pos;
JCModifiers mods = modifiersOpt();
- if (S.token() == CLASS ||
- S.token() == INTERFACE ||
- allowEnums && S.token() == ENUM) {
+ if (token.kind == CLASS ||
+ token.kind == INTERFACE ||
+ allowEnums && token.kind == ENUM) {
return List.<JCTree>of(classOrInterfaceOrEnumDeclaration(mods, dc));
- } else if (S.token() == LBRACE && !isInterface &&
+ } else if (token.kind == LBRACE && !isInterface &&
(mods.flags & Flags.StandardFlags & ~Flags.STATIC) == 0 &&
mods.annotations.isEmpty()) {
return List.<JCTree>of(block(pos, mods.flags));
} else {
- pos = S.pos();
+ pos = token.pos;
List<JCTypeParameter> typarams = typeParametersOpt();
// if there are type parameters but no modifiers, save the start
// position of the method in the modifiers.
@@ -2622,26 +2607,26 @@
mods.pos = pos;
storeEnd(mods, pos);
}
- Name name = S.name();
- pos = S.pos();
+ Token tk = token;
+ pos = token.pos;
JCExpression type;
- boolean isVoid = S.token() == VOID;
+ boolean isVoid = token.kind == VOID;
if (isVoid) {
type = to(F.at(pos).TypeIdent(TypeTags.VOID));
- S.nextToken();
+ nextToken();
} else {
type = parseType();
}
- if (S.token() == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
- if (isInterface || name != className)
+ if (token.kind == LPAREN && !isInterface && type.getTag() == JCTree.IDENT) {
+ if (isInterface || tk.name() != className)
error(pos, "invalid.meth.decl.ret.type.req");
return List.of(methodDeclaratorRest(
pos, mods, null, names.init, typarams,
isInterface, true, dc));
} else {
- pos = S.pos();
- name = ident();
- if (S.token() == LPAREN) {
+ pos = token.pos;
+ Name name = ident();
+ if (token.kind == LPAREN) {
return List.of(methodDeclaratorRest(
pos, mods, type, name, typarams,
isInterface, isVoid, dc));
@@ -2649,16 +2634,16 @@
List<JCTree> defs =
variableDeclaratorsRest(pos, mods, type, name, isInterface, dc,
new ListBuffer<JCTree>()).toList();
- storeEnd(defs.last(), S.endPos());
+ storeEnd(defs.last(), token.endPos);
accept(SEMI);
return defs;
} else {
- pos = S.pos();
+ pos = token.pos;
List<JCTree> err = isVoid
? List.<JCTree>of(toP(F.at(pos).MethodDef(mods, name, type, typarams,
List.<JCVariableDecl>nil(), List.<JCExpression>nil(), null, null)))
: null;
- return List.<JCTree>of(syntaxError(S.pos(), err, "expected", LPAREN));
+ return List.<JCTree>of(syntaxError(token.pos, err, "expected", LPAREN));
}
}
}
@@ -2686,27 +2671,27 @@
List<JCVariableDecl> params = formalParameters();
if (!isVoid) type = bracketsOpt(type);
List<JCExpression> thrown = List.nil();
- if (S.token() == THROWS) {
- S.nextToken();
+ if (token.kind == THROWS) {
+ nextToken();
thrown = qualidentList();
}
JCBlock body = null;
JCExpression defaultValue;
- if (S.token() == LBRACE) {
+ if (token.kind == LBRACE) {
body = block();
defaultValue = null;
} else {
- if (S.token() == DEFAULT) {
+ if (token.kind == DEFAULT) {
accept(DEFAULT);
defaultValue = annotationValue();
} else {
defaultValue = null;
}
accept(SEMI);
- if (S.pos() <= errorEndPos) {
+ if (token.pos <= errorEndPos) {
// error recovery
skip(false, true, false, false);
- if (S.token() == LBRACE) {
+ if (token.kind == LBRACE) {
body = block();
}
}
@@ -2725,8 +2710,8 @@
List<JCExpression> qualidentList() {
ListBuffer<JCExpression> ts = new ListBuffer<JCExpression>();
ts.append(qualident());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
ts.append(qualident());
}
return ts.toList();
@@ -2735,13 +2720,13 @@
/** TypeParametersOpt = ["<" TypeParameter {"," TypeParameter} ">"]
*/
List<JCTypeParameter> typeParametersOpt() {
- if (S.token() == LT) {
+ if (token.kind == LT) {
checkGenerics();
ListBuffer<JCTypeParameter> typarams = new ListBuffer<JCTypeParameter>();
- S.nextToken();
+ nextToken();
typarams.append(typeParameter());
- while (S.token() == COMMA) {
- S.nextToken();
+ while (token.kind == COMMA) {
+ nextToken();
typarams.append(typeParameter());
}
accept(GT);
@@ -2756,14 +2741,14 @@
* TypeVariable = Ident
*/
JCTypeParameter typeParameter() {
- int pos = S.pos();
+ int pos = token.pos;
Name name = ident();
ListBuffer<JCExpression> bounds = new ListBuffer<JCExpression>();
- if (S.token() == EXTENDS) {
- S.nextToken();
+ if (token.kind == EXTENDS) {
+ nextToken();
bounds.append(parseType());
- while (S.token() == AMP) {
- S.nextToken();
+ while (token.kind == AMP) {
+ nextToken();
bounds.append(parseType());
}
}
@@ -2778,10 +2763,10 @@
ListBuffer<JCVariableDecl> params = new ListBuffer<JCVariableDecl>();
JCVariableDecl lastParam = null;
accept(LPAREN);
- if (S.token() != RPAREN) {
+ if (token.kind != RPAREN) {
params.append(lastParam = formalParameter());
- while ((lastParam.mods.flags & Flags.VARARGS) == 0 && S.token() == COMMA) {
- S.nextToken();
+ while ((lastParam.mods.flags & Flags.VARARGS) == 0 && token.kind == COMMA) {
+ nextToken();
params.append(lastParam = formalParameter());
}
}
@@ -2802,11 +2787,11 @@
protected JCVariableDecl formalParameter() {
JCModifiers mods = optFinal(Flags.PARAMETER);
JCExpression type = parseType();
- if (S.token() == ELLIPSIS) {
+ if (token.kind == ELLIPSIS) {
checkVarargs();
mods.flags |= Flags.VARARGS;
- type = to(F.at(S.pos()).TypeArray(type));
- S.nextToken();
+ type = to(F.at(token.pos).TypeArray(type));
+ nextToken();
}
return variableDeclaratorId(mods, type);
}
@@ -2849,7 +2834,7 @@
/** Return precedence of operator represented by token,
* -1 if token is not a binary operator. @see TreeInfo.opPrec
*/
- static int prec(Token token) {
+ static int prec(TokenKind token) {
int oc = optag(token);
return (oc >= 0) ? TreeInfo.opPrec(oc) : -1;
}
@@ -2869,7 +2854,7 @@
/** Return operation tag of binary operator represented by token,
* -1 if token is not a binary operator.
*/
- static int optag(Token token) {
+ static int optag(TokenKind token) {
switch (token) {
case BARBAR:
return JCTree.OR;
@@ -2941,7 +2926,7 @@
/** Return operation tag of unary operator represented by token,
* -1 if token is not a binary operator.
*/
- static int unoptag(Token token) {
+ static int unoptag(TokenKind token) {
switch (token) {
case PLUS:
return JCTree.POS;
@@ -2963,7 +2948,7 @@
/** Return type tag of basic type represented by token,
* -1 if token is not a basic type identifier.
*/
- static int typetag(Token token) {
+ static int typetag(TokenKind token) {
switch (token) {
case BYTE:
return TypeTags.BYTE;
@@ -2988,49 +2973,49 @@
void checkGenerics() {
if (!allowGenerics) {
- error(S.pos(), "generics.not.supported.in.source", source.name);
+ error(token.pos, "generics.not.supported.in.source", source.name);
allowGenerics = true;
}
}
void checkVarargs() {
if (!allowVarargs) {
- error(S.pos(), "varargs.not.supported.in.source", source.name);
+ error(token.pos, "varargs.not.supported.in.source", source.name);
allowVarargs = true;
}
}
void checkForeach() {
if (!allowForeach) {
- error(S.pos(), "foreach.not.supported.in.source", source.name);
+ error(token.pos, "foreach.not.supported.in.source", source.name);
allowForeach = true;
}
}
void checkStaticImports() {
if (!allowStaticImport) {
- error(S.pos(), "static.import.not.supported.in.source", source.name);
+ error(token.pos, "static.import.not.supported.in.source", source.name);
allowStaticImport = true;
}
}
void checkAnnotations() {
if (!allowAnnotations) {
- error(S.pos(), "annotations.not.supported.in.source", source.name);
+ error(token.pos, "annotations.not.supported.in.source", source.name);
allowAnnotations = true;
}
}
void checkDiamond() {
if (!allowDiamond) {
- error(S.pos(), "diamond.not.supported.in.source", source.name);
+ error(token.pos, "diamond.not.supported.in.source", source.name);
allowDiamond = true;
}
}
void checkMulticatch() {
if (!allowMulticatch) {
- error(S.pos(), "multicatch.not.supported.in.source", source.name);
+ error(token.pos, "multicatch.not.supported.in.source", source.name);
allowMulticatch = true;
}
}
void checkTryWithResources() {
if (!allowTWR) {
- error(S.pos(), "try.with.resources.not.supported.in.source", source.name);
+ error(token.pos, "try.with.resources.not.supported.in.source", source.name);
allowTWR = true;
}
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavadocTokenizer.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,412 @@
+/*
+ * Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.parser.Tokens.Token;
+import com.sun.tools.javac.util.*;
+
+import java.nio.*;
+
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** An extension to the base lexical analyzer that captures
+ * and processes the contents of doc comments. It does so by
+ * translating Unicode escape sequences and by stripping the
+ * leading whitespace and starts from each line of the comment.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public class JavadocTokenizer extends JavaTokenizer {
+
+ /** Create a scanner from the input buffer. buffer must implement
+ * array() and compact(), and remaining() must be less than limit().
+ */
+ protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
+ super(fac, buffer);
+ }
+
+ /** Create a scanner from the input array. The array must have at
+ * least a single character of extra space.
+ */
+ protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
+ super(fac, input, inputLength);
+ }
+
+ /** The comment input buffer, index of next chacter to be read,
+ * index of one past last character in buffer.
+ */
+ private char[] buf;
+ private int bp;
+ private int buflen;
+
+ /** The current character.
+ */
+ private char ch;
+
+ /** The column number position of the current character.
+ */
+ private int col;
+
+ /** The buffer index of the last converted Unicode character
+ */
+ private int unicodeConversionBp = 0;
+
+ /**
+ * Buffer for doc comment.
+ */
+ private char[] docCommentBuffer = new char[1024];
+
+ /**
+ * Number of characters in doc comment buffer.
+ */
+ private int docCommentCount;
+
+ /**
+ * Translated and stripped contents of doc comment
+ */
+ private String docComment = null;
+
+
+ /** Unconditionally expand the comment buffer.
+ */
+ private void expandCommentBuffer() {
+ char[] newBuffer = new char[docCommentBuffer.length * 2];
+ System.arraycopy(docCommentBuffer, 0, newBuffer,
+ 0, docCommentBuffer.length);
+ docCommentBuffer = newBuffer;
+ }
+
+ /** Convert an ASCII digit from its base (8, 10, or 16)
+ * to its value.
+ */
+ private int digit(int base) {
+ char c = ch;
+ int result = Character.digit(c, base);
+ if (result >= 0 && c > 0x7f) {
+ ch = "0123456789abcdef".charAt(result);
+ }
+ return result;
+ }
+
+ /** Convert Unicode escape; bp points to initial '\' character
+ * (Spec 3.3).
+ */
+ private void convertUnicode() {
+ if (ch == '\\' && unicodeConversionBp != bp) {
+ bp++; ch = buf[bp]; col++;
+ if (ch == 'u') {
+ do {
+ bp++; ch = buf[bp]; col++;
+ } while (ch == 'u');
+ int limit = bp + 3;
+ if (limit < buflen) {
+ int d = digit(16);
+ int code = d;
+ while (bp < limit && d >= 0) {
+ bp++; ch = buf[bp]; col++;
+ d = digit(16);
+ code = (code << 4) + d;
+ }
+ if (d >= 0) {
+ ch = (char)code;
+ unicodeConversionBp = bp;
+ return;
+ }
+ }
+ // "illegal.Unicode.esc", reported by base scanner
+ } else {
+ bp--;
+ ch = '\\';
+ col--;
+ }
+ }
+ }
+
+
+ /** Read next character.
+ */
+ private void scanChar() {
+ bp++;
+ ch = buf[bp];
+ switch (ch) {
+ case '\r': // return
+ col = 0;
+ break;
+ case '\n': // newline
+ if (bp == 0 || buf[bp-1] != '\r') {
+ col = 0;
+ }
+ break;
+ case '\t': // tab
+ col = (col / TabInc * TabInc) + TabInc;
+ break;
+ case '\\': // possible Unicode
+ col++;
+ convertUnicode();
+ break;
+ default:
+ col++;
+ break;
+ }
+ }
+
+ @Override
+ public Token readToken() {
+ docComment = null;
+ Token tk = super.readToken();
+ tk.docComment = docComment;
+ return tk;
+ }
+
+ /**
+ * Read next character in doc comment, skipping over double '\' characters.
+ * If a double '\' is skipped, put in the buffer and update buffer count.
+ */
+ private void scanDocCommentChar() {
+ scanChar();
+ if (ch == '\\') {
+ if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = ch;
+ bp++; col++;
+ } else {
+ convertUnicode();
+ }
+ }
+ }
+
+ /**
+ * Process a doc comment and make the string content available.
+ * Strips leading whitespace and stars.
+ */
+ @SuppressWarnings("fallthrough")
+ protected void processComment(int pos, int endPos, CommentStyle style) {
+ if (style != CommentStyle.JAVADOC) {
+ return;
+ }
+
+ buf = reader.getRawCharacters(pos, endPos);
+ buflen = buf.length;
+ bp = 0;
+ col = 0;
+
+ docCommentCount = 0;
+
+ boolean firstLine = true;
+
+ // Skip over first slash
+ scanDocCommentChar();
+ // Skip over first star
+ scanDocCommentChar();
+
+ // consume any number of stars
+ while (bp < buflen && ch == '*') {
+ scanDocCommentChar();
+ }
+ // is the comment in the form /**/, /***/, /****/, etc. ?
+ if (bp < buflen && ch == '/') {
+ docComment = "";
+ return;
+ }
+
+ // skip a newline on the first line of the comment.
+ if (bp < buflen) {
+ if (ch == LF) {
+ scanDocCommentChar();
+ firstLine = false;
+ } else if (ch == CR) {
+ scanDocCommentChar();
+ if (ch == LF) {
+ scanDocCommentChar();
+ firstLine = false;
+ }
+ }
+ }
+
+ outerLoop:
+
+ // The outerLoop processes the doc comment, looping once
+ // for each line. For each line, it first strips off
+ // whitespace, then it consumes any stars, then it
+ // puts the rest of the line into our buffer.
+ while (bp < buflen) {
+
+ // The wsLoop consumes whitespace from the beginning
+ // of each line.
+ wsLoop:
+
+ while (bp < buflen) {
+ switch(ch) {
+ case ' ':
+ scanDocCommentChar();
+ break;
+ case '\t':
+ col = ((col - 1) / TabInc * TabInc) + TabInc;
+ scanDocCommentChar();
+ break;
+ case FF:
+ col = 0;
+ scanDocCommentChar();
+ break;
+// Treat newline at beginning of line (blank line, no star)
+// as comment text. Old Javadoc compatibility requires this.
+/*---------------------------------*
+ case CR: // (Spec 3.4)
+ scanDocCommentChar();
+ if (ch == LF) {
+ col = 0;
+ scanDocCommentChar();
+ }
+ break;
+ case LF: // (Spec 3.4)
+ scanDocCommentChar();
+ break;
+*---------------------------------*/
+ default:
+ // we've seen something that isn't whitespace;
+ // jump out.
+ break wsLoop;
+ }
+ }
+
+ // Are there stars here? If so, consume them all
+ // and check for the end of comment.
+ if (ch == '*') {
+ // skip all of the stars
+ do {
+ scanDocCommentChar();
+ } while (ch == '*');
+
+ // check for the closing slash.
+ if (ch == '/') {
+ // We're done with the doc comment
+ // scanChar() and breakout.
+ break outerLoop;
+ }
+ } else if (! firstLine) {
+ //The current line does not begin with a '*' so we will indent it.
+ for (int i = 1; i < col; i++) {
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = ' ';
+ }
+ }
+
+ // The textLoop processes the rest of the characters
+ // on the line, adding them to our buffer.
+ textLoop:
+ while (bp < buflen) {
+ switch (ch) {
+ case '*':
+ // Is this just a star? Or is this the
+ // end of a comment?
+ scanDocCommentChar();
+ if (ch == '/') {
+ // This is the end of the comment,
+ // set ch and return our buffer.
+ break outerLoop;
+ }
+ // This is just an ordinary star. Add it to
+ // the buffer.
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = '*';
+ break;
+ case ' ':
+ case '\t':
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = ch;
+ scanDocCommentChar();
+ break;
+ case FF:
+ scanDocCommentChar();
+ break textLoop; // treat as end of line
+ case CR: // (Spec 3.4)
+ scanDocCommentChar();
+ if (ch != LF) {
+ // Canonicalize CR-only line terminator to LF
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = (char)LF;
+ break textLoop;
+ }
+ /* fall through to LF case */
+ case LF: // (Spec 3.4)
+ // We've seen a newline. Add it to our
+ // buffer and break out of this loop,
+ // starting fresh on a new line.
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = ch;
+ scanDocCommentChar();
+ break textLoop;
+ default:
+ // Add the character to our buffer.
+ if (docCommentCount == docCommentBuffer.length)
+ expandCommentBuffer();
+ docCommentBuffer[docCommentCount++] = ch;
+ scanDocCommentChar();
+ }
+ } // end textLoop
+ firstLine = false;
+ } // end outerLoop
+
+ if (docCommentCount > 0) {
+ int i = docCommentCount - 1;
+ trailLoop:
+ while (i > -1) {
+ switch (docCommentBuffer[i]) {
+ case '*':
+ i--;
+ break;
+ default:
+ break trailLoop;
+ }
+ }
+ docCommentCount = i + 1;
+
+ // Store the text of the doc comment
+ docComment = new String(docCommentBuffer, 0 , docCommentCount);
+ } else {
+ docComment = "";
+ }
+ }
+
+ /** Build a map for translating between line numbers and
+ * positions in the input.
+ *
+ * @return a LineMap */
+ public Position.LineMap getLineMap() {
+ char[] buf = reader.getRawCharacters();
+ return Position.makeLineMap(buf, buf.length, true);
+ }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Keywords.java Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,98 +0,0 @@
-/*
- * Copyright (c) 2002, 2010, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation. Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import com.sun.tools.javac.util.Context;
-import com.sun.tools.javac.util.Log;
-import com.sun.tools.javac.util.Name;
-import com.sun.tools.javac.util.Names;
-
-import static com.sun.tools.javac.parser.Token.*;
-
-/**
- * Map from Name to Token and Token to String.
- *
- * <p><b>This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.</b>
- */
-public class Keywords {
- public static final Context.Key<Keywords> keywordsKey =
- new Context.Key<Keywords>();
-
- public static Keywords instance(Context context) {
- Keywords instance = context.get(keywordsKey);
- if (instance == null)
- instance = new Keywords(context);
- return instance;
- }
-
- private final Names names;
-
- protected Keywords(Context context) {
- context.put(keywordsKey, this);
- names = Names.instance(context);
-
- for (Token t : Token.values()) {
- if (t.name != null)
- enterKeyword(t.name, t);
- else
- tokenName[t.ordinal()] = null;
- }
-
- key = new Token[maxKey+1];
- for (int i = 0; i <= maxKey; i++) key[i] = IDENTIFIER;
- for (Token t : Token.values()) {
- if (t.name != null)
- key[tokenName[t.ordinal()].getIndex()] = t;
- }
- }
-
-
- public Token key(Name name) {
- return (name.getIndex() > maxKey) ? IDENTIFIER : key[name.getIndex()];
- }
-
- /**
- * Keyword array. Maps name indices to Token.
- */
- private final Token[] key;
-
- /** The number of the last entered keyword.
- */
- private int maxKey = 0;
-
- /** The names of all tokens.
- */
- private Name[] tokenName = new Name[Token.values().length];
-
- private void enterKeyword(String s, Token token) {
- Name n = names.fromString(s);
- tokenName[token.ordinal()] = n;
- if (n.getIndex() > maxKey) maxKey = n.getIndex();
- }
-}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Lexer.java Mon Oct 24 13:00:20 2011 +0100
@@ -25,7 +25,7 @@
package com.sun.tools.javac.parser;
-import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.parser.Tokens.*;
import com.sun.tools.javac.util.Position.LineMap;
/**
@@ -40,22 +40,26 @@
public interface Lexer {
/**
- * Has a @deprecated been encountered in last doc comment?
- * This needs to be reset by client with resetDeprecatedFlag.
+ * Consume the next token.
*/
- boolean deprecatedFlag();
+ void nextToken();
- void resetDeprecatedFlag();
+ /**
+ * Return current token.
+ */
+ Token token();
/**
- * Returns the documentation string of the current token.
+ * Return the last character position of the previous token.
*/
- String docComment();
+ Token prevToken();
/**
- * Return the last character position of the current token.
+ * Splits the current token in two and return the first (splitted) token.
+ * For instance '<<<' is splitted into two tokens '<' and '<<' respectively,
+ * and the latter is returned.
*/
- int endPos();
+ Token split();
/**
* Return the position where a lexical error occurred;
@@ -74,69 +78,4 @@
* @return a LineMap
*/
LineMap getLineMap();
-
- /**
- * Returns a copy of the input buffer, up to its inputLength.
- * Unicode escape sequences are not translated.
- */
- char[] getRawCharacters();
-
- /**
- * Returns a copy of a character array subset of the input buffer.
- * The returned array begins at the <code>beginIndex</code> and
- * extends to the character at index <code>endIndex - 1</code>.
- * Thus the length of the substring is <code>endIndex-beginIndex</code>.
- * This behavior is like
- * <code>String.substring(beginIndex, endIndex)</code>.
- * Unicode escape sequences are not translated.
- *
- * @param beginIndex the beginning index, inclusive.
- * @param endIndex the ending index, exclusive.
- * @throws IndexOutOfBounds if either offset is outside of the
- * array bounds
- */
- char[] getRawCharacters(int beginIndex, int endIndex);
-
- /**
- * Return the name of an identifier or token for the current token.
- */
- Name name();
-
- /**
- * Read token.
- */
- void nextToken();
-
- /**
- * Return the current token's position: a 0-based
- * offset from beginning of the raw input stream
- * (before unicode translation)
- */
- int pos();
-
- /**
- * Return the last character position of the previous token.
- */
- int prevEndPos();
-
- /**
- * Return the radix of a numeric literal token.
- */
- int radix();
-
- /**
- * The value of a literal token, recorded as a string.
- * For integers, leading 0x and 'l' suffixes are suppressed.
- */
- String stringVal();
-
- /**
- * Return the current token, set by nextToken().
- */
- Token token();
-
- /**
- * Sets the current token.
- */
- void token(Token token);
}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ParserFactory.java Mon Oct 24 13:00:20 2011 +0100
@@ -55,7 +55,7 @@
final TreeMaker F;
final Log log;
- final Keywords keywords;
+ final Tokens tokens;
final Source source;
final Names names;
final Options options;
@@ -67,7 +67,7 @@
this.F = TreeMaker.instance(context);
this.log = Log.instance(context);
this.names = Names.instance(context);
- this.keywords = Keywords.instance(context);
+ this.tokens = Tokens.instance(context);
this.source = Source.instance(context);
this.options = Options.instance(context);
this.scannerFactory = ScannerFactory.instance(context);
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java Mon Oct 24 13:00:20 2011 +0100
@@ -27,13 +27,11 @@
import java.nio.*;
-import com.sun.tools.javac.code.Source;
-import com.sun.tools.javac.file.JavacFileManager;
import com.sun.tools.javac.util.*;
+import com.sun.tools.javac.util.Position.LineMap;
+import com.sun.tools.javac.parser.JavaTokenizer.*;
-
-import static com.sun.tools.javac.parser.Token.*;
-import static com.sun.tools.javac.util.LayoutCharacters.*;
+import static com.sun.tools.javac.parser.Tokens.*;
/** The lexical analyzer maps an input stream consisting of
* ASCII characters and Unicode escapes into a token sequence.
@@ -45,119 +43,17 @@
*/
public class Scanner implements Lexer {
- private static boolean scannerDebug = false;
-
- /* Output variables; set by nextToken():
- */
+ private Tokens tokens;
/** The token, set by nextToken().
*/
private Token token;
- /** Allow hex floating-point literals.
- */
- private boolean allowHexFloats;
-
- /** Allow binary literals.
- */
- private boolean allowBinaryLiterals;
-
- /** Allow underscores in literals.
- */
- private boolean allowUnderscoresInLiterals;
-
- /** The source language setting.
- */
- private Source source;
-
- /** The token's position, 0-based offset from beginning of text.
- */
- private int pos;
-
- /** Character position just after the last character of the token.
+ /** The previous token, set by nextToken().
*/
- private int endPos;
-
- /** The last character position of the previous token.
- */
- private int prevEndPos;
-
- /** The position where a lexical error occurred;
- */
- private int errPos = Position.NOPOS;
-
- /** The name of an identifier or token:
- */
- private Name name;
-
- /** The radix of a numeric literal token.
- */
- private int radix;
-
- /** Has a @deprecated been encountered in last doc comment?
- * this needs to be reset by client.
- */
- protected boolean deprecatedFlag = false;
-
- /** A character buffer for literals.
- */
- private char[] sbuf = new char[128];
- private int sp;
+ private Token prevToken;
- /** The input buffer, index of next chacter to be read,
- * index of one past last character in buffer.
- */
- private char[] buf;
- private int bp;
- private int buflen;
- private int eofPos;
-
- /** The current character.
- */
- private char ch;
-
- /** The buffer index of the last converted unicode character
- */
- private int unicodeConversionBp = -1;
-
- /** The log to be used for error reporting.
- */
- private final Log log;
-
- /** The name table. */
- private final Names names;
-
- /** The keyword table. */
- private final Keywords keywords;
-
- /** Common code for constructors. */
- private Scanner(ScannerFactory fac) {
- log = fac.log;
- names = fac.names;
- keywords = fac.keywords;
- source = fac.source;
- allowBinaryLiterals = source.allowBinaryLiterals();
- allowHexFloats = source.allowHexFloats();
- allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
- }
-
- private static final boolean hexFloatsWork = hexFloatsWork();
- private static boolean hexFloatsWork() {
- try {
- Float.valueOf("0x1.0p1");
- return true;
- } catch (NumberFormatException ex) {
- return false;
- }
- }
-
- /** Create a scanner from the input buffer. buffer must implement
- * array() and compact(), and remaining() must be less than limit().
- */
- protected Scanner(ScannerFactory fac, CharBuffer buffer) {
- this(fac, JavacFileManager.toArray(buffer), buffer.limit());
- }
-
+ private JavaTokenizer tokenizer;
/**
* Create a scanner from the input array. This method might
* modify the array. To avoid copying the input array, ensure
@@ -169,972 +65,49 @@
* @param inputLength the size of the input.
* Must be positive and less than or equal to input.length.
*/
- protected Scanner(ScannerFactory fac, char[] input, int inputLength) {
- this(fac);
- eofPos = inputLength;
- if (inputLength == input.length) {
- if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
- inputLength--;
- } else {
- char[] newInput = new char[inputLength + 1];
- System.arraycopy(input, 0, newInput, 0, input.length);
- input = newInput;
- }
- }
- buf = input;
- buflen = inputLength;
- buf[buflen] = EOI;
- bp = -1;
- scanChar();
- }
-
- /** Report an error at the given position using the provided arguments.
- */
- private void lexError(int pos, String key, Object... args) {
- log.error(pos, key, args);
- token = ERROR;
- errPos = pos;
- }
-
- /** Report an error at the current token position using the provided
- * arguments.
- */
- private void lexError(String key, Object... args) {
- lexError(pos, key, args);
- }
-
- /** Convert an ASCII digit from its base (8, 10, or 16)
- * to its value.
- */
- private int digit(int base) {
- char c = ch;
- int result = Character.digit(c, base);
- if (result >= 0 && c > 0x7f) {
- lexError(pos+1, "illegal.nonascii.digit");
- ch = "0123456789abcdef".charAt(result);
- }
- return result;
- }
-
- /** Convert unicode escape; bp points to initial '\' character
- * (Spec 3.3).
- */
- private void convertUnicode() {
- if (ch == '\\' && unicodeConversionBp != bp) {
- bp++; ch = buf[bp];
- if (ch == 'u') {
- do {
- bp++; ch = buf[bp];
- } while (ch == 'u');
- int limit = bp + 3;
- if (limit < buflen) {
- int d = digit(16);
- int code = d;
- while (bp < limit && d >= 0) {
- bp++; ch = buf[bp];
- d = digit(16);
- code = (code << 4) + d;
- }
- if (d >= 0) {
- ch = (char)code;
- unicodeConversionBp = bp;
- return;
- }
- }
- lexError(bp, "illegal.unicode.esc");
- } else {
- bp--;
- ch = '\\';
- }
- }
- }
-
- /** Read next character.
- */
- private void scanChar() {
- ch = buf[++bp];
- if (ch == '\\') {
- convertUnicode();
- }
- }
-
- /** Read next character in comment, skipping over double '\' characters.
- */
- private void scanCommentChar() {
- scanChar();
- if (ch == '\\') {
- if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
- bp++;
- } else {
- convertUnicode();
- }
- }
- }
-
- /** Append a character to sbuf.
- */
- private void putChar(char ch) {
- if (sp == sbuf.length) {
- char[] newsbuf = new char[sbuf.length * 2];
- System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
- sbuf = newsbuf;
- }
- sbuf[sp++] = ch;
- }
-
- /** Read next character in character or string literal and copy into sbuf.
- */
- private void scanLitChar() {
- if (ch == '\\') {
- if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
- bp++;
- putChar('\\');
- scanChar();
- } else {
- scanChar();
- switch (ch) {
- case '0': case '1': case '2': case '3':
- case '4': case '5': case '6': case '7':
- char leadch = ch;
- int oct = digit(8);
- scanChar();
- if ('0' <= ch && ch <= '7') {
- oct = oct * 8 + digit(8);
- scanChar();
- if (leadch <= '3' && '0' <= ch && ch <= '7') {
- oct = oct * 8 + digit(8);
- scanChar();
- }
- }
- putChar((char)oct);
- break;
- case 'b':
- putChar('\b'); scanChar(); break;
- case 't':
- putChar('\t'); scanChar(); break;
- case 'n':
- putChar('\n'); scanChar(); break;
- case 'f':
- putChar('\f'); scanChar(); break;
- case 'r':
- putChar('\r'); scanChar(); break;
- case '\'':
- putChar('\''); scanChar(); break;
- case '\"':
- putChar('\"'); scanChar(); break;
- case '\\':
- putChar('\\'); scanChar(); break;
- default:
- lexError(bp, "illegal.esc.char");
- }
- }
- } else if (bp != buflen) {
- putChar(ch); scanChar();
- }
- }
-
- private void scanDigits(int digitRadix) {
- char saveCh;
- int savePos;
- do {
- if (ch != '_') {
- putChar(ch);
- } else {
- if (!allowUnderscoresInLiterals) {
- lexError("unsupported.underscore.lit", source.name);
- allowUnderscoresInLiterals = true;
- }
- }
- saveCh = ch;
- savePos = bp;
- scanChar();
- } while (digit(digitRadix) >= 0 || ch == '_');
- if (saveCh == '_')
- lexError(savePos, "illegal.underscore");
- }
-
- /** Read fractional part of hexadecimal floating point number.
- */
- private void scanHexExponentAndSuffix() {
- if (ch == 'p' || ch == 'P') {
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (ch == '+' || ch == '-') {
- putChar(ch);
- scanChar();
- }
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- if (!allowHexFloats) {
- lexError("unsupported.fp.lit", source.name);
- allowHexFloats = true;
- }
- else if (!hexFloatsWork)
- lexError("unsupported.cross.fp.lit");
- } else
- lexError("malformed.fp.lit");
- } else {
- lexError("malformed.fp.lit");
- }
- if (ch == 'f' || ch == 'F') {
- putChar(ch);
- scanChar();
- token = FLOATLITERAL;
- } else {
- if (ch == 'd' || ch == 'D') {
- putChar(ch);
- scanChar();
- }
- token = DOUBLELITERAL;
- }
- }
-
- /** Read fractional part of floating point number.
- */
- private void scanFraction() {
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- }
- int sp1 = sp;
- if (ch == 'e' || ch == 'E') {
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (ch == '+' || ch == '-') {
- putChar(ch);
- scanChar();
- }
- skipIllegalUnderscores();
- if ('0' <= ch && ch <= '9') {
- scanDigits(10);
- return;
- }
- lexError("malformed.fp.lit");
- sp = sp1;
- }
- }
-
- /** Read fractional part and 'd' or 'f' suffix of floating point number.
- */
- private void scanFractionAndSuffix() {
- this.radix = 10;
- scanFraction();
- if (ch == 'f' || ch == 'F') {
- putChar(ch);
- scanChar();
- token = FLOATLITERAL;
- } else {
- if (ch == 'd' || ch == 'D') {
- putChar(ch);
- scanChar();
- }
- token = DOUBLELITERAL;
- }
- }
-
- /** Read fractional part and 'd' or 'f' suffix of floating point number.
- */
- private void scanHexFractionAndSuffix(boolean seendigit) {
- this.radix = 16;
- Assert.check(ch == '.');
- putChar(ch);
- scanChar();
- skipIllegalUnderscores();
- if (digit(16) >= 0) {
- seendigit = true;
- scanDigits(16);
- }
- if (!seendigit)
- lexError("invalid.hex.number");
- else
- scanHexExponentAndSuffix();
- }
-
- private void skipIllegalUnderscores() {
- if (ch == '_') {
- lexError(bp, "illegal.underscore");
- while (ch == '_')
- scanChar();
- }
- }
-
- /** Read a number.
- * @param radix The radix of the number; one of 2, j8, 10, 16.
- */
- private void scanNumber(int radix) {
- this.radix = radix;
- // for octal, allow base-10 digit in case it's a float literal
- int digitRadix = (radix == 8 ? 10 : radix);
- boolean seendigit = false;
- if (digit(digitRadix) >= 0) {
- seendigit = true;
- scanDigits(digitRadix);
- }
- if (radix == 16 && ch == '.') {
- scanHexFractionAndSuffix(seendigit);
- } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) {
- scanHexExponentAndSuffix();
- } else if (digitRadix == 10 && ch == '.') {
- putChar(ch);
- scanChar();
- scanFractionAndSuffix();
- } else if (digitRadix == 10 &&
- (ch == 'e' || ch == 'E' ||
- ch == 'f' || ch == 'F' ||
- ch == 'd' || ch == 'D')) {
- scanFractionAndSuffix();
- } else {
- if (ch == 'l' || ch == 'L') {
- scanChar();
- token = LONGLITERAL;
- } else {
- token = INTLITERAL;
- }
- }
- }
-
- /** Read an identifier.
- */
- private void scanIdent() {
- boolean isJavaIdentifierPart;
- char high;
- do {
- if (sp == sbuf.length) putChar(ch); else sbuf[sp++] = ch;
- // optimization, was: putChar(ch);
-
- scanChar();
- switch (ch) {
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F': case 'G': case 'H': case 'I': case 'J':
- case 'K': case 'L': case 'M': case 'N': case 'O':
- case 'P': case 'Q': case 'R': case 'S': case 'T':
- case 'U': case 'V': case 'W': case 'X': case 'Y':
- case 'Z':
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f': case 'g': case 'h': case 'i': case 'j':
- case 'k': case 'l': case 'm': case 'n': case 'o':
- case 'p': case 'q': case 'r': case 's': case 't':
- case 'u': case 'v': case 'w': case 'x': case 'y':
- case 'z':
- case '$': case '_':
- case '0': case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- case '\u0000': case '\u0001': case '\u0002': case '\u0003':
- case '\u0004': case '\u0005': case '\u0006': case '\u0007':
- case '\u0008': case '\u000E': case '\u000F': case '\u0010':
- case '\u0011': case '\u0012': case '\u0013': case '\u0014':
- case '\u0015': case '\u0016': case '\u0017':
- case '\u0018': case '\u0019': case '\u001B':
- case '\u007F':
- break;
- case '\u001A': // EOI is also a legal identifier part
- if (bp >= buflen) {
- name = names.fromChars(sbuf, 0, sp);
- token = keywords.key(name);
- return;
- }
- break;
- default:
- if (ch < '\u0080') {
- // all ASCII range chars already handled, above
- isJavaIdentifierPart = false;
- } else {
- high = scanSurrogates();
- if (high != 0) {
- if (sp == sbuf.length) {
- putChar(high);
- } else {
- sbuf[sp++] = high;
- }
- isJavaIdentifierPart = Character.isJavaIdentifierPart(
- Character.toCodePoint(high, ch));
- } else {
- isJavaIdentifierPart = Character.isJavaIdentifierPart(ch);
- }
- }
- if (!isJavaIdentifierPart) {
- name = names.fromChars(sbuf, 0, sp);
- token = keywords.key(name);
- return;
- }
- }
- } while (true);
+ protected Scanner(ScannerFactory fac, CharBuffer buf) {
+ this(fac, new JavaTokenizer(fac, buf));
}
- /** Are surrogates supported?
- */
- final static boolean surrogatesSupported = surrogatesSupported();
- private static boolean surrogatesSupported() {
- try {
- Character.isHighSurrogate('a');
- return true;
- } catch (NoSuchMethodError ex) {
- return false;
- }
- }
-
- /** Scan surrogate pairs. If 'ch' is a high surrogate and
- * the next character is a low surrogate, then put the low
- * surrogate in 'ch', and return the high surrogate.
- * otherwise, just return 0.
- */
- private char scanSurrogates() {
- if (surrogatesSupported && Character.isHighSurrogate(ch)) {
- char high = ch;
-
- scanChar();
-
- if (Character.isLowSurrogate(ch)) {
- return high;
- }
-
- ch = high;
- }
-
- return 0;
- }
-
- /** Return true if ch can be part of an operator.
- */
- private boolean isSpecial(char ch) {
- switch (ch) {
- case '!': case '%': case '&': case '*': case '?':
- case '+': case '-': case ':': case '<': case '=':
- case '>': case '^': case '|': case '~':
- case '@':
- return true;
- default:
- return false;
- }
- }
-
- /** Read longest possible sequence of special characters and convert
- * to token.
- */
- private void scanOperator() {
- while (true) {
- putChar(ch);
- Name newname = names.fromChars(sbuf, 0, sp);
- if (keywords.key(newname) == IDENTIFIER) {
- sp--;
- break;
- }
- name = newname;
- token = keywords.key(newname);
- scanChar();
- if (!isSpecial(ch)) break;
- }
- }
-
- /**
- * Scan a documention comment; determine if a deprecated tag is present.
- * Called once the initial /, * have been skipped, positioned at the second *
- * (which is treated as the beginning of the first line).
- * Stops positioned at the closing '/'.
- */
- @SuppressWarnings("fallthrough")
- private void scanDocComment() {
- boolean deprecatedPrefix = false;
-
- forEachLine:
- while (bp < buflen) {
-
- // Skip optional WhiteSpace at beginning of line
- while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
- scanCommentChar();
- }
-
- // Skip optional consecutive Stars
- while (bp < buflen && ch == '*') {
- scanCommentChar();
- if (ch == '/') {
- return;
- }
- }
-
- // Skip optional WhiteSpace after Stars
- while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
- scanCommentChar();
- }
-
- deprecatedPrefix = false;
- // At beginning of line in the JavaDoc sense.
- if (bp < buflen && ch == '@' && !deprecatedFlag) {
- scanCommentChar();
- if (bp < buflen && ch == 'd') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'p') {
- scanCommentChar();
- if (bp < buflen && ch == 'r') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'c') {
- scanCommentChar();
- if (bp < buflen && ch == 'a') {
- scanCommentChar();
- if (bp < buflen && ch == 't') {
- scanCommentChar();
- if (bp < buflen && ch == 'e') {
- scanCommentChar();
- if (bp < buflen && ch == 'd') {
- deprecatedPrefix = true;
- scanCommentChar();
- }}}}}}}}}}}
- if (deprecatedPrefix && bp < buflen) {
- if (Character.isWhitespace(ch)) {
- deprecatedFlag = true;
- } else if (ch == '*') {
- scanCommentChar();
- if (ch == '/') {
- deprecatedFlag = true;
- return;
- }
- }
- }
-
- // Skip rest of line
- while (bp < buflen) {
- switch (ch) {
- case '*':
- scanCommentChar();
- if (ch == '/') {
- return;
- }
- break;
- case CR: // (Spec 3.4)
- scanCommentChar();
- if (ch != LF) {
- continue forEachLine;
- }
- /* fall through to LF case */
- case LF: // (Spec 3.4)
- scanCommentChar();
- continue forEachLine;
- default:
- scanCommentChar();
- }
- } // rest of line
- } // forEachLine
- return;
- }
-
- /** The value of a literal token, recorded as a string.
- * For integers, leading 0x and 'l' suffixes are suppressed.
- */
- public String stringVal() {
- return new String(sbuf, 0, sp);
+ protected Scanner(ScannerFactory fac, char[] buf, int inputLength) {
+ this(fac, new JavaTokenizer(fac, buf, inputLength));
}
- /** Read token.
- */
- public void nextToken() {
-
- try {
- prevEndPos = endPos;
- sp = 0;
-
- while (true) {
- pos = bp;
- switch (ch) {
- case ' ': // (Spec 3.6)
- case '\t': // (Spec 3.6)
- case FF: // (Spec 3.6)
- do {
- scanChar();
- } while (ch == ' ' || ch == '\t' || ch == FF);
- endPos = bp;
- processWhiteSpace();
- break;
- case LF: // (Spec 3.4)
- scanChar();
- endPos = bp;
- processLineTerminator();
- break;
- case CR: // (Spec 3.4)
- scanChar();
- if (ch == LF) {
- scanChar();
- }
- endPos = bp;
- processLineTerminator();
- break;
- case 'A': case 'B': case 'C': case 'D': case 'E':
- case 'F': case 'G': case 'H': case 'I': case 'J':
- case 'K': case 'L': case 'M': case 'N': case 'O':
- case 'P': case 'Q': case 'R': case 'S': case 'T':
- case 'U': case 'V': case 'W': case 'X': case 'Y':
- case 'Z':
- case 'a': case 'b': case 'c': case 'd': case 'e':
- case 'f': case 'g': case 'h': case 'i': case 'j':
- case 'k': case 'l': case 'm': case 'n': case 'o':
- case 'p': case 'q': case 'r': case 's': case 't':
- case 'u': case 'v': case 'w': case 'x': case 'y':
- case 'z':
- case '$': case '_':
- scanIdent();
- return;
- case '0':
- scanChar();
- if (ch == 'x' || ch == 'X') {
- scanChar();
- skipIllegalUnderscores();
- if (ch == '.') {
- scanHexFractionAndSuffix(false);
- } else if (digit(16) < 0) {
- lexError("invalid.hex.number");
- } else {
- scanNumber(16);
- }
- } else if (ch == 'b' || ch == 'B') {
- if (!allowBinaryLiterals) {
- lexError("unsupported.binary.lit", source.name);
- allowBinaryLiterals = true;
- }
- scanChar();
- skipIllegalUnderscores();
- if (digit(2) < 0) {
- lexError("invalid.binary.number");
- } else {
- scanNumber(2);
- }
- } else {
- putChar('0');
- if (ch == '_') {
- int savePos = bp;
- do {
- scanChar();
- } while (ch == '_');
- if (digit(10) < 0) {
- lexError(savePos, "illegal.underscore");
- }
- }
- scanNumber(8);
- }
- return;
- case '1': case '2': case '3': case '4':
- case '5': case '6': case '7': case '8': case '9':
- scanNumber(10);
- return;
- case '.':
- scanChar();
- if ('0' <= ch && ch <= '9') {
- putChar('.');
- scanFractionAndSuffix();
- } else if (ch == '.') {
- putChar('.'); putChar('.');
- scanChar();
- if (ch == '.') {
- scanChar();
- putChar('.');
- token = ELLIPSIS;
- } else {
- lexError("malformed.fp.lit");
- }
- } else {
- token = DOT;
- }
- return;
- case ',':
- scanChar(); token = COMMA; return;
- case ';':
- scanChar(); token = SEMI; return;
- case '(':
- scanChar(); token = LPAREN; return;
- case ')':
- scanChar(); token = RPAREN; return;
- case '[':
- scanChar(); token = LBRACKET; return;
- case ']':
- scanChar(); token = RBRACKET; return;
- case '{':
- scanChar(); token = LBRACE; return;
- case '}':
- scanChar(); token = RBRACE; return;
- case '/':
- scanChar();
- if (ch == '/') {
- do {
- scanCommentChar();
- } while (ch != CR && ch != LF && bp < buflen);
- if (bp < buflen) {
- endPos = bp;
- processComment(CommentStyle.LINE);
- }
- break;
- } else if (ch == '*') {
- scanChar();
- CommentStyle style;
- if (ch == '*') {
- style = CommentStyle.JAVADOC;
- scanDocComment();
- } else {
- style = CommentStyle.BLOCK;
- while (bp < buflen) {
- if (ch == '*') {
- scanChar();
- if (ch == '/') break;
- } else {
- scanCommentChar();
- }
- }
- }
- if (ch == '/') {
- scanChar();
- endPos = bp;
- processComment(style);
- break;
- } else {
- lexError("unclosed.comment");
- return;
- }
- } else if (ch == '=') {
- name = names.slashequals;
- token = SLASHEQ;
- scanChar();
- } else {
- name = names.slash;
- token = SLASH;
- }
- return;
- case '\'':
- scanChar();
- if (ch == '\'') {
- lexError("empty.char.lit");
- } else {
- if (ch == CR || ch == LF)
- lexError(pos, "illegal.line.end.in.char.lit");
- scanLitChar();
- if (ch == '\'') {
- scanChar();
- token = CHARLITERAL;
- } else {
- lexError(pos, "unclosed.char.lit");
- }
- }
- return;
- case '\"':
- scanChar();
- while (ch != '\"' && ch != CR && ch != LF && bp < buflen)
- scanLitChar();
- if (ch == '\"') {
- token = STRINGLITERAL;
- scanChar();
- } else {
- lexError(pos, "unclosed.str.lit");
- }
- return;
- default:
- if (isSpecial(ch)) {
- scanOperator();
- } else {
- boolean isJavaIdentifierStart;
- if (ch < '\u0080') {
- // all ASCII range chars already handled, above
- isJavaIdentifierStart = false;
- } else {
- char high = scanSurrogates();
- if (high != 0) {
- if (sp == sbuf.length) {
- putChar(high);
- } else {
- sbuf[sp++] = high;
- }
-
- isJavaIdentifierStart = Character.isJavaIdentifierStart(
- Character.toCodePoint(high, ch));
- } else {
- isJavaIdentifierStart = Character.isJavaIdentifierStart(ch);
- }
- }
- if (isJavaIdentifierStart) {
- scanIdent();
- } else if (bp == buflen || ch == EOI && bp+1 == buflen) { // JLS 3.5
- token = EOF;
- pos = bp = eofPos;
- } else {
- lexError("illegal.char", String.valueOf((int)ch));
- scanChar();
- }
- }
- return;
- }
- }
- } finally {
- endPos = bp;
- if (scannerDebug)
- System.out.println("nextToken(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
- }
+ protected Scanner(ScannerFactory fac, JavaTokenizer tokenizer) {
+ this.tokenizer = tokenizer;
+ tokens = fac.tokens;
+ token = prevToken = DUMMY;
}
- /** Return the current token, set by nextToken().
- */
public Token token() {
return token;
}
- /** Sets the current token.
- * This method is primarily used to update the token stream when the
- * parser is handling the end of nested type arguments such as
- * {@code List<List<String>>} and needs to disambiguate between
- * repeated use of ">" and relation operators such as ">>" and ">>>". Noting
- * that this does not handle arbitrary tokens containing Unicode escape
- * sequences.
- */
- public void token(Token token) {
- pos += this.token.name.length() - token.name.length();
- prevEndPos = pos;
- this.token = token;
- }
-
- /** Return the current token's position: a 0-based
- * offset from beginning of the raw input stream
- * (before unicode translation)
- */
- public int pos() {
- return pos;
- }
-
- /** Return the last character position of the current token.
- */
- public int endPos() {
- return endPos;
- }
-
- /** Return the last character position of the previous token.
- */
- public int prevEndPos() {
- return prevEndPos;
+ public Token prevToken() {
+ return prevToken;
}
- /** Return the position where a lexical error occurred;
- */
- public int errPos() {
- return errPos;
- }
-
- /** Set the position where a lexical error occurred;
- */
- public void errPos(int pos) {
- errPos = pos;
- }
-
- /** Return the name of an identifier or token for the current token.
- */
- public Name name() {
- return name;
- }
-
- /** Return the radix of a numeric literal token.
- */
- public int radix() {
- return radix;
- }
-
- /** Has a @deprecated been encountered in last doc comment?
- * This needs to be reset by client with resetDeprecatedFlag.
- */
- public boolean deprecatedFlag() {
- return deprecatedFlag;
- }
-
- public void resetDeprecatedFlag() {
- deprecatedFlag = false;
- }
-
- /**
- * Returns the documentation string of the current token.
- */
- public String docComment() {
- return null;
+ public void nextToken() {
+ prevToken = token;
+ token = tokenizer.readToken();
}
- /**
- * Returns a copy of the input buffer, up to its inputLength.
- * Unicode escape sequences are not translated.
- */
- public char[] getRawCharacters() {
- char[] chars = new char[buflen];
- System.arraycopy(buf, 0, chars, 0, buflen);
- return chars;
- }
-
- /**
- * Returns a copy of a character array subset of the input buffer.
- * The returned array begins at the <code>beginIndex</code> and
- * extends to the character at index <code>endIndex - 1</code>.
- * Thus the length of the substring is <code>endIndex-beginIndex</code>.
- * This behavior is like
- * <code>String.substring(beginIndex, endIndex)</code>.
- * Unicode escape sequences are not translated.
- *
- * @param beginIndex the beginning index, inclusive.
- * @param endIndex the ending index, exclusive.
- * @throws IndexOutOfBounds if either offset is outside of the
- * array bounds
- */
- public char[] getRawCharacters(int beginIndex, int endIndex) {
- int length = endIndex - beginIndex;
- char[] chars = new char[length];
- System.arraycopy(buf, beginIndex, chars, 0, length);
- return chars;
- }
-
- public enum CommentStyle {
- LINE,
- BLOCK,
- JAVADOC,
+ public Token split() {
+ Token[] splitTokens = token.split(tokens);
+ prevToken = splitTokens[0];
+ token = splitTokens[1];
+ return token;
}
- /**
- * Called when a complete comment has been scanned. pos and endPos
- * will mark the comment boundary.
- */
- protected void processComment(CommentStyle style) {
- if (scannerDebug)
- System.out.println("processComment(" + pos
- + "," + endPos + "," + style + ")=|"
- + new String(getRawCharacters(pos, endPos))
- + "|");
+ public LineMap getLineMap() {
+ return tokenizer.getLineMap();
}
- /**
- * Called when a complete whitespace run has been scanned. pos and endPos
- * will mark the whitespace boundary.
- */
- protected void processWhiteSpace() {
- if (scannerDebug)
- System.out.println("processWhitespace(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
+ public int errPos() {
+ return tokenizer.errPos();
}
- /**
- * Called when a line terminator has been processed.
- */
- protected void processLineTerminator() {
- if (scannerDebug)
- System.out.println("processTerminator(" + pos
- + "," + endPos + ")=|" +
- new String(getRawCharacters(pos, endPos))
- + "|");
+ public void errPos(int pos) {
+ tokenizer.errPos(pos);
}
-
- /** Build a map for translating between line numbers and
- * positions in the input.
- *
- * @return a LineMap */
- public Position.LineMap getLineMap() {
- return Position.makeLineMap(buf, buflen, false);
- }
-
}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/ScannerFactory.java Mon Oct 24 13:00:20 2011 +0100
@@ -57,7 +57,7 @@
final Log log;
final Names names;
final Source source;
- final Keywords keywords;
+ final Tokens tokens;
/** Create a new scanner factory. */
protected ScannerFactory(Context context) {
@@ -65,14 +65,14 @@
this.log = Log.instance(context);
this.names = Names.instance(context);
this.source = Source.instance(context);
- this.keywords = Keywords.instance(context);
+ this.tokens = Tokens.instance(context);
}
public Scanner newScanner(CharSequence input, boolean keepDocComments) {
if (input instanceof CharBuffer) {
CharBuffer buf = (CharBuffer) input;
if (keepDocComments)
- return new DocCommentScanner(this, buf);
+ return new Scanner(this, new JavadocTokenizer(this, buf));
else
return new Scanner(this, buf);
} else {
@@ -83,7 +83,7 @@
public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
if (keepDocComments)
- return new DocCommentScanner(this, input, inputLength);
+ return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
else
return new Scanner(this, input, inputLength);
}
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/Token.java Fri Oct 21 14:14:29 2011 -0700
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,198 +0,0 @@
-/*
- * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation. Oracle designates this
- * particular file as subject to the "Classpath" exception as provided
- * by Oracle in the LICENSE file that accompanied this code.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-
-package com.sun.tools.javac.parser;
-
-import java.util.Locale;
-
-import com.sun.tools.javac.api.Formattable;
-import com.sun.tools.javac.api.Messages;
-
-/** An interface that defines codes for Java source tokens
- * returned from lexical analysis.
- *
- * <p><b>This is NOT part of any supported API.
- * If you write code that depends on this, you do so at your own risk.
- * This code and its internal interfaces are subject to change or
- * deletion without notice.</b>
- */
-public enum Token implements Formattable {
- EOF,
- ERROR,
- IDENTIFIER,
- ABSTRACT("abstract"),
- ASSERT("assert"),
- BOOLEAN("boolean"),
- BREAK("break"),
- BYTE("byte"),
- CASE("case"),
- CATCH("catch"),
- CHAR("char"),
- CLASS("class"),
- CONST("const"),
- CONTINUE("continue"),
- DEFAULT("default"),
- DO("do"),
- DOUBLE("double"),
- ELSE("else"),
- ENUM("enum"),
- EXTENDS("extends"),
- FINAL("final"),
- FINALLY("finally"),
- FLOAT("float"),
- FOR("for"),
- GOTO("goto"),
- IF("if"),
- IMPLEMENTS("implements"),
- IMPORT("import"),
- INSTANCEOF("instanceof"),
- INT("int"),
- INTERFACE("interface"),
- LONG("long"),
- NATIVE("native"),
- NEW("new"),
- PACKAGE("package"),
- PRIVATE("private"),
- PROTECTED("protected"),
- PUBLIC("public"),
- RETURN("return"),
- SHORT("short"),
- STATIC("static"),
- STRICTFP("strictfp"),
- SUPER("super"),
- SWITCH("switch"),
- SYNCHRONIZED("synchronized"),
- THIS("this"),
- THROW("throw"),
- THROWS("throws"),
- TRANSIENT("transient"),
- TRY("try"),
- VOID("void"),
- VOLATILE("volatile"),
- WHILE("while"),
- INTLITERAL,
- LONGLITERAL,
- FLOATLITERAL,
- DOUBLELITERAL,
- CHARLITERAL,
- STRINGLITERAL,
- TRUE("true"),
- FALSE("false"),
- NULL("null"),
- LPAREN("("),
- RPAREN(")"),
- LBRACE("{"),
- RBRACE("}"),
- LBRACKET("["),
- RBRACKET("]"),
- SEMI(";"),
- COMMA(","),
- DOT("."),
- ELLIPSIS("..."),
- EQ("="),
- GT(">"),
- LT("<"),
- BANG("!"),
- TILDE("~"),
- QUES("?"),
- COLON(":"),
- EQEQ("=="),
- LTEQ("<="),
- GTEQ(">="),
- BANGEQ("!="),
- AMPAMP("&&"),
- BARBAR("||"),
- PLUSPLUS("++"),
- SUBSUB("--"),
- PLUS("+"),
- SUB("-"),
- STAR("*"),
- SLASH("/"),
- AMP("&"),
- BAR("|"),
- CARET("^"),
- PERCENT("%"),
- LTLT("<<"),
- GTGT(">>"),
- GTGTGT(">>>"),
- PLUSEQ("+="),
- SUBEQ("-="),
- STAREQ("*="),
- SLASHEQ("/="),
- AMPEQ("&="),
- BAREQ("|="),
- CARETEQ("^="),
- PERCENTEQ("%="),
- LTLTEQ("<<="),
- GTGTEQ(">>="),
- GTGTGTEQ(">>>="),
- MONKEYS_AT("@"),
- CUSTOM;
-
- Token() {
- this(null);
- }
- Token(String name) {
- this.name = name;
- }
-
- public final String name;
-
- public String toString() {
- switch (this) {
- case IDENTIFIER:
- return "token.identifier";
- case CHARLITERAL:
- return "token.character";
- case STRINGLITERAL:
- return "token.string";
- case INTLITERAL:
- return "token.integer";
- case LONGLITERAL:
- return "token.long-integer";
- case FLOATLITERAL:
- return "token.float";
- case DOUBLELITERAL:
- return "token.double";
- case ERROR:
- return "token.bad-symbol";
- case EOF:
- return "token.end-of-input";
- case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
- case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
- return "'" + name + "'";
- default:
- return name;
- }
- }
-
- public String getKind() {
- return "Token";
- }
-
- public String toString(Locale locale, Messages messages) {
- return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
- }
-}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/Tokens.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,423 @@
+/*
+ * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.Locale;
+
+import com.sun.tools.javac.api.Formattable;
+import com.sun.tools.javac.api.Messages;
+import com.sun.tools.javac.parser.Tokens.Token.Tag;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Names;
+
+/** A class that defines codes/utilities for Java source tokens
+ * returned from lexical analysis.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public class Tokens {
+
+ private final Names names;
+
+ /**
+ * Keyword array. Maps name indices to Token.
+ */
+ private final TokenKind[] key;
+
+ /** The number of the last entered keyword.
+ */
+ private int maxKey = 0;
+
+ /** The names of all tokens.
+ */
+ private Name[] tokenName = new Name[TokenKind.values().length];
+
+ public static final Context.Key<Tokens> tokensKey =
+ new Context.Key<Tokens>();
+
+ public static Tokens instance(Context context) {
+ Tokens instance = context.get(tokensKey);
+ if (instance == null)
+ instance = new Tokens(context);
+ return instance;
+ }
+
+ protected Tokens(Context context) {
+ context.put(tokensKey, this);
+ names = Names.instance(context);
+
+ for (TokenKind t : TokenKind.values()) {
+ if (t.name != null)
+ enterKeyword(t.name, t);
+ else
+ tokenName[t.ordinal()] = null;
+ }
+
+ key = new TokenKind[maxKey+1];
+ for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
+ for (TokenKind t : TokenKind.values()) {
+ if (t.name != null)
+ key[tokenName[t.ordinal()].getIndex()] = t;
+ }
+ }
+
+ private void enterKeyword(String s, TokenKind token) {
+ Name n = names.fromString(s);
+ tokenName[token.ordinal()] = n;
+ if (n.getIndex() > maxKey) maxKey = n.getIndex();
+ }
+
+ /**
+ * Create a new token given a name; if the name corresponds to a token name,
+ * a new token of the corresponding kind is returned; otherwise, an
+ * identifier token is returned.
+ */
+ TokenKind lookupKind(Name name) {
+ return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
+ }
+
+ TokenKind lookupKind(String name) {
+ return lookupKind(names.fromString(name));
+ }
+
+ /**
+ * This enum defines all tokens used by the javac scanner. A token is
+ * optionally associated with a name.
+ */
+ public enum TokenKind implements Formattable {
+ EOF(),
+ ERROR(),
+ IDENTIFIER(Tag.NAMED),
+ ABSTRACT("abstract"),
+ ASSERT("assert", Tag.NAMED),
+ BOOLEAN("boolean", Tag.NAMED),
+ BREAK("break"),
+ BYTE("byte", Tag.NAMED),
+ CASE("case"),
+ CATCH("catch"),
+ CHAR("char", Tag.NAMED),
+ CLASS("class"),
+ CONST("const"),
+ CONTINUE("continue"),
+ DEFAULT("default"),
+ DO("do"),
+ DOUBLE("double", Tag.NAMED),
+ ELSE("else"),
+ ENUM("enum", Tag.NAMED),
+ EXTENDS("extends"),
+ FINAL("final"),
+ FINALLY("finally"),
+ FLOAT("float", Tag.NAMED),
+ FOR("for"),
+ GOTO("goto"),
+ IF("if"),
+ IMPLEMENTS("implements"),
+ IMPORT("import"),
+ INSTANCEOF("instanceof"),
+ INT("int", Tag.NAMED),
+ INTERFACE("interface"),
+ LONG("long", Tag.NAMED),
+ NATIVE("native"),
+ NEW("new"),
+ PACKAGE("package"),
+ PRIVATE("private"),
+ PROTECTED("protected"),
+ PUBLIC("public"),
+ RETURN("return"),
+ SHORT("short", Tag.NAMED),
+ STATIC("static"),
+ STRICTFP("strictfp"),
+ SUPER("super", Tag.NAMED),
+ SWITCH("switch"),
+ SYNCHRONIZED("synchronized"),
+ THIS("this", Tag.NAMED),
+ THROW("throw"),
+ THROWS("throws"),
+ TRANSIENT("transient"),
+ TRY("try"),
+ VOID("void", Tag.NAMED),
+ VOLATILE("volatile"),
+ WHILE("while"),
+ INTLITERAL(Tag.NUMERIC),
+ LONGLITERAL(Tag.NUMERIC),
+ FLOATLITERAL(Tag.NUMERIC),
+ DOUBLELITERAL(Tag.NUMERIC),
+ CHARLITERAL(Tag.NUMERIC),
+ STRINGLITERAL(Tag.STRING),
+ TRUE("true", Tag.NAMED),
+ FALSE("false", Tag.NAMED),
+ NULL("null", Tag.NAMED),
+ LPAREN("("),
+ RPAREN(")"),
+ LBRACE("{"),
+ RBRACE("}"),
+ LBRACKET("["),
+ RBRACKET("]"),
+ SEMI(";"),
+ COMMA(","),
+ DOT("."),
+ ELLIPSIS("..."),
+ EQ("="),
+ GT(">"),
+ LT("<"),
+ BANG("!"),
+ TILDE("~"),
+ QUES("?"),
+ COLON(":"),
+ EQEQ("=="),
+ LTEQ("<="),
+ GTEQ(">="),
+ BANGEQ("!="),
+ AMPAMP("&&"),
+ BARBAR("||"),
+ PLUSPLUS("++"),
+ SUBSUB("--"),
+ PLUS("+"),
+ SUB("-"),
+ STAR("*"),
+ SLASH("/"),
+ AMP("&"),
+ BAR("|"),
+ CARET("^"),
+ PERCENT("%"),
+ LTLT("<<"),
+ GTGT(">>"),
+ GTGTGT(">>>"),
+ PLUSEQ("+="),
+ SUBEQ("-="),
+ STAREQ("*="),
+ SLASHEQ("/="),
+ AMPEQ("&="),
+ BAREQ("|="),
+ CARETEQ("^="),
+ PERCENTEQ("%="),
+ LTLTEQ("<<="),
+ GTGTEQ(">>="),
+ GTGTGTEQ(">>>="),
+ MONKEYS_AT("@"),
+ CUSTOM;
+
+ public final String name;
+ public final Tag tag;
+
+ TokenKind() {
+ this(null, Tag.DEFAULT);
+ }
+
+ TokenKind(String name) {
+ this(name, Tag.DEFAULT);
+ }
+
+ TokenKind(Tag tag) {
+ this(null, tag);
+ }
+
+ TokenKind(String name, Tag tag) {
+ this.name = name;
+ this.tag = tag;
+ }
+
+ public String toString() {
+ switch (this) {
+ case IDENTIFIER:
+ return "token.identifier";
+ case CHARLITERAL:
+ return "token.character";
+ case STRINGLITERAL:
+ return "token.string";
+ case INTLITERAL:
+ return "token.integer";
+ case LONGLITERAL:
+ return "token.long-integer";
+ case FLOATLITERAL:
+ return "token.float";
+ case DOUBLELITERAL:
+ return "token.double";
+ case ERROR:
+ return "token.bad-symbol";
+ case EOF:
+ return "token.end-of-input";
+ case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
+ case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
+ return "'" + name + "'";
+ default:
+ return name;
+ }
+ }
+
+ public String getKind() {
+ return "Token";
+ }
+
+ public String toString(Locale locale, Messages messages) {
+ return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
+ }
+ }
+
+ /**
+ * This is the class representing a javac token. Each token has several fields
+ * that are set by the javac lexer (i.e. start/end position, string value, etc).
+ */
+ public static class Token {
+
+ /** tags constants **/
+ enum Tag {
+ DEFAULT,
+ NAMED,
+ STRING,
+ NUMERIC;
+ }
+
+ /** The token kind */
+ public final TokenKind kind;
+
+ /** The start position of this token */
+ public final int pos;
+
+ /** The end position of this token */
+ public final int endPos;
+
+ /** Is this token preceeded by a deprecated comment? */
+ public final boolean deprecatedFlag;
+
+ /** Is this token preceeded by a deprecated comment? */
+ public String docComment;
+
+ Token(TokenKind kind, int pos, int endPos,
+ boolean deprecatedFlag) {
+ this.kind = kind;
+ this.pos = pos;
+ this.endPos = endPos;
+ this.deprecatedFlag = deprecatedFlag;
+ checkKind();
+ }
+
+ Token[] split(Tokens tokens) {
+ if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
+ throw new AssertionError("Cant split" + kind);
+ }
+
+ TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
+ TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
+
+ if (t1 == null || t2 == null) {
+ throw new AssertionError("Cant split - bad subtokens");
+ }
+ return new Token[] {
+ new Token(t1, pos, pos + t1.name.length(), deprecatedFlag),
+ new Token(t2, pos + t1.name.length(), endPos, false)
+ };
+ }
+
+ protected void checkKind() {
+ if (kind.tag != Tag.DEFAULT) {
+ throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+ }
+ }
+
+ public Name name() {
+ throw new UnsupportedOperationException();
+ }
+
+ public String stringVal() {
+ throw new UnsupportedOperationException();
+ }
+
+ public int radix() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ final static class NamedToken extends Token {
+ /** The name of this token */
+ public final Name name;
+
+ public NamedToken(TokenKind kind, int pos, int endPos, Name name, boolean deprecatedFlag) {
+ super(kind, pos, endPos, deprecatedFlag);
+ this.name = name;
+ }
+
+ protected void checkKind() {
+ if (kind.tag != Tag.NAMED) {
+ throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
+ }
+ }
+
+ @Override
+ public Name name() {
+ return name;
+ }
+ }
+
+ static class StringToken extends Token {
+ /** The string value of this token */
+ public final String stringVal;
+
+ public StringToken(TokenKind kind, int pos, int endPos, String stringVal, boolean deprecatedFlag) {
+ super(kind, pos, endPos, deprecatedFlag);
+ this.stringVal = stringVal;
+ }
+
+ protected void checkKind() {
+ if (kind.tag != Tag.STRING) {
+ throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+ }
+ }
+
+ @Override
+ public String stringVal() {
+ return stringVal;
+ }
+ }
+
+ final static class NumericToken extends StringToken {
+ /** The 'radix' value of this token */
+ public final int radix;
+
+ public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, boolean deprecatedFlag) {
+ super(kind, pos, endPos, stringVal, deprecatedFlag);
+ this.radix = radix;
+ }
+
+ protected void checkKind() {
+ if (kind.tag != Tag.NUMERIC) {
+ throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
+ }
+ }
+
+ @Override
+ public int radix() {
+ return radix;
+ }
+ }
+
+ public static final Token DUMMY =
+ new Token(TokenKind.ERROR, 0, 0, false);
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.file.JavacFileManager;
+import java.nio.CharBuffer;
+import com.sun.tools.javac.util.Log;
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
+ * characters contained in the input stream, handling unicode escape accordingly.
+ * Additionally, it provide features for saving chars into a buffer and to retrieve
+ * them at a later stage.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public class UnicodeReader {
+
+ /** The input buffer, index of next character to be read,
+ * index of one past last character in buffer.
+ */
+ protected char[] buf;
+ protected int bp;
+ protected final int buflen;
+
+ /** The current character.
+ */
+ protected char ch;
+
+ /** The buffer index of the last converted unicode character
+ */
+ protected int unicodeConversionBp = -1;
+
+ protected Log log;
+
+ /**
+ * Create a scanner from the input array. This method might
+ * modify the array. To avoid copying the input array, ensure
+ * that {@code inputLength < input.length} or
+ * {@code input[input.length -1]} is a white space character.
+ *
+ * @param fac the factory which created this Scanner
+ * @param input the input, might be modified
+ * @param inputLength the size of the input.
+ * Must be positive and less than or equal to input.length.
+ */
+ protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
+ this(sf, JavacFileManager.toArray(buffer), buffer.limit());
+ }
+
+ protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
+ log = sf.log;
+ if (inputLength == input.length) {
+ if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
+ inputLength--;
+ } else {
+ char[] newInput = new char[inputLength + 1];
+ System.arraycopy(input, 0, newInput, 0, input.length);
+ input = newInput;
+ }
+ }
+ buf = input;
+ buflen = inputLength;
+ buf[buflen] = EOI;
+ bp = -1;
+ scanChar();
+ }
+
+ /** Read next character.
+ */
+ protected void scanChar() {
+ if (bp < buflen) {
+ ch = buf[++bp];
+ if (ch == '\\') {
+ convertUnicode();
+ }
+ }
+ }
+
+ /** Convert unicode escape; bp points to initial '\' character
+ * (Spec 3.3).
+ */
+ protected void convertUnicode() {
+ if (ch == '\\' && unicodeConversionBp != bp) {
+ bp++; ch = buf[bp];
+ if (ch == 'u') {
+ do {
+ bp++; ch = buf[bp];
+ } while (ch == 'u');
+ int limit = bp + 3;
+ if (limit < buflen) {
+ int d = digit(bp, 16);
+ int code = d;
+ while (bp < limit && d >= 0) {
+ bp++; ch = buf[bp];
+ d = digit(bp, 16);
+ code = (code << 4) + d;
+ }
+ if (d >= 0) {
+ ch = (char)code;
+ unicodeConversionBp = bp;
+ return;
+ }
+ }
+ log.error(bp, "illegal.unicode.esc");
+ } else {
+ bp--;
+ ch = '\\';
+ }
+ }
+ }
+
+ /** Are surrogates supported?
+ */
+ final static boolean surrogatesSupported = surrogatesSupported();
+ private static boolean surrogatesSupported() {
+ try {
+ Character.isHighSurrogate('a');
+ return true;
+ } catch (NoSuchMethodError ex) {
+ return false;
+ }
+ }
+
+ /** Scan surrogate pairs. If 'ch' is a high surrogate and
+ * the next character is a low surrogate, then put the low
+ * surrogate in 'ch', and return the high surrogate.
+ * otherwise, just return 0.
+ */
+ protected char scanSurrogates() {
+ if (surrogatesSupported && Character.isHighSurrogate(ch)) {
+ char high = ch;
+
+ scanChar();
+
+ if (Character.isLowSurrogate(ch)) {
+ return high;
+ }
+
+ ch = high;
+ }
+
+ return 0;
+ }
+
+ /** Convert an ASCII digit from its base (8, 10, or 16)
+ * to its value.
+ */
+ protected int digit(int pos, int base) {
+ char c = ch;
+ int result = Character.digit(c, base);
+ if (result >= 0 && c > 0x7f) {
+ log.error(pos + 1, "illegal.nonascii.digit");
+ ch = "0123456789abcdef".charAt(result);
+ }
+ return result;
+ }
+
+ protected boolean isUnicode() {
+ return unicodeConversionBp == bp;
+ }
+
+ protected void skipChar() {
+ bp++;
+ }
+
+ protected char peekChar() {
+ return buf[bp + 1];
+ }
+
+ /**
+ * Returns a copy of the input buffer, up to its inputLength.
+ * Unicode escape sequences are not translated.
+ */
+ public char[] getRawCharacters() {
+ char[] chars = new char[buflen];
+ System.arraycopy(buf, 0, chars, 0, buflen);
+ return chars;
+ }
+
+ /**
+ * Returns a copy of a character array subset of the input buffer.
+ * The returned array begins at the <code>beginIndex</code> and
+ * extends to the character at index <code>endIndex - 1</code>.
+ * Thus the length of the substring is <code>endIndex-beginIndex</code>.
+ * This behavior is like
+ * <code>String.substring(beginIndex, endIndex)</code>.
+ * Unicode escape sequences are not translated.
+ *
+ * @param beginIndex the beginning index, inclusive.
+ * @param endIndex the ending index, exclusive.
+ * @throws IndexOutOfBounds if either offset is outside of the
+ * array bounds
+ */
+ public char[] getRawCharacters(int beginIndex, int endIndex) {
+ int length = endIndex - beginIndex;
+ char[] chars = new char[length];
+ System.arraycopy(buf, beginIndex, chars, 0, length);
+ return chars;
+ }
+}
--- a/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javac/processing/JavacProcessingEnvironment.java Mon Oct 24 13:00:20 2011 +0100
@@ -1072,9 +1072,9 @@
Assert.checkNonNull(names);
next.put(Names.namesKey, names);
- Keywords keywords = Keywords.instance(context);
- Assert.checkNonNull(keywords);
- next.put(Keywords.keywordsKey, keywords);
+ Tokens tokens = Tokens.instance(context);
+ Assert.checkNonNull(tokens);
+ next.put(Tokens.tokensKey, tokens);
JavaCompiler oldCompiler = JavaCompiler.instance(context);
JavaCompiler nextCompiler = JavaCompiler.instance(next);
--- a/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/src/share/classes/com/sun/tools/javadoc/JavadocTool.java Mon Oct 24 13:00:20 2011 +0100
@@ -39,7 +39,6 @@
import com.sun.tools.javac.code.Symbol.CompletionFailure;
import com.sun.tools.javac.comp.Annotate;
-import com.sun.tools.javac.parser.DocCommentScanner;
import com.sun.tools.javac.tree.JCTree;
import com.sun.tools.javac.tree.JCTree.JCClassDecl;
import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
--- a/langtools/test/tools/javac/api/TestJavacTaskScanner.java Fri Oct 21 14:14:29 2011 -0700
+++ b/langtools/test/tools/javac/api/TestJavacTaskScanner.java Mon Oct 24 13:00:20 2011 +0100
@@ -32,6 +32,7 @@
import com.sun.tools.javac.api.JavacTaskImpl;
import com.sun.tools.javac.parser.*;
+import com.sun.tools.javac.parser.Tokens.Token;
import com.sun.tools.javac.util.*;
import java.io.*;
import java.net.*;
@@ -93,7 +94,7 @@
check(numTokens, "#Tokens", 1222);
check(numParseTypeElements, "#parseTypeElements", 136);
- check(numAllMembers, "#allMembers", 67);
+ check(numAllMembers, "#allMembers", 52);
}
void check(int value, String name, int expected) {
@@ -206,7 +207,8 @@
public void nextToken() {
super.nextToken();
- System.err.format("Saw token %s (%s)%n", token(), name());
+ Token tk = token();
+ System.err.format("Saw token %s %n", tk.kind);
test.numTokens++;
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/test/tools/javac/depDocComment/DeprecatedDocComment3.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ * @compile -Xlint -Werror DeprecatedDocComment3.java
+ */
+
+class DeprecatedDocComment3 {
+ static class Foo { }
+
+ ; /** @deprecated */ ;
+
+ static class A {}
+
+ static class B {
+ A a; //not deprecated!
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/test/tools/javac/tree/DocCommentToplevelTest.java Mon Oct 24 13:00:20 2011 +0100
@@ -0,0 +1,196 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 7096014
+ * @summary Javac tokens should retain state
+ */
+
+import com.sun.source.tree.*;
+import com.sun.source.util.*;
+import com.sun.tools.javac.tree.JCTree;
+
+import java.net.URI;
+import java.util.*;
+import javax.tools.*;
+
+
+public class DocCommentToplevelTest {
+
+ enum PackageKind {
+ HAS_PKG("package pkg;"),
+ NO_PKG("");
+
+ String pkgStr;
+
+ PackageKind(String pkgStr) {
+ this.pkgStr = pkgStr;
+ }
+ }
+
+ enum ImportKind {
+ ZERO(""),
+ ONE("import java.lang.*;"),
+ TWO("import java.lang.*; import java.util.*;");
+
+ String importStr;
+
+ ImportKind(String importStr) {
+ this.importStr = importStr;
+ }
+ }
+
+ enum ModifierKind {
+ DEFAULT(""),
+ PUBLIC("public");
+
+ String modStr;
+
+ ModifierKind(String modStr) {
+ this.modStr = modStr;
+ }
+ }
+
+ enum ToplevelDocKind {
+ HAS_DOC("/** Toplevel! */"),
+ NO_DOC("");
+
+ String docStr;
+
+ ToplevelDocKind(String docStr) {
+ this.docStr = docStr;
+ }
+ }
+
+ static int errors;
+ static int checks;
+
+ public static void main(String... args) throws Exception {
+ //create default shared JavaCompiler - reused across multiple compilations
+ JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
+ StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);
+
+ for (PackageKind pk : PackageKind.values()) {
+ for (ImportKind ik : ImportKind.values()) {
+ for (ModifierKind mk1 : ModifierKind.values()) {
+ for (ModifierKind mk2 : ModifierKind.values()) {
+ for (ToplevelDocKind tdk : ToplevelDocKind.values()) {
+ new DocCommentToplevelTest(pk, ik, mk1, mk2, tdk).run(comp, fm);
+ }
+ }
+ }
+ }
+ }
+
+ if (errors > 0)
+ throw new AssertionError(errors + " errors found");
+
+ System.out.println(checks + " checks were made");
+ }
+
+ PackageKind pk;
+ ImportKind ik;
+ ModifierKind mk1;
+ ModifierKind mk2;
+ ToplevelDocKind tdk;
+ JavaSource source;
+
+ DocCommentToplevelTest(PackageKind pk, ImportKind ik, ModifierKind mk1, ModifierKind mk2, ToplevelDocKind tdk) {
+ this.pk = pk;
+ this.ik = ik;
+ this.mk1 = mk1;
+ this.mk2 = mk2;
+ this.tdk = tdk;
+ source = new JavaSource();
+ }
+
+ void run(JavaCompiler comp, JavaFileManager fm) throws Exception {
+ JavacTask task = (JavacTask)comp.getTask(null, fm, null, Arrays.asList("-printsource"), null, Arrays.asList(source));
+ for (CompilationUnitTree cu: task.parse()) {
+ check(cu);
+ }
+ }
+
+ void check(CompilationUnitTree cu) {
+ checks++;
+
+ new TreeScanner<ClassTree,Void>() {
+
+ Map<JCTree, String> docComments;
+
+ @Override
+ public ClassTree visitCompilationUnit(CompilationUnitTree node, Void unused) {
+ docComments = ((JCTree.JCCompilationUnit)node).docComments;
+ boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
+ (pk != PackageKind.NO_PKG || ik != ImportKind.ZERO);
+ boolean foundComment = docComments.get(node) != null;
+ if (expectedComment != foundComment) {
+ error("Unexpected comment " + docComments.get(node) + " on toplevel");
+ }
+ return super.visitCompilationUnit(node, null);
+ }
+
+ @Override
+ public ClassTree visitClass(ClassTree node, Void unused) {
+ boolean expectedComment = tdk == ToplevelDocKind.HAS_DOC &&
+ pk == PackageKind.NO_PKG && ik == ImportKind.ZERO &&
+ node.getSimpleName().toString().equals("First");
+ boolean foundComment = docComments.get(node) != null;
+ if (expectedComment != foundComment) {
+ error("Unexpected comment " + docComments.get(node) + " on class " + node.getSimpleName());
+ }
+ return super.visitClass(node, unused);
+ }
+ }.scan(cu, null);
+ }
+
+ void error(String msg) {
+ System.err.println("Error: " + msg);
+ System.err.println("Source: " + source.source);
+ errors++;
+ }
+
+ class JavaSource extends SimpleJavaFileObject {
+
+ String template = "#D\n#P\n#I\n" +
+ "#M1 class First { }\n" +
+ "#M2 class Second { }\n";
+
+ String source;
+
+ public JavaSource() {
+ super(URI.create("myfo:/Test.java"), JavaFileObject.Kind.SOURCE);
+ source = template.replace("#P", pk.pkgStr)
+ .replace("#I", ik.importStr)
+ .replace("#M1", mk1.modStr)
+ .replace("#M2", mk2.modStr)
+ .replace("#D", tdk.docStr);
+ }
+
+ @Override
+ public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+ return source;
+ }
+ }
+}