8021130: Comments need to be tokens
Reviewed-by: lagergren, attila
Contributed-by: james.laskey@oracle.com
--- a/nashorn/src/jdk/nashorn/internal/parser/AbstractParser.java Tue Jul 23 18:17:25 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/parser/AbstractParser.java Tue Jul 23 12:00:29 2013 -0300
@@ -25,6 +25,7 @@
package jdk.nashorn.internal.parser;
+import static jdk.nashorn.internal.parser.TokenType.COMMENT;
import static jdk.nashorn.internal.parser.TokenType.EOF;
import static jdk.nashorn.internal.parser.TokenType.EOL;
import static jdk.nashorn.internal.parser.TokenType.IDENT;
@@ -135,14 +136,27 @@
}
/**
- * Seek next token that is not an EOL.
+ * Seek next token that is not an EOL or comment.
*
* @return tokenType of next token.
*/
protected final TokenType next() {
do {
nextOrEOL();
- } while (type == EOL);
+ } while (type == EOL || type == COMMENT);
+
+ return type;
+ }
+
+ /**
+ * Seek next token or EOL (skipping comments.)
+ *
+ * @return tokenType of next token.
+ */
+ protected final TokenType nextOrEOL() {
+ do {
+ nextToken();
+ } while (type == COMMENT);
return type;
}
@@ -152,7 +166,7 @@
*
* @return tokenType of next token.
*/
- protected final TokenType nextOrEOL() {
+ private final TokenType nextToken() {
// Capture last token tokenType.
last = type;
if (type != EOF) {
--- a/nashorn/src/jdk/nashorn/internal/parser/Lexer.java Tue Jul 23 18:17:25 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/parser/Lexer.java Tue Jul 23 12:00:29 2013 -0300
@@ -26,6 +26,7 @@
package jdk.nashorn.internal.parser;
import static jdk.nashorn.internal.parser.TokenType.ADD;
+import static jdk.nashorn.internal.parser.TokenType.COMMENT;
import static jdk.nashorn.internal.parser.TokenType.DECIMAL;
import static jdk.nashorn.internal.parser.TokenType.EOF;
import static jdk.nashorn.internal.parser.TokenType.EOL;
@@ -426,6 +427,9 @@
* @return True if a comment.
*/
protected boolean skipComments() {
+ // Save the current position.
+ final int start = position;
+
if (ch0 == '/') {
// Is it a // comment.
if (ch1 == '/') {
@@ -436,10 +440,9 @@
skip(1);
}
// Did detect a comment.
+ add(COMMENT, start);
return true;
} else if (ch1 == '*') {
- // Record beginning of comment.
- final int start = position;
// Skip over /*.
skip(2);
// Scan for */.
@@ -461,11 +464,11 @@
}
// Did detect a comment.
+ add(COMMENT, start);
return true;
}
- }
-
- if (scripting && ch0 == '#') {
+ } else if (ch0 == '#') {
+ assert scripting;
// shell style comment
// Skip over #.
skip(1);
@@ -474,6 +477,7 @@
skip(1);
}
// Did detect a comment.
+ add(COMMENT, start);
return true;
}
@@ -562,7 +566,7 @@
*
* @param token the token.
* @param startTokenType the token type.
- * @parasm lir LineInfoReceiver that receives line info for multi-line string literals.
+ * @param lir LineInfoReceiver that receives line info for multi-line string literals.
* @return True if a literal beginning with startToken was found and scanned.
*/
protected boolean scanLiteral(final long token, final TokenType startTokenType, final LineInfoReceiver lir) {
@@ -1460,11 +1464,10 @@
final State restState = saveState();
// keep line number updated
int lastLine = line;
- int lastLinePosition = linePosition;
skipLine(false);
lastLine++;
- lastLinePosition = position;
+ int lastLinePosition = position;
restState.setLimit(position);
// Record beginning of string.
--- a/nashorn/src/jdk/nashorn/internal/parser/TokenType.java Tue Jul 23 18:17:25 2013 +0530
+++ b/nashorn/src/jdk/nashorn/internal/parser/TokenType.java Tue Jul 23 12:00:29 2013 -0300
@@ -44,6 +44,7 @@
ERROR (SPECIAL, null),
EOF (SPECIAL, null),
EOL (SPECIAL, null),
+ COMMENT (SPECIAL, null),
NOT (UNARY, "!", 14, false),
NE (BINARY, "!=", 9, true),