8057931: Instead of not skipping small functions in parser, make lexer avoid them instead
Reviewed-by: hannesw, sundar
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java Tue Sep 09 15:34:14 2014 +0200
@@ -326,18 +326,28 @@
}
/**
- * Check next token and advance.
+ * Check current token and advance to the next token.
*
* @param expected Expected tokenType.
*
* @throws ParserException on unexpected token type
*/
protected final void expect(final TokenType expected) throws ParserException {
+ expectDontAdvance(expected);
+ next();
+ }
+
+ /**
+ * Check current token, but don't advance to the next token.
+ *
+ * @param expected Expected tokenType.
+ *
+ * @throws ParserException on unexpected token type
+ */
+ protected final void expectDontAdvance(final TokenType expected) throws ParserException {
if (type != expected) {
throw error(expectMessage(expected));
}
-
- next();
}
/**
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Tue Sep 09 15:34:14 2014 +0200
@@ -35,6 +35,7 @@
import static jdk.nashorn.internal.parser.TokenType.ESCSTRING;
import static jdk.nashorn.internal.parser.TokenType.EXECSTRING;
import static jdk.nashorn.internal.parser.TokenType.FLOATING;
+import static jdk.nashorn.internal.parser.TokenType.FUNCTION;
import static jdk.nashorn.internal.parser.TokenType.HEXADECIMAL;
import static jdk.nashorn.internal.parser.TokenType.LBRACE;
import static jdk.nashorn.internal.parser.TokenType.LPAREN;
@@ -85,6 +86,9 @@
/** Type of last token added. */
private TokenType last;
+ private final boolean pauseOnFunctionBody;
+ private boolean pauseOnNextLeftBrace;
+
private static final String SPACETAB = " \t"; // ASCII space and tab
private static final String LFCR = "\n\r"; // line feed and carriage return (ctrl-m)
@@ -182,20 +186,23 @@
* @param scripting are we in scripting mode
*/
public Lexer(final Source source, final TokenStream stream, final boolean scripting) {
- this(source, 0, source.getLength(), stream, scripting);
+ this(source, 0, source.getLength(), stream, scripting, false);
}
/**
- * Contructor
+ * Constructor
*
* @param source the source
* @param start start position in source from which to start lexing
* @param len length of source segment to lex
* @param stream token stream to lex
* @param scripting are we in scripting mode
+ * @param pauseOnFunctionBody if true, lexer will return from {@link #lexify()} when it encounters a
+ * function body. This is used with the feature where the parser is skipping nested function bodies to
+ * avoid reading ahead unnecessarily when we skip the function bodies.
*/
- public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting) {
+ public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting, final boolean pauseOnFunctionBody) {
super(source.getContent(), 1, start, len);
this.source = source;
this.stream = stream;
@@ -203,6 +210,8 @@
this.nested = false;
this.pendingLine = 1;
this.last = EOL;
+
+ this.pauseOnFunctionBody = pauseOnFunctionBody;
}
private Lexer(final Lexer lexer, final State state) {
@@ -216,6 +225,7 @@
pendingLine = state.pendingLine;
linePosition = state.linePosition;
last = EOL;
+ pauseOnFunctionBody = false;
}
static class State extends Scanner.State {
@@ -810,6 +820,9 @@
final int length = scanIdentifier();
// Check to see if it is a keyword.
final TokenType type = TokenLookup.lookupKeyword(content, start, length);
+ if (type == FUNCTION && pauseOnFunctionBody) {
+ pauseOnNextLeftBrace = true;
+ }
// Add keyword or identifier token.
add(type, start);
}
@@ -1597,6 +1610,9 @@
// We break to let the parser decide what it is.
if (canStartLiteral(type)) {
break;
+ } else if (type == LBRACE && pauseOnNextLeftBrace) {
+ pauseOnNextLeftBrace = false;
+ break;
}
} else if (Character.isJavaIdentifierStart(ch0) || ch0 == '\\' && ch1 == 'u') {
// Scan and add identifier or keyword.
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Tue Sep 09 15:34:14 2014 +0200
@@ -272,7 +272,7 @@
try {
stream = new TokenStream();
- lexer = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions);
+ lexer = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions, reparsedFunction != null);
lexer.line = lexer.pendingLine = lineOffset + 1;
line = lineOffset;
@@ -2869,9 +2869,9 @@
}
functionNode.setFinish(lastFinish);
} else {
- expect(LBRACE);
- final int lastLexed = stream.last();
+ expectDontAdvance(LBRACE);
if (parseBody || !skipFunctionBody(functionNode)) {
+ next();
// Gather the function elements.
final List<Statement> prevFunctionDecls = functionDeclarations;
functionDeclarations = new ArrayList<>();
@@ -2883,13 +2883,7 @@
}
lastToken = token;
- // Avoiding storing parser state if the function body was small (that is, the next token
- // to be read from the token stream is before the last token lexed before we entered
- // function body). That'll force the function to be reparsed instead of skipped. Skipping
- // involves throwing away and recreating the lexer and the token stream, so for small
- // functions it is likely more economical to not bother with skipping (both in terms of
- // storing the state, and in terms of throwing away lexer and token stream).
- if (parseBody && lastLexed < stream.first()) {
+ if (parseBody) {
// Since the lexer can read ahead and lexify some number of tokens in advance and have
// them buffered in the TokenStream, we need to produce a lexer state as it was just
// before it lexified RBRACE, and not whatever is its current (quite possibly well read
@@ -2964,10 +2958,7 @@
return false;
}
final ParserState parserState = (ParserState)data.getEndParserState();
- if (parserState == null) {
- // The function has no stored parser state; it was deemed too small to be skipped.
- return false;
- }
+ assert parserState != null;
stream.reset();
lexer = parserState.createLexer(source, lexer, stream, scripting && !env._no_syntax_extensions);
@@ -2998,7 +2989,7 @@
}
Lexer createLexer(final Source source, final Lexer lexer, final TokenStream stream, final boolean scripting) {
- final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting);
+ final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting, true);
newLexer.restoreState(new Lexer.State(position, Integer.MAX_VALUE, line, -1, linePosition, SEMICOLON));
return newLexer;
}