# HG changeset patch # User attila # Date 1410269654 -7200 # Node ID d29bf8787b4398711afe779dc1a22f7d2ed542bb # Parent ed05e2f4c2dbac2279d301d8a96630e3073e99b1 8057931: Instead of not skipping small functions in parser, make lexer avoid them instead Reviewed-by: hannesw, sundar diff -r ed05e2f4c2db -r d29bf8787b43 nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java Tue Sep 09 15:33:58 2014 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java Tue Sep 09 15:34:14 2014 +0200 @@ -326,18 +326,28 @@ } /** - * Check next token and advance. + * Check current token and advance to the next token. * * @param expected Expected tokenType. * * @throws ParserException on unexpected token type */ protected final void expect(final TokenType expected) throws ParserException { + expectDontAdvance(expected); + next(); + } + + /** + * Check current token, but don't advance to the next token. + * + * @param expected Expected tokenType. + * + * @throws ParserException on unexpected token type + */ + protected final void expectDontAdvance(final TokenType expected) throws ParserException { if (type != expected) { throw error(expectMessage(expected)); } - - next(); } /** diff -r ed05e2f4c2db -r d29bf8787b43 nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Tue Sep 09 15:33:58 2014 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java Tue Sep 09 15:34:14 2014 +0200 @@ -35,6 +35,7 @@ import static jdk.nashorn.internal.parser.TokenType.ESCSTRING; import static jdk.nashorn.internal.parser.TokenType.EXECSTRING; import static jdk.nashorn.internal.parser.TokenType.FLOATING; +import static jdk.nashorn.internal.parser.TokenType.FUNCTION; import static jdk.nashorn.internal.parser.TokenType.HEXADECIMAL; import static jdk.nashorn.internal.parser.TokenType.LBRACE; import static jdk.nashorn.internal.parser.TokenType.LPAREN; @@ -85,6 +86,9 @@ /** Type of last token added. */ private TokenType last; + private final boolean pauseOnFunctionBody; + private boolean pauseOnNextLeftBrace; + private static final String SPACETAB = " \t"; // ASCII space and tab private static final String LFCR = "\n\r"; // line feed and carriage return (ctrl-m) @@ -182,20 +186,23 @@ * @param scripting are we in scripting mode */ public Lexer(final Source source, final TokenStream stream, final boolean scripting) { - this(source, 0, source.getLength(), stream, scripting); + this(source, 0, source.getLength(), stream, scripting, false); } /** - * Contructor + * Constructor * * @param source the source * @param start start position in source from which to start lexing * @param len length of source segment to lex * @param stream token stream to lex * @param scripting are we in scripting mode + * @param pauseOnFunctionBody if true, lexer will return from {@link #lexify()} when it encounters a + * function body. This is used with the feature where the parser is skipping nested function bodies to + * avoid reading ahead unnecessarily when we skip the function bodies. */ - public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting) { + public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting, final boolean pauseOnFunctionBody) { super(source.getContent(), 1, start, len); this.source = source; this.stream = stream; @@ -203,6 +210,8 @@ this.nested = false; this.pendingLine = 1; this.last = EOL; + + this.pauseOnFunctionBody = pauseOnFunctionBody; } private Lexer(final Lexer lexer, final State state) { @@ -216,6 +225,7 @@ pendingLine = state.pendingLine; linePosition = state.linePosition; last = EOL; + pauseOnFunctionBody = false; } static class State extends Scanner.State { @@ -810,6 +820,9 @@ final int length = scanIdentifier(); // Check to see if it is a keyword. final TokenType type = TokenLookup.lookupKeyword(content, start, length); + if (type == FUNCTION && pauseOnFunctionBody) { + pauseOnNextLeftBrace = true; + } // Add keyword or identifier token. add(type, start); } @@ -1597,6 +1610,9 @@ // We break to let the parser decide what it is. if (canStartLiteral(type)) { break; + } else if (type == LBRACE && pauseOnNextLeftBrace) { + pauseOnNextLeftBrace = false; + break; } } else if (Character.isJavaIdentifierStart(ch0) || ch0 == '\\' && ch1 == 'u') { // Scan and add identifier or keyword. diff -r ed05e2f4c2db -r d29bf8787b43 nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java --- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Tue Sep 09 15:33:58 2014 +0200 +++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java Tue Sep 09 15:34:14 2014 +0200 @@ -272,7 +272,7 @@ try { stream = new TokenStream(); - lexer = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions); + lexer = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions, reparsedFunction != null); lexer.line = lexer.pendingLine = lineOffset + 1; line = lineOffset; @@ -2869,9 +2869,9 @@ } functionNode.setFinish(lastFinish); } else { - expect(LBRACE); - final int lastLexed = stream.last(); + expectDontAdvance(LBRACE); if (parseBody || !skipFunctionBody(functionNode)) { + next(); // Gather the function elements. final List prevFunctionDecls = functionDeclarations; functionDeclarations = new ArrayList<>(); @@ -2883,13 +2883,7 @@ } lastToken = token; - // Avoiding storing parser state if the function body was small (that is, the next token - // to be read from the token stream is before the last token lexed before we entered - // function body). That'll force the function to be reparsed instead of skipped. Skipping - // involves throwing away and recreating the lexer and the token stream, so for small - // functions it is likely more economical to not bother with skipping (both in terms of - // storing the state, and in terms of throwing away lexer and token stream). - if (parseBody && lastLexed < stream.first()) { + if (parseBody) { // Since the lexer can read ahead and lexify some number of tokens in advance and have // them buffered in the TokenStream, we need to produce a lexer state as it was just // before it lexified RBRACE, and not whatever is its current (quite possibly well read @@ -2964,10 +2958,7 @@ return false; } final ParserState parserState = (ParserState)data.getEndParserState(); - if (parserState == null) { - // The function has no stored parser state; it was deemed too small to be skipped. - return false; - } + assert parserState != null; stream.reset(); lexer = parserState.createLexer(source, lexer, stream, scripting && !env._no_syntax_extensions); @@ -2998,7 +2989,7 @@ } Lexer createLexer(final Source source, final Lexer lexer, final TokenStream stream, final boolean scripting) { - final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting); + final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting, true); newLexer.restoreState(new Lexer.State(position, Integer.MAX_VALUE, line, -1, linePosition, SEMICOLON)); return newLexer; }