8057931: Instead of not skipping small functions in parser, make lexer avoid them instead
authorattila
Tue, 09 Sep 2014 15:34:14 +0200
changeset 26505 d29bf8787b43
parent 26504 ed05e2f4c2db
child 26506 7d55cc7fc301
8057931: Instead of not skipping small functions in parser, make lexer avoid them instead Reviewed-by: hannesw, sundar
nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java
nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java
nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java	Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/AbstractParser.java	Tue Sep 09 15:34:14 2014 +0200
@@ -326,18 +326,28 @@
     }
 
     /**
-     * Check next token and advance.
+     * Check current token and advance to the next token.
      *
      * @param expected Expected tokenType.
      *
      * @throws ParserException on unexpected token type
      */
     protected final void expect(final TokenType expected) throws ParserException {
+        expectDontAdvance(expected);
+        next();
+    }
+
+    /**
+     * Check current token, but don't advance to the next token.
+     *
+     * @param expected Expected tokenType.
+     *
+     * @throws ParserException on unexpected token type
+     */
+    protected final void expectDontAdvance(final TokenType expected) throws ParserException {
         if (type != expected) {
             throw error(expectMessage(expected));
         }
-
-        next();
     }
 
     /**
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java	Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Lexer.java	Tue Sep 09 15:34:14 2014 +0200
@@ -35,6 +35,7 @@
 import static jdk.nashorn.internal.parser.TokenType.ESCSTRING;
 import static jdk.nashorn.internal.parser.TokenType.EXECSTRING;
 import static jdk.nashorn.internal.parser.TokenType.FLOATING;
+import static jdk.nashorn.internal.parser.TokenType.FUNCTION;
 import static jdk.nashorn.internal.parser.TokenType.HEXADECIMAL;
 import static jdk.nashorn.internal.parser.TokenType.LBRACE;
 import static jdk.nashorn.internal.parser.TokenType.LPAREN;
@@ -85,6 +86,9 @@
     /** Type of last token added. */
     private TokenType last;
 
+    private final boolean pauseOnFunctionBody;
+    private boolean pauseOnNextLeftBrace;
+
     private static final String SPACETAB = " \t";  // ASCII space and tab
     private static final String LFCR     = "\n\r"; // line feed and carriage return (ctrl-m)
 
@@ -182,20 +186,23 @@
      * @param scripting are we in scripting mode
      */
     public Lexer(final Source source, final TokenStream stream, final boolean scripting) {
-        this(source, 0, source.getLength(), stream, scripting);
+        this(source, 0, source.getLength(), stream, scripting, false);
     }
 
     /**
-     * Contructor
+     * Constructor
      *
      * @param source    the source
      * @param start     start position in source from which to start lexing
      * @param len       length of source segment to lex
      * @param stream    token stream to lex
      * @param scripting are we in scripting mode
+     * @param pauseOnFunctionBody if true, lexer will return from {@link #lexify()} when it encounters a
+     * function body. This is used with the feature where the parser is skipping nested function bodies to
+     * avoid reading ahead unnecessarily when we skip the function bodies.
      */
 
-    public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting) {
+    public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting, final boolean pauseOnFunctionBody) {
         super(source.getContent(), 1, start, len);
         this.source      = source;
         this.stream      = stream;
@@ -203,6 +210,8 @@
         this.nested      = false;
         this.pendingLine = 1;
         this.last        = EOL;
+
+        this.pauseOnFunctionBody = pauseOnFunctionBody;
     }
 
     private Lexer(final Lexer lexer, final State state) {
@@ -216,6 +225,7 @@
         pendingLine = state.pendingLine;
         linePosition = state.linePosition;
         last = EOL;
+        pauseOnFunctionBody = false;
     }
 
     static class State extends Scanner.State {
@@ -810,6 +820,9 @@
         final int length = scanIdentifier();
         // Check to see if it is a keyword.
         final TokenType type = TokenLookup.lookupKeyword(content, start, length);
+        if (type == FUNCTION && pauseOnFunctionBody) {
+            pauseOnNextLeftBrace = true;
+        }
         // Add keyword or identifier token.
         add(type, start);
     }
@@ -1597,6 +1610,9 @@
                 // We break to let the parser decide what it is.
                 if (canStartLiteral(type)) {
                     break;
+                } else if (type == LBRACE && pauseOnNextLeftBrace) {
+                    pauseOnNextLeftBrace = false;
+                    break;
                 }
             } else if (Character.isJavaIdentifierStart(ch0) || ch0 == '\\' && ch1 == 'u') {
                 // Scan and add identifier or keyword.
--- a/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java	Tue Sep 09 15:33:58 2014 +0200
+++ b/nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Parser.java	Tue Sep 09 15:34:14 2014 +0200
@@ -272,7 +272,7 @@
 
         try {
             stream = new TokenStream();
-            lexer  = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions);
+            lexer  = new Lexer(source, startPos, len, stream, scripting && !env._no_syntax_extensions, reparsedFunction != null);
             lexer.line = lexer.pendingLine = lineOffset + 1;
             line = lineOffset;
 
@@ -2869,9 +2869,9 @@
                 }
                 functionNode.setFinish(lastFinish);
             } else {
-                expect(LBRACE);
-                final int lastLexed = stream.last();
+                expectDontAdvance(LBRACE);
                 if (parseBody || !skipFunctionBody(functionNode)) {
+                    next();
                     // Gather the function elements.
                     final List<Statement> prevFunctionDecls = functionDeclarations;
                     functionDeclarations = new ArrayList<>();
@@ -2883,13 +2883,7 @@
                     }
 
                     lastToken = token;
-                    // Avoiding storing parser state if the function body was small (that is, the next token
-                    // to be read from the token stream is before the last token lexed before we entered
-                    // function body). That'll force the function to be reparsed instead of skipped. Skipping
-                    // involves throwing away and recreating the lexer and the token stream, so for small
-                    // functions it is likely more economical to not bother with skipping (both in terms of
-                    // storing the state, and in terms of throwing away lexer and token stream).
-                    if (parseBody && lastLexed < stream.first()) {
+                    if (parseBody) {
                         // Since the lexer can read ahead and lexify some number of tokens in advance and have
                         // them buffered in the TokenStream, we need to produce a lexer state as it was just
                         // before it lexified RBRACE, and not whatever is its current (quite possibly well read
@@ -2964,10 +2958,7 @@
             return false;
         }
         final ParserState parserState = (ParserState)data.getEndParserState();
-        if (parserState == null) {
-            // The function has no stored parser state; it was deemed too small to be skipped.
-            return false;
-        }
+        assert parserState != null;
 
         stream.reset();
         lexer = parserState.createLexer(source, lexer, stream, scripting && !env._no_syntax_extensions);
@@ -2998,7 +2989,7 @@
         }
 
         Lexer createLexer(final Source source, final Lexer lexer, final TokenStream stream, final boolean scripting) {
-            final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting);
+            final Lexer newLexer = new Lexer(source, position, lexer.limit - position, stream, scripting, true);
             newLexer.restoreState(new Lexer.State(position, Integer.MAX_VALUE, line, -1, linePosition, SEMICOLON));
             return newLexer;
         }