/*

 * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.  Oracle designates this

 * particular file as subject to the "Classpath" exception as provided

 * by Oracle in the LICENSE file that accompanied this code.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 */

package jdk.nashorn.internal.parser;

import static jdk.nashorn.internal.parser.TokenType.ADD;

import static jdk.nashorn.internal.parser.TokenType.BINARY_NUMBER;

import static jdk.nashorn.internal.parser.TokenType.COMMENT;

import static jdk.nashorn.internal.parser.TokenType.DECIMAL;

import static jdk.nashorn.internal.parser.TokenType.DIRECTIVE_COMMENT;

import static jdk.nashorn.internal.parser.TokenType.EOF;

import static jdk.nashorn.internal.parser.TokenType.EOL;

import static jdk.nashorn.internal.parser.TokenType.ERROR;

import static jdk.nashorn.internal.parser.TokenType.ESCSTRING;

import static jdk.nashorn.internal.parser.TokenType.EXECSTRING;

import static jdk.nashorn.internal.parser.TokenType.FLOATING;

import static jdk.nashorn.internal.parser.TokenType.FUNCTION;

import static jdk.nashorn.internal.parser.TokenType.HEXADECIMAL;

import static jdk.nashorn.internal.parser.TokenType.LBRACE;

import static jdk.nashorn.internal.parser.TokenType.LPAREN;

import static jdk.nashorn.internal.parser.TokenType.OCTAL;

import static jdk.nashorn.internal.parser.TokenType.OCTAL_LEGACY;

import static jdk.nashorn.internal.parser.TokenType.RBRACE;

import static jdk.nashorn.internal.parser.TokenType.REGEX;

import static jdk.nashorn.internal.parser.TokenType.RPAREN;

import static jdk.nashorn.internal.parser.TokenType.STRING;

import static jdk.nashorn.internal.parser.TokenType.TEMPLATE;

import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_HEAD;

import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_MIDDLE;

import static jdk.nashorn.internal.parser.TokenType.TEMPLATE_TAIL;

import static jdk.nashorn.internal.parser.TokenType.XML;

import java.io.Serializable;

import jdk.nashorn.internal.runtime.ECMAErrors;

import jdk.nashorn.internal.runtime.ErrorManager;

import jdk.nashorn.internal.runtime.JSErrorType;

import jdk.nashorn.internal.runtime.JSType;

import jdk.nashorn.internal.runtime.ParserException;

import jdk.nashorn.internal.runtime.Source;

import jdk.nashorn.internal.runtime.options.Options;

/**

 * Responsible for converting source content into a stream of tokens.

 *

 */

@SuppressWarnings("fallthrough")

public class Lexer extends Scanner {

    private static final long MIN_INT_L = Integer.MIN_VALUE;

    private static final long MAX_INT_L = Integer.MAX_VALUE;

    private static final boolean XML_LITERALS = Options.getBooleanProperty("nashorn.lexer.xmlliterals");

    /** Content source. */

    private final Source source;

    /** Buffered stream for tokens. */

    private final TokenStream stream;

    /** True if here and edit strings are supported. */

    private final boolean scripting;

    /** True if parsing in ECMAScript 6 mode. */

    private final boolean es6;

    /** True if a nested scan. (scan to completion, no EOF.) */

    private final boolean nested;

    /** Pending new line number and position. */

    int pendingLine;

    /** Position of last EOL + 1. */

    private int linePosition;

    /** Type of last token added. */

    private TokenType last;

    private final boolean pauseOnFunctionBody;

    private boolean pauseOnNextLeftBrace;

    private int templateExpressionOpenBraces;

    private static final String JAVASCRIPT_OTHER_WHITESPACE =

        "\u2028" + // line separator

        "\u2029" + // paragraph separator

        "\u00a0" + // Latin-1 space

        "\u1680" + // Ogham space mark

        "\u180e" + // separator, Mongolian vowel

        "\u2000" + // en quad

        "\u2001" + // em quad

        "\u2002" + // en space

        "\u2003" + // em space

        "\u2004" + // three-per-em space

        "\u2005" + // four-per-em space

        "\u2006" + // six-per-em space

        "\u2007" + // figure space

        "\u2008" + // punctuation space

        "\u2009" + // thin space

        "\u200a" + // hair space

        "\u202f" + // narrow no-break space

        "\u205f" + // medium mathematical space

        "\u3000" + // ideographic space

        "\ufeff"   // byte order mark

        ;

    private static final String JAVASCRIPT_WHITESPACE_IN_REGEXP =

        "\\u000a" + // line feed

        "\\u000d" + // carriage return (ctrl-m)

        "\\u2028" + // line separator

        "\\u2029" + // paragraph separator

        "\\u0009" + // tab

        "\\u0020" + // ASCII space

        "\\u000b" + // tabulation line

        "\\u000c" + // ff (ctrl-l)

        "\\u00a0" + // Latin-1 space

        "\\u1680" + // Ogham space mark

        "\\u180e" + // separator, Mongolian vowel

        "\\u2000" + // en quad

        "\\u2001" + // em quad

        "\\u2002" + // en space

        "\\u2003" + // em space

        "\\u2004" + // three-per-em space

        "\\u2005" + // four-per-em space

        "\\u2006" + // six-per-em space

        "\\u2007" + // figure space

        "\\u2008" + // punctuation space

        "\\u2009" + // thin space

        "\\u200a" + // hair space

        "\\u202f" + // narrow no-break space

        "\\u205f" + // medium mathematical space

        "\\u3000" + // ideographic space

        "\\ufeff"   // byte order mark

        ;

    static String unicodeEscape(final char ch) {

        final StringBuilder sb = new StringBuilder();

        sb.append("\\u");

        final String hex = Integer.toHexString(ch);

        for (int i = hex.length(); i < 4; i++) {

            sb.append('0');

        }

        sb.append(hex);

        return sb.toString();

    }

    /**

     * Constructor

     *

     * @param source    the source

     * @param stream    the token stream to lex

     */

    public Lexer(final Source source, final TokenStream stream) {

        this(source, stream, false, false);

    }

    /**

     * Constructor

     *

     * @param source    the source

     * @param stream    the token stream to lex

     * @param scripting are we in scripting mode

     * @param es6       are we in ECMAScript 6 mode

     */

    public Lexer(final Source source, final TokenStream stream, final boolean scripting, final boolean es6) {

        this(source, 0, source.getLength(), stream, scripting, es6, false);

    }

    /**

     * Constructor

     *

     * @param source    the source

     * @param start     start position in source from which to start lexing

     * @param len       length of source segment to lex

     * @param stream    token stream to lex

     * @param scripting are we in scripting mode

     * @param es6       are we in ECMAScript 6 mode

     * @param pauseOnFunctionBody if true, lexer will return from {@link #lexify()} when it encounters a

     * function body. This is used with the feature where the parser is skipping nested function bodies to

     * avoid reading ahead unnecessarily when we skip the function bodies.

     */

    public Lexer(final Source source, final int start, final int len, final TokenStream stream, final boolean scripting, final boolean es6, final boolean pauseOnFunctionBody) {

        super(source.getContent(), 1, start, len);

        this.source      = source;

        this.stream      = stream;

        this.scripting   = scripting;

        this.es6         = es6;

        this.nested      = false;

        this.pendingLine = 1;

        this.last        = EOL;

        this.pauseOnFunctionBody = pauseOnFunctionBody;

    }

    private Lexer(final Lexer lexer, final State state) {

        super(lexer, state);

        source = lexer.source;

        stream = lexer.stream;

        scripting = lexer.scripting;

        es6 = lexer.es6;

        nested = true;

        pendingLine = state.pendingLine;

        linePosition = state.linePosition;

        last = EOL;

        pauseOnFunctionBody = false;

    }

    static class State extends Scanner.State {

        /** Pending new line number and position. */

        public final int pendingLine;

        /** Position of last EOL + 1. */

        public final int linePosition;

        /** Type of last token added. */

        public final TokenType last;

        /*

         * Constructor.

         */

        State(final int position, final int limit, final int line, final int pendingLine, final int linePosition, final TokenType last) {

            super(position, limit, line);

            this.pendingLine = pendingLine;

            this.linePosition = linePosition;

            this.last = last;

        }

    }

    /**

     * Save the state of the scan.

     *

     * @return Captured state.

     */

    @Override

    State saveState() {

        return new State(position, limit, line, pendingLine, linePosition, last);

    }

    /**

     * Restore the state of the scan.

     *

     * @param state

     *            Captured state.

     */

    void restoreState(final State state) {

        super.restoreState(state);

        pendingLine = state.pendingLine;

        linePosition = state.linePosition;

        last = state.last;

    }

    /**

     * Add a new token to the stream.

     *

     * @param type

     *            Token type.

     * @param start

     *            Start position.

     * @param end

     *            End position.

     */

    protected void add(final TokenType type, final int start, final int end) {

        // Record last token.

        last = type;

        // Only emit the last EOL in a cluster.

        if (type == EOL) {

            pendingLine = end;

            linePosition = start;

        } else {

            // Write any pending EOL to stream.

            if (pendingLine != -1) {

                stream.put(Token.toDesc(EOL, linePosition, pendingLine));

                pendingLine = -1;

            }

            // Write token to stream.

            stream.put(Token.toDesc(type, start, end - start));

        }

    }

    /**

     * Add a new token to the stream.

     *

     * @param type

     *            Token type.

     * @param start

     *            Start position.

     */

    protected void add(final TokenType type, final int start) {

        add(type, start, position);

    }

    /**

     * Return the String of valid whitespace characters for regular

     * expressions in JavaScript

     * @return regexp whitespace string

     */

    public static String getWhitespaceRegExp() {

        return JAVASCRIPT_WHITESPACE_IN_REGEXP;

    }

    /**

     * Skip end of line.

     *

     * @param addEOL true if EOL token should be recorded.

     */

    private void skipEOL(final boolean addEOL) {

        if (ch0 == '\r') { // detect \r\n pattern

            skip(1);

            if (ch0 == '\n') {

                skip(1);

            }

        } else { // all other space, ch0 is guaranteed to be EOL or \0

            skip(1);

        }

        // bump up line count

        line++;

        if (addEOL) {

            // Add an EOL token.

            add(EOL, position, line);

        }

    }

    /**

     * Skip over rest of line including end of line.

     *

     * @param addEOL true if EOL token should be recorded.

     */

    private void skipLine(final boolean addEOL) {

        // Ignore characters.

        while (!isEOL(ch0) && !atEOF()) {

            skip(1);

        }

        // Skip over end of line.

        skipEOL(addEOL);

    }

    /**

     * Test whether a char is valid JavaScript whitespace

     * @param ch a char

     * @return true if valid JavaScript whitespace

     */

    public static boolean isJSWhitespace(final char ch) {

        return ch == ' '                  // space

            || ch >= '\t' && ch <= '\r'   // 0x09..0x0d: tab, line feed, tabulation line, ff, carriage return

            || ch >= 160 && isOtherJSWhitespace(ch);

    }

    private static boolean isOtherJSWhitespace(final char ch) {

        return JAVASCRIPT_OTHER_WHITESPACE.indexOf(ch) != -1;

    }

    /**

     * Test whether a char is valid JavaScript end of line

     * @param ch a char

     * @return true if valid JavaScript end of line

     */

    public static boolean isJSEOL(final char ch) {

        return ch == '\n'      // line feed

            || ch == '\r'      // carriage return (ctrl-m)

            || ch == '\u2028'  // line separator

            || ch == '\u2029'; // paragraph separator

    }

    /**

     * Test if char is a string delimiter, e.g. '\' or '"'.

     * @param ch a char

     * @return true if string delimiter

     */

    protected boolean isStringDelimiter(final char ch) {

        return ch == '\'' || ch == '"';

    }

    /**

     * Test if char is a template literal delimiter ('`').

     */

    private static boolean isTemplateDelimiter(final char ch) {

        return ch == '`';

    }

    /**

     * Test whether a char is valid JavaScript whitespace

     * @param ch a char

     * @return true if valid JavaScript whitespace

     */

    protected boolean isWhitespace(final char ch) {

        return Lexer.isJSWhitespace(ch);

    }

    /**

     * Test whether a char is valid JavaScript end of line

     * @param ch a char

     * @return true if valid JavaScript end of line

     */

    protected boolean isEOL(final char ch) {

        return Lexer.isJSEOL(ch);

    }

    /**

     * Skip over whitespace and detect end of line, adding EOL tokens if

     * encountered.

     *

     * @param addEOL true if EOL tokens should be recorded.

     */

    private void skipWhitespace(final boolean addEOL) {

        while (isWhitespace(ch0)) {

            if (isEOL(ch0)) {

                skipEOL(addEOL);

            } else {

                skip(1);

            }

        }

    }

    /**

     * Skip over comments.

     *

     * @return True if a comment.

     */

    protected boolean skipComments() {

        // Save the current position.

        final int start = position;

        if (ch0 == '/') {

            // Is it a // comment.

            if (ch1 == '/') {

                // Skip over //.

                skip(2);

                boolean directiveComment = false;

                if ((ch0 == '#' || ch0 == '@') && (ch1 == ' ')) {

                    directiveComment = true;

                }

                // Scan for EOL.

                while (!atEOF() && !isEOL(ch0)) {

                    skip(1);

                }

                // Did detect a comment.

                add(directiveComment? DIRECTIVE_COMMENT : COMMENT, start);

                return true;

            } else if (ch1 == '*') {

                // Skip over /*.

                skip(2);

                // Scan for */.

                while (!atEOF() && !(ch0 == '*' && ch1 == '/')) {

                    // If end of line handle else skip character.

                    if (isEOL(ch0)) {

                        skipEOL(true);

                    } else {

                        skip(1);

                    }

                }

                if (atEOF()) {

                    // TODO - Report closing */ missing in parser.

                    add(ERROR, start);

                } else {

                    // Skip */.

                    skip(2);

                }

                // Did detect a comment.

                add(COMMENT, start);

                return true;

            }

        } else if (ch0 == '#') {

            assert scripting;

            // shell style comment

            // Skip over #.

            skip(1);

            // Scan for EOL.

            while (!atEOF() && !isEOL(ch0)) {

                skip(1);

            }

            // Did detect a comment.

            add(COMMENT, start);

            return true;

        }