/*

 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.  Oracle designates this

 * particular file as subject to the "Classpath" exception as provided

 * by Oracle in the LICENSE file that accompanied this code.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 */

package com.sun.tools.javac.parser;

import com.sun.tools.javac.code.Lint;

import com.sun.tools.javac.code.Lint.LintCategory;

import com.sun.tools.javac.code.Preview;

import com.sun.tools.javac.code.Source;

import com.sun.tools.javac.code.Source.Feature;

import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;

import com.sun.tools.javac.resources.CompilerProperties.Errors;

import com.sun.tools.javac.resources.CompilerProperties.Warnings;

import com.sun.tools.javac.util.*;

import com.sun.tools.javac.util.JCDiagnostic.*;

import java.lang.reflect.InvocationTargetException;

import java.lang.reflect.Method;

import java.nio.CharBuffer;

import java.util.HashSet;

import java.util.Set;

import static com.sun.tools.javac.parser.Tokens.*;

import static com.sun.tools.javac.util.LayoutCharacters.*;

/** The lexical analyzer maps an input stream consisting of

 *  ASCII characters and Unicode escapes into a token sequence.

 *

 *  <p><b>This is NOT part of any supported API.

 *  If you write code that depends on this, you do so at your own risk.

 *  This code and its internal interfaces are subject to change or

 *  deletion without notice.</b>

 */

public class JavaTokenizer {

    private static final boolean scannerDebug = false;

    /** The source language setting.

     */

    private Source source;

    /** The preview language setting. */

    private Preview preview;

    /** The log to be used for error reporting.

     */

    private final Log log;

    /** The token factory. */

    private final Tokens tokens;

    /** The token kind, set by nextToken().

     */

    protected TokenKind tk;

    /** The token's radix, set by nextToken().

     */

    protected int radix;

    /** The token's name, set by nextToken().

     */

    protected Name name;

    /** The position where a lexical error occurred;

     */

    protected int errPos = Position.NOPOS;

    /** The Unicode reader (low-level stream reader).

     */

    protected UnicodeReader reader;

    /** Should the string stripped of indentation?

     */

    protected boolean shouldStripIndent;

    /** Should the string's escapes be translated?

     */

    protected boolean shouldTranslateEscapes;

    /** Has the string broken escapes?

     */

    protected boolean hasBrokenEscapes;

    protected ScannerFactory fac;

    // The set of lint options currently in effect. It is initialized

    // from the context, and then is set/reset as needed by Attr as it

    // visits all the various parts of the trees during attribution.

    protected Lint lint;

    private static final boolean hexFloatsWork = hexFloatsWork();

    private static boolean hexFloatsWork() {

        try {

            Float.valueOf("0x1.0p1");

            return true;

        } catch (NumberFormatException ex) {

            return false;

        }

    }

    /**

     * Create a scanner from the input array.  This method might

     * modify the array.  To avoid copying the input array, ensure

     * that {@code inputLength < input.length} or

     * {@code input[input.length -1]} is a white space character.

     *

     * @param fac the factory which created this Scanner

     * @param buf the input, might be modified

     * Must be positive and less than or equal to input.length.

     */

    protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {

        this(fac, new UnicodeReader(fac, buf));

    }

    protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {

        this(fac, new UnicodeReader(fac, buf, inputLength));

    }

    protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {

        this.fac = fac;

        this.log = fac.log;

        this.tokens = fac.tokens;

        this.source = fac.source;

        this.preview = fac.preview;

        this.reader = reader;

        this.lint = fac.lint;

    }

    protected void checkSourceLevel(int pos, Feature feature) {

        if (preview.isPreview(feature) && !preview.isEnabled()) {

            //preview feature without --preview flag, error

            lexError(DiagnosticFlag.SOURCE_LEVEL, pos, preview.disabledError(feature));

        } else if (!feature.allowedInSource(source)) {

            //incompatible source level, error

            lexError(DiagnosticFlag.SOURCE_LEVEL, pos, feature.error(source.name));

        } else if (preview.isPreview(feature)) {

            //use of preview feature, warn

            preview.warnPreview(pos, feature);

        }

    }

    /** Report an error at the given position using the provided arguments.

     */

    protected void lexError(int pos, JCDiagnostic.Error key) {

        log.error(pos, key);

        tk = TokenKind.ERROR;

        errPos = pos;

    }

    protected void lexError(DiagnosticFlag flags, int pos, JCDiagnostic.Error key) {

        log.error(flags, pos, key);

        tk = TokenKind.ERROR;

        errPos = pos;

    }

    protected void lexWarning(LintCategory lc, int pos, JCDiagnostic.Warning key) {

        DiagnosticPosition dp = new SimpleDiagnosticPosition(pos) ;

        log.warning(lc, dp, key);

    }

    /** Read next character in character or string literal and copy into sbuf.

     */

    private void scanLitChar(int pos) {

        if (reader.ch == '\\') {

            if (reader.peekChar() == '\\' && !reader.isUnicode()) {

                reader.skipChar();

                reader.putChar('\\', true);

            } else {

                reader.scanChar();

                switch (reader.ch) {

                case '0': case '1': case '2': case '3':

                case '4': case '5': case '6': case '7':

                    char leadch = reader.ch;

                    int oct = reader.digit(pos, 8);

                    reader.scanChar();

                    if ('0' <= reader.ch && reader.ch <= '7') {

                        oct = oct * 8 + reader.digit(pos, 8);

                        reader.scanChar();

                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {

                            oct = oct * 8 + reader.digit(pos, 8);

                            reader.scanChar();

                        }

                    }

                    reader.putChar((char)oct);

                    break;

                case 'b':

                    reader.putChar('\b', true); break;

                case 't':

                    reader.putChar('\t', true); break;

                case 'n':

                    reader.putChar('\n', true); break;

                case 'f':

                    reader.putChar('\f', true); break;

                case 'r':

                    reader.putChar('\r', true); break;

                case '\'':

                    reader.putChar('\'', true); break;

                case '\"':

                    reader.putChar('\"', true); break;

                case '\\':

                    reader.putChar('\\', true); break;

                default:

                    lexError(reader.bp, Errors.IllegalEscChar);

                }

            }

        } else if (reader.bp != reader.buflen) {

            reader.putChar(true);

        }

    }

    /** Read next character in character or string literal and copy into sbuf

     *  without translating escapes. Used by text blocks to preflight verify

     *  escapes sequences.

     */

    private void scanLitCharRaw(int pos) {

        if (reader.ch == '\\') {

            if (reader.peekChar() == '\\' && !reader.isUnicode()) {

                reader.skipChar();

                reader.putChar('\\', false);

                reader.putChar('\\', true);

            } else {

                reader.putChar('\\', true);

                switch (reader.ch) {

                case '0': case '1': case '2': case '3':

                case '4': case '5': case '6': case '7':

                    char leadch = reader.ch;

                    reader.putChar(true);

                    if ('0' <= reader.ch && reader.ch <= '7') {

                        reader.putChar(true);

                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {

                            reader.putChar(true);

                        }

                    }

                    break;

                // Effectively list of valid escape sequences.

                case 'b':

                case 't':

                case 'n':

                case 'f':

                case 'r':

                case '\'':

                case '\"':

                case '\\':

                    reader.putChar(true); break;

                default:

                    hasBrokenEscapes = true;

                    lexError(reader.bp, Errors.IllegalEscChar);

                }

            }

        } else if (reader.bp != reader.buflen) {

            reader.putChar(true);

        }

    }

    /** Interim access to String methods used to support text blocks.

     *  Required to handle bootstrapping with pre-text block jdks.

     *  Could be reworked in the 'next' jdk.

     */

    static class TextBlockSupport {

        /** Reflection method to remove incidental indentation.

         */

        private static final Method stripIndent;

        /** Reflection method to translate escape sequences.

         */

        private static final Method translateEscapes;

        /** true if stripIndent and translateEscapes are available in the bootstrap jdk.

         */

        private static final boolean hasSupport;

        /** Get a string method via refection or null if not available.

         */

        private static Method getStringMethodOrNull(String name) {

            try {

                return String.class.getMethod(name);

            } catch (Exception ex) {

                // Method not available, return null.

            }

            return null;

        }

        static {

            // Get text block string methods.

            stripIndent = getStringMethodOrNull("stripIndent");

            translateEscapes = getStringMethodOrNull("translateEscapes");

            // true if stripIndent and translateEscapes are available in the bootstrap jdk.

            hasSupport = stripIndent != null && translateEscapes != null;

        }

        /** Return true if stripIndent and translateEscapes are available in the bootstrap jdk.

         */

        static boolean hasSupport() {

            return hasSupport;

        }

        /** Return the leading whitespace count (indentation) of the line.

         */

        private static int indent(String line) {

            return line.length() - line.stripLeading().length();

        }

        enum WhitespaceChecks {

            INCONSISTENT,

            TRAILING

        };

        /** Check that the use of white space in content is not problematic.

         */

        static Set<WhitespaceChecks> checkWhitespace(String string) {

            // Start with empty result set.

            Set<WhitespaceChecks> checks = new HashSet<>();

            // No need to check empty strings.

            if (string.isEmpty()) {

                return checks;

            }

            // Maximum common indentation.

            int outdent = 0;

            // No need to check indentation if opting out (last line is empty.)

            char lastChar = string.charAt(string.length() - 1);

            boolean optOut = lastChar == '\n' || lastChar == '\r';

            // Split string based at line terminators.

            String[] lines = string.split("\\R");

            int length = lines.length;

            // Extract last line.

            String lastLine = lines[length - 1];

            if (!optOut) {

                // Prime with the last line indentation (may be blank.)

                outdent = indent(lastLine);

                for (String line : lines) {

                    // Blanks lines have no influence (last line accounted for.)

                    if (!line.isBlank()) {

                        outdent = Integer.min(outdent, indent(line));

                        if (outdent == 0) {

                            break;

                        }

                    }

                }

            }

            // Last line is representative.

            String start = lastLine.substring(0, outdent);

            for (String line : lines) {

                // Fail if a line does not have the same indentation.

                if (!line.isBlank() && !line.startsWith(start)) {

                    // Mix of different white space

                    checks.add(WhitespaceChecks.INCONSISTENT);

                }

                // Line has content even after indent is removed.

                if (outdent < line.length()) {

                    // Is the last character a white space.

                    lastChar = line.charAt(line.length() - 1);

                    if (Character.isWhitespace(lastChar)) {

                        // Has trailing white space.

                        checks.add(WhitespaceChecks.TRAILING);

                    }

                }

            }

            return checks;

        }

        /** Invoke String::stripIndent through reflection.

         */

        static String stripIndent(String string) {

            try {

                string = (String)stripIndent.invoke(string);

            } catch (InvocationTargetException | IllegalAccessException ex) {

                throw new RuntimeException(ex);

            }

            return string;

        }

        /** Invoke String::translateEscapes through reflection.

         */

        static String translateEscapes(String string) {

            try {

                string = (String)translateEscapes.invoke(string);

            } catch (InvocationTargetException | IllegalAccessException ex) {

                throw new RuntimeException(ex);

            }

            return string;

        }

    }

    /** Test for EOLN.

     */

    private boolean isEOLN() {

        return reader.ch == LF || reader.ch == CR;

    }

    /** Test for CRLF.

     */

    private boolean isCRLF() {

        return reader.ch == CR && reader.peekChar() == LF;

    }

    /** Count and skip repeated occurances of the specified character.

     */

    private int countChar(char ch, int max) {

        int count = 0;

        for ( ; count < max && reader.bp < reader.buflen && reader.ch == ch; count++) {

            reader.scanChar();

        }

        return count;

    }

    /** Scan a string literal or text block.

     */

    private void scanString(int pos) {

        // Clear flags.

        shouldStripIndent = false;

        shouldTranslateEscapes = false;

        hasBrokenEscapes = false;

        // Check if text block string methods are present.

        boolean hasTextBlockSupport = TextBlockSupport.hasSupport();

        // Track the end of first line for error recovery.

        int firstEOLN = -1;

        // Attempt to scan for up to 3 double quotes.

        int openCount = countChar('\"', 3);

        switch (openCount) {

        case 1: // Starting a string literal.

            break;

        case 2: // Starting an empty string literal.

            // Start again but only consume one quote.

            reader.reset(pos);

            openCount = countChar('\"', 1);

            break;

        case 3: // Starting a text block.

            // Check if preview feature is enabled for text blocks.

            checkSourceLevel(pos, Feature.TEXT_BLOCKS);

            // Only proceed if text block string methods are present.

            if (hasTextBlockSupport) {

                // Indicate that the final string should have incidental indentation removed.

                shouldStripIndent = true;

                // Verify the open delimiter sequence.

                boolean hasOpenEOLN = false;

                while (reader.bp < reader.buflen && Character.isWhitespace(reader.ch)) {

                    hasOpenEOLN = isEOLN();

                    if (hasOpenEOLN) {

                        break;

                    }

                    reader.scanChar();

                }

                // Error if the open delimiter sequence not is """<Whitespace>*<LineTerminator>.

                if (!hasOpenEOLN) {

                    lexError(reader.bp, Errors.IllegalTextBlockOpen);

                    return;

                }

                // Skip line terminator.

                int start = reader.bp;

                if (isCRLF()) {

                    reader.scanChar();

                }

                reader.scanChar();

                processLineTerminator(start, reader.bp);

            } else {

                // No text block string methods are present, so reset and treat like string literal.

                reader.reset(pos);

                openCount = countChar('\"', 1);

            }

            break;

        }

        // While characters are available.

        while (reader.bp < reader.buflen) {

            // If possible close delimiter sequence.

            if (reader.ch == '\"') {

                // Check to see if enough double quotes are present.

                int closeCount = countChar('\"', openCount);

                if (openCount == closeCount) {

                    // Good result.

                    tk = Tokens.TokenKind.STRINGLITERAL;

                    return;

                }

                // False alarm, add double quotes to string buffer.

                reader.repeat('\"', closeCount);

            } else if (isEOLN()) {

                // Line terminator in string literal is an error.

                // Fall out to unclosed string literal error.

                if (openCount == 1) {