src/jdk.compiler/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java
author jlaskey
Thu, 06 Jun 2019 12:24:44 -0300
changeset 55263 830ca7b43b95
parent 53227 f15d443f9731
child 55700 1bb102c1cf27
permissions -rw-r--r--
8223967: Implement Text Blocks (Preview) in the Java compiler Reviewed-by: vromero, jlahoda, abuckley
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     1
/*
50898
12133a6e2613 8205418: Assorted improvements to source code model
jlahoda
parents: 49580
diff changeset
     2
 * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     4
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     7
 * published by the Free Software Foundation.  Oracle designates this
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     8
 * particular file as subject to the "Classpath" exception as provided
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     9
 * by Oracle in the LICENSE file that accompanied this code.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    10
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    11
 * This code is distributed in the hope that it will be useful, but WITHOUT
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    12
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    13
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    14
 * version 2 for more details (a copy is included in the LICENSE file that
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    15
 * accompanied this code).
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    16
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    17
 * You should have received a copy of the GNU General Public License version
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    18
 * 2 along with this work; if not, write to the Free Software Foundation,
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    19
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    20
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    22
 * or visit www.oracle.com if you need additional information or have any
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    23
 * questions.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    24
 */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    25
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    26
package com.sun.tools.javac.parser;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    27
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    28
import com.sun.tools.javac.code.Lint;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    29
import com.sun.tools.javac.code.Lint.LintCategory;
49580
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
    30
import com.sun.tools.javac.code.Preview;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    31
import com.sun.tools.javac.code.Source;
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
    32
import com.sun.tools.javac.code.Source.Feature;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    33
import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
    34
import com.sun.tools.javac.resources.CompilerProperties.Errors;
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    35
import com.sun.tools.javac.resources.CompilerProperties.Warnings;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    36
import com.sun.tools.javac.util.*;
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    37
import com.sun.tools.javac.util.JCDiagnostic.*;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    38
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    39
import java.lang.reflect.InvocationTargetException;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    40
import java.lang.reflect.Method;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    41
import java.nio.CharBuffer;
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    42
import java.util.HashSet;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    43
import java.util.Set;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    44
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    45
import static com.sun.tools.javac.parser.Tokens.*;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    46
import static com.sun.tools.javac.util.LayoutCharacters.*;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    47
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    48
/** The lexical analyzer maps an input stream consisting of
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    49
 *  ASCII characters and Unicode escapes into a token sequence.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    50
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    51
 *  <p><b>This is NOT part of any supported API.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    52
 *  If you write code that depends on this, you do so at your own risk.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    53
 *  This code and its internal interfaces are subject to change or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    54
 *  deletion without notice.</b>
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    55
 */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    56
public class JavaTokenizer {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    57
14801
d66cab4ef397 8003967: detect and remove all mutable implicit static enum fields in langtools
vromero
parents: 14720
diff changeset
    58
    private static final boolean scannerDebug = false;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    59
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    60
    /** The source language setting.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    61
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    62
    private Source source;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    63
49580
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
    64
    /** The preview language setting. */
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
    65
    private Preview preview;
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
    66
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    67
    /** The log to be used for error reporting.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    68
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    69
    private final Log log;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    70
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    71
    /** The token factory. */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    72
    private final Tokens tokens;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    73
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    74
    /** The token kind, set by nextToken().
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    75
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    76
    protected TokenKind tk;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    77
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    78
    /** The token's radix, set by nextToken().
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    79
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    80
    protected int radix;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    81
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    82
    /** The token's name, set by nextToken().
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    83
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    84
    protected Name name;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    85
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    86
    /** The position where a lexical error occurred;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    87
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    88
    protected int errPos = Position.NOPOS;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    89
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    90
    /** The Unicode reader (low-level stream reader).
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    91
     */
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    92
    protected UnicodeReader reader;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    93
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    94
    /** Should the string stripped of indentation?
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    95
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    96
    protected boolean shouldStripIndent;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    97
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    98
    /** Should the string's escapes be translated?
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
    99
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   100
    protected boolean shouldTranslateEscapes;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   101
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   102
    protected ScannerFactory fac;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   103
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   104
    // The set of lint options currently in effect. It is initialized
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   105
    // from the context, and then is set/reset as needed by Attr as it
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   106
    // visits all the various parts of the trees during attribution.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   107
    protected Lint lint;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   108
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   109
    private static final boolean hexFloatsWork = hexFloatsWork();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   110
    private static boolean hexFloatsWork() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   111
        try {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   112
            Float.valueOf("0x1.0p1");
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   113
            return true;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   114
        } catch (NumberFormatException ex) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   115
            return false;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   116
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   117
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   118
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   119
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   120
     * Create a scanner from the input array.  This method might
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   121
     * modify the array.  To avoid copying the input array, ensure
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   122
     * that {@code inputLength < input.length} or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   123
     * {@code input[input.length -1]} is a white space character.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   124
     *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   125
     * @param fac the factory which created this Scanner
14259
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14062
diff changeset
   126
     * @param buf the input, might be modified
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   127
     * Must be positive and less than or equal to input.length.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   128
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   129
    protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   130
        this(fac, new UnicodeReader(fac, buf));
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   131
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   132
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   133
    protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   134
        this(fac, new UnicodeReader(fac, buf, inputLength));
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   135
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   136
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   137
    protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   138
        this.fac = fac;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   139
        this.log = fac.log;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   140
        this.tokens = fac.tokens;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   141
        this.source = fac.source;
49580
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   142
        this.preview = fac.preview;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   143
        this.reader = reader;
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   144
        this.lint = fac.lint;
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   145
    }
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   146
49580
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   147
    protected void checkSourceLevel(int pos, Feature feature) {
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   148
        if (preview.isPreview(feature) && !preview.isEnabled()) {
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   149
            //preview feature without --preview flag, error
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   150
            lexError(DiagnosticFlag.SOURCE_LEVEL, pos, preview.disabledError(feature));
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   151
        } else if (!feature.allowedInSource(source)) {
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   152
            //incompatible source level, error
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   153
            lexError(DiagnosticFlag.SOURCE_LEVEL, pos, feature.error(source.name));
49580
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   154
        } else if (preview.isPreview(feature)) {
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   155
            //use of preview feature, warn
62b908c9f0e6 8199194: Add javac support for preview features
mcimadamore
parents: 48054
diff changeset
   156
            preview.warnPreview(pos, feature);
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   157
        }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   158
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   159
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   160
    /** Report an error at the given position using the provided arguments.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   161
     */
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   162
    protected void lexError(int pos, JCDiagnostic.Error key) {
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   163
        log.error(pos, key);
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   164
        tk = TokenKind.ERROR;
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   165
        errPos = pos;
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   166
    }
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   167
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   168
    protected void lexError(DiagnosticFlag flags, int pos, JCDiagnostic.Error key) {
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   169
        log.error(flags, pos, key);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   170
        tk = TokenKind.ERROR;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   171
        errPos = pos;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   172
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   173
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   174
    protected void lexWarning(LintCategory lc, int pos, JCDiagnostic.Warning key) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   175
        DiagnosticPosition dp = new SimpleDiagnosticPosition(pos) ;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   176
        log.warning(lc, dp, key);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   177
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   178
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   179
    /** Read next character in character or string literal and copy into sbuf.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   180
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   181
    private void scanLitChar(int pos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   182
        if (reader.ch == '\\') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   183
            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   184
                reader.skipChar();
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   185
                reader.putChar('\\', true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   186
            } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   187
                reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   188
                switch (reader.ch) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   189
                case '0': case '1': case '2': case '3':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   190
                case '4': case '5': case '6': case '7':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   191
                    char leadch = reader.ch;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   192
                    int oct = reader.digit(pos, 8);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   193
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   194
                    if ('0' <= reader.ch && reader.ch <= '7') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   195
                        oct = oct * 8 + reader.digit(pos, 8);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   196
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   197
                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   198
                            oct = oct * 8 + reader.digit(pos, 8);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   199
                            reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   200
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   201
                    }
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   202
                    reader.putChar((char)oct);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   203
                    break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   204
                case 'b':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   205
                    reader.putChar('\b', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   206
                case 't':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   207
                    reader.putChar('\t', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   208
                case 'n':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   209
                    reader.putChar('\n', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   210
                case 'f':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   211
                    reader.putChar('\f', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   212
                case 'r':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   213
                    reader.putChar('\r', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   214
                case '\'':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   215
                    reader.putChar('\'', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   216
                case '\"':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   217
                    reader.putChar('\"', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   218
                case '\\':
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   219
                    reader.putChar('\\', true); break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   220
                default:
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   221
                    lexError(reader.bp, Errors.IllegalEscChar);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   222
                }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   223
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   224
        } else if (reader.bp != reader.buflen) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   225
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   226
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   227
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   228
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   229
    /** Read next character in character or string literal and copy into sbuf
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   230
     *  without translating escapes. Used by text blocks to preflight verify
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   231
     *  escapes sequences.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   232
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   233
    private void scanLitCharRaw(int pos) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   234
        if (reader.ch == '\\') {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   235
            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   236
                reader.skipChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   237
                reader.putChar('\\', false);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   238
                reader.putChar('\\', true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   239
            } else {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   240
                reader.putChar('\\', true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   241
                switch (reader.ch) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   242
                case '0': case '1': case '2': case '3':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   243
                case '4': case '5': case '6': case '7':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   244
                    char leadch = reader.ch;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   245
                    reader.putChar(true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   246
                    if ('0' <= reader.ch && reader.ch <= '7') {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   247
                        reader.putChar(true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   248
                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   249
                            reader.putChar(true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   250
                        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   251
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   252
                    break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   253
                // Effectively list of valid escape sequences.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   254
                case 'b':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   255
                case 't':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   256
                case 'n':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   257
                case 'f':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   258
                case 'r':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   259
                case '\'':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   260
                case '\"':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   261
                case '\\':
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   262
                    reader.putChar(true); break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   263
                default:
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   264
                    lexError(reader.bp, Errors.IllegalEscChar);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   265
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   266
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   267
        } else if (reader.bp != reader.buflen) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   268
            reader.putChar(true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   269
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   270
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   271
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   272
    /** Interim access to String methods used to support text blocks.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   273
     *  Required to handle bootstrapping with pre-text block jdks.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   274
     *  Could be reworked in the 'next' jdk.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   275
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   276
    static class TextBlockSupport {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   277
        /** Reflection method to remove incidental indentation.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   278
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   279
        private static final Method stripIndent;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   280
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   281
        /** Reflection method to translate escape sequences.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   282
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   283
        private static final Method translateEscapes;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   284
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   285
        /** true if stripIndent and translateEscapes are available in the bootstrap jdk.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   286
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   287
        private static final boolean hasSupport;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   288
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   289
        /** Get a string method via refection or null if not available.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   290
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   291
        private static Method getStringMethodOrNull(String name) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   292
            try {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   293
                return String.class.getMethod(name);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   294
            } catch (Exception ex) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   295
                // Method not available, return null.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   296
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   297
            return null;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   298
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   299
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   300
        static {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   301
            // Get text block string methods.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   302
            stripIndent = getStringMethodOrNull("stripIndent");
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   303
            translateEscapes = getStringMethodOrNull("translateEscapes");
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   304
            // true if stripIndent and translateEscapes are available in the bootstrap jdk.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   305
            hasSupport = stripIndent != null && translateEscapes != null;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   306
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   307
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   308
        /** Return true if stripIndent and translateEscapes are available in the bootstrap jdk.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   309
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   310
        static boolean hasSupport() {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   311
            return hasSupport;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   312
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   313
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   314
        /** Return the leading whitespace count (indentation) of the line.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   315
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   316
        private static int indent(String line) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   317
            return line.length() - line.stripLeading().length();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   318
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   319
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   320
        enum WhitespaceChecks {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   321
            INCONSISTENT,
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   322
            TRAILING
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   323
        };
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   324
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   325
        /** Check that the use of white space in content is not problematic.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   326
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   327
        static Set<WhitespaceChecks> checkWhitespace(String string) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   328
            // Start with empty result set.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   329
            Set<WhitespaceChecks> checks = new HashSet<>();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   330
            // No need to check empty strings.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   331
            if (string.isEmpty()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   332
                return checks;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   333
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   334
            // Maximum common indentation.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   335
            int outdent = 0;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   336
            // No need to check indentation if opting out (last line is empty.)
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   337
            char lastChar = string.charAt(string.length() - 1);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   338
            boolean optOut = lastChar == '\n' || lastChar == '\r';
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   339
            // Split string based at line terminators.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   340
            String[] lines = string.split("\\R");
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   341
            int length = lines.length;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   342
            // Extract last line.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   343
            String lastLine = lines[length - 1];
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   344
            if (!optOut) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   345
                // Prime with the last line indentation (may be blank.)
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   346
                outdent = indent(lastLine);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   347
                for (String line : lines) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   348
                    // Blanks lines have no influence (last line accounted for.)
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   349
                    if (!line.isBlank()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   350
                        outdent = Integer.min(outdent, indent(line));
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   351
                        if (outdent == 0) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   352
                            break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   353
                        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   354
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   355
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   356
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   357
            // Last line is representative.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   358
            String start = lastLine.substring(0, outdent);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   359
            for (String line : lines) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   360
                // Fail if a line does not have the same indentation.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   361
                if (!line.isBlank() && !line.startsWith(start)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   362
                    // Mix of different white space
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   363
                    checks.add(WhitespaceChecks.INCONSISTENT);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   364
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   365
                // Line has content even after indent is removed.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   366
                if (outdent < line.length()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   367
                    // Is the last character a white space.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   368
                    lastChar = line.charAt(line.length() - 1);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   369
                    if (Character.isWhitespace(lastChar)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   370
                        // Has trailing white space.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   371
                        checks.add(WhitespaceChecks.TRAILING);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   372
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   373
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   374
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   375
            return checks;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   376
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   377
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   378
        /** Invoke String::stripIndent through reflection.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   379
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   380
        static String stripIndent(String string) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   381
            try {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   382
                string = (String)stripIndent.invoke(string);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   383
            } catch (InvocationTargetException | IllegalAccessException ex) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   384
                throw new RuntimeException(ex);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   385
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   386
            return string;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   387
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   388
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   389
        /** Invoke String::translateEscapes through reflection.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   390
         */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   391
        static String translateEscapes(String string) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   392
            try {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   393
                string = (String)translateEscapes.invoke(string);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   394
            } catch (InvocationTargetException | IllegalAccessException ex) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   395
                throw new RuntimeException(ex);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   396
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   397
            return string;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   398
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   399
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   400
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   401
    /** Test for EOLN.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   402
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   403
    private boolean isEOLN() {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   404
        return reader.ch == LF || reader.ch == CR;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   405
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   406
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   407
    /** Test for CRLF.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   408
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   409
    private boolean isCRLF() {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   410
        return reader.ch == CR && reader.peekChar() == LF;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   411
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   412
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   413
    /** Count and skip repeated occurances of the specified character.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   414
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   415
    private int countChar(char ch, int max) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   416
        int count = 0;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   417
        for ( ; count < max && reader.bp < reader.buflen && reader.ch == ch; count++) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   418
            reader.scanChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   419
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   420
        return count;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   421
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   422
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   423
    /** Scan a string literal or text block.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   424
     */
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   425
    private void scanString(int pos) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   426
        // Clear flags.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   427
        shouldStripIndent = false;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   428
        shouldTranslateEscapes = false;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   429
        // Check if text block string methods are present.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   430
        boolean hasTextBlockSupport = TextBlockSupport.hasSupport();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   431
        // Track the end of first line for error recovery.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   432
        int firstEOLN = -1;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   433
        // Attempt to scan for up to 3 double quotes.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   434
        int openCount = countChar('\"', 3);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   435
        switch (openCount) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   436
        case 1: // Starting a string literal.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   437
            break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   438
        case 2: // Starting an empty string literal.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   439
            // Start again but only consume one quote.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   440
            reader.reset(pos);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   441
            openCount = countChar('\"', 1);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   442
            break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   443
        case 3: // Starting a text block.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   444
            // Check if preview feature is enabled for text blocks.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   445
            checkSourceLevel(pos, Feature.TEXT_BLOCKS);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   446
            // Only proceed if text block string methods are present.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   447
            if (hasTextBlockSupport) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   448
                // Indicate that the final string should have incidental indentation removed.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   449
                shouldStripIndent = true;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   450
                // Verify the open delimiter sequence.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   451
                boolean hasOpenEOLN = false;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   452
                while (reader.bp < reader.buflen && Character.isWhitespace(reader.ch)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   453
                    hasOpenEOLN = isEOLN();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   454
                    if (hasOpenEOLN) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   455
                        break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   456
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   457
                    reader.scanChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   458
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   459
                // Error if the open delimiter sequence not is """<Whitespace>*<LineTerminator>.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   460
                if (!hasOpenEOLN) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   461
                    lexError(reader.bp, Errors.IllegalTextBlockOpen);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   462
                    return;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   463
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   464
                // Skip line terminator.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   465
                int start = reader.bp;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   466
                if (isCRLF()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   467
                    reader.scanChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   468
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   469
                reader.scanChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   470
                processLineTerminator(start, reader.bp);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   471
            } else {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   472
                // No text block string methods are present, so reset and treat like string literal.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   473
                reader.reset(pos);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   474
                openCount = countChar('\"', 1);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   475
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   476
            break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   477
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   478
        // While characters are available.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   479
        while (reader.bp < reader.buflen) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   480
            // If possible close delimiter sequence.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   481
            if (reader.ch == '\"') {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   482
                // Check to see if enough double quotes are present.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   483
                int closeCount = countChar('\"', openCount);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   484
                if (openCount == closeCount) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   485
                    // Good result.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   486
                    tk = Tokens.TokenKind.STRINGLITERAL;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   487
                    return;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   488
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   489
                // False alarm, add double quotes to string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   490
                reader.repeat('\"', closeCount);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   491
            } else if (isEOLN()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   492
                // Line terminator in string literal is an error.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   493
                // Fall out to unclosed string literal error.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   494
                if (openCount == 1) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   495
                    break;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   496
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   497
                 // Add line terminator to string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   498
                int start = reader.bp;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   499
                if (isCRLF()) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   500
                    reader.scanChar();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   501
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   502
                reader.putChar('\n', true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   503
                processLineTerminator(start, reader.bp);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   504
                // Record first line terminator for error recovery.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   505
                if (firstEOLN == -1) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   506
                    firstEOLN = reader.bp;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   507
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   508
            } else if (reader.ch == '\\') {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   509
                // Handle escape sequences.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   510
                if (hasTextBlockSupport) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   511
                    // Indicate that the final string should have escapes translated.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   512
                    shouldTranslateEscapes = true;
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   513
                    // Validate escape sequence and add to string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   514
                    scanLitCharRaw(pos);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   515
                } else {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   516
                    // Translate escape sequence and add result to string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   517
                    scanLitChar(pos);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   518
                }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   519
            } else {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   520
                // Add character to string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   521
                reader.putChar(true);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   522
            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   523
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   524
        // String ended without close delimiter sequence.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   525
        lexError(pos, openCount == 1 ? Errors.UnclosedStrLit : Errors.UnclosedTextBlock);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   526
        if (firstEOLN  != -1) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   527
            // Reset recovery position to point after open delimiter sequence.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   528
            reader.reset(firstEOLN);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   529
        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   530
    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   531
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   532
    private void scanDigits(int pos, int digitRadix) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   533
        char saveCh;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   534
        int savePos;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   535
        do {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   536
            if (reader.ch != '_') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   537
                reader.putChar(false);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   538
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   539
            saveCh = reader.ch;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   540
            savePos = reader.bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   541
            reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   542
        } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   543
        if (saveCh == '_')
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   544
            lexError(savePos, Errors.IllegalUnderscore);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   545
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   546
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   547
    /** Read fractional part of hexadecimal floating point number.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   548
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   549
    private void scanHexExponentAndSuffix(int pos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   550
        if (reader.ch == 'p' || reader.ch == 'P') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   551
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   552
            skipIllegalUnderscores();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   553
            if (reader.ch == '+' || reader.ch == '-') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   554
                reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   555
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   556
            skipIllegalUnderscores();
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   557
            if (reader.digit(pos, 10) >= 0) {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   558
                scanDigits(pos, 10);
25445
603f0c93d5c9 8011044: Remove support for 1.5 and earlier source and target options
ntoda
parents: 22163
diff changeset
   559
                if (!hexFloatsWork)
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   560
                    lexError(pos, Errors.UnsupportedCrossFpLit);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   561
            } else
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   562
                lexError(pos, Errors.MalformedFpLit);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   563
        } else {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   564
            lexError(pos, Errors.MalformedFpLit);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   565
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   566
        if (reader.ch == 'f' || reader.ch == 'F') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   567
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   568
            tk = TokenKind.FLOATLITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   569
            radix = 16;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   570
        } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   571
            if (reader.ch == 'd' || reader.ch == 'D') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   572
                reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   573
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   574
            tk = TokenKind.DOUBLELITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   575
            radix = 16;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   576
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   577
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   578
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   579
    /** Read fractional part of floating point number.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   580
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   581
    private void scanFraction(int pos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   582
        skipIllegalUnderscores();
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   583
        if (reader.digit(pos, 10) >= 0) {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   584
            scanDigits(pos, 10);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   585
        }
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   586
        int sp1 = reader.sp;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   587
        if (reader.ch == 'e' || reader.ch == 'E') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   588
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   589
            skipIllegalUnderscores();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   590
            if (reader.ch == '+' || reader.ch == '-') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   591
                reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   592
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   593
            skipIllegalUnderscores();
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   594
            if (reader.digit(pos, 10) >= 0) {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   595
                scanDigits(pos, 10);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   596
                return;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   597
            }
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   598
            lexError(pos, Errors.MalformedFpLit);
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   599
            reader.sp = sp1;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   600
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   601
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   602
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   603
    /** Read fractional part and 'd' or 'f' suffix of floating point number.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   604
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   605
    private void scanFractionAndSuffix(int pos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   606
        radix = 10;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   607
        scanFraction(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   608
        if (reader.ch == 'f' || reader.ch == 'F') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   609
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   610
            tk = TokenKind.FLOATLITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   611
        } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   612
            if (reader.ch == 'd' || reader.ch == 'D') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   613
                reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   614
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   615
            tk = TokenKind.DOUBLELITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   616
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   617
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   618
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   619
    /** Read fractional part and 'd' or 'f' suffix of floating point number.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   620
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   621
    private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   622
        radix = 16;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   623
        Assert.check(reader.ch == '.');
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   624
        reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   625
        skipIllegalUnderscores();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   626
        if (reader.digit(pos, 16) >= 0) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   627
            seendigit = true;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   628
            scanDigits(pos, 16);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   629
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   630
        if (!seendigit)
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   631
            lexError(pos, Errors.InvalidHexNumber);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   632
        else
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   633
            scanHexExponentAndSuffix(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   634
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   635
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   636
    private void skipIllegalUnderscores() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   637
        if (reader.ch == '_') {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   638
            lexError(reader.bp, Errors.IllegalUnderscore);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   639
            while (reader.ch == '_')
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   640
                reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   641
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   642
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   643
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   644
    /** Read a number.
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   645
     *  @param radix  The radix of the number; one of 2, 8, 10, 16.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   646
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   647
    private void scanNumber(int pos, int radix) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   648
        // for octal, allow base-10 digit in case it's a float literal
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   649
        this.radix = radix;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   650
        int digitRadix = (radix == 8 ? 10 : radix);
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   651
        int firstDigit = reader.digit(pos, Math.max(10, digitRadix));
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   652
        boolean seendigit = firstDigit >= 0;
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   653
        boolean seenValidDigit = firstDigit >= 0 && firstDigit < digitRadix;
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   654
        if (seendigit) {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   655
            scanDigits(pos, digitRadix);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   656
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   657
        if (radix == 16 && reader.ch == '.') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   658
            scanHexFractionAndSuffix(pos, seendigit);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   659
        } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   660
            scanHexExponentAndSuffix(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   661
        } else if (digitRadix == 10 && reader.ch == '.') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   662
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   663
            scanFractionAndSuffix(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   664
        } else if (digitRadix == 10 &&
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   665
                   (reader.ch == 'e' || reader.ch == 'E' ||
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   666
                    reader.ch == 'f' || reader.ch == 'F' ||
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   667
                    reader.ch == 'd' || reader.ch == 'D')) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   668
            scanFractionAndSuffix(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   669
        } else {
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   670
            if (!seenValidDigit) {
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   671
                switch (radix) {
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   672
                case 2:
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   673
                    lexError(pos, Errors.InvalidBinaryNumber);
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   674
                    break;
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   675
                case 16:
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   676
                    lexError(pos, Errors.InvalidHexNumber);
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   677
                    break;
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   678
                }
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   679
            }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   680
            if (reader.ch == 'l' || reader.ch == 'L') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   681
                reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   682
                tk = TokenKind.LONGLITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   683
            } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   684
                tk = TokenKind.INTLITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   685
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   686
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   687
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   688
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   689
    /** Read an identifier.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   690
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   691
    private void scanIdent() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   692
        boolean isJavaIdentifierPart;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   693
        char high;
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   694
        reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   695
        do {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   696
            switch (reader.ch) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   697
            case 'A': case 'B': case 'C': case 'D': case 'E':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   698
            case 'F': case 'G': case 'H': case 'I': case 'J':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   699
            case 'K': case 'L': case 'M': case 'N': case 'O':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   700
            case 'P': case 'Q': case 'R': case 'S': case 'T':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   701
            case 'U': case 'V': case 'W': case 'X': case 'Y':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   702
            case 'Z':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   703
            case 'a': case 'b': case 'c': case 'd': case 'e':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   704
            case 'f': case 'g': case 'h': case 'i': case 'j':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   705
            case 'k': case 'l': case 'm': case 'n': case 'o':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   706
            case 'p': case 'q': case 'r': case 's': case 't':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   707
            case 'u': case 'v': case 'w': case 'x': case 'y':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   708
            case 'z':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   709
            case '$': case '_':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   710
            case '0': case '1': case '2': case '3': case '4':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   711
            case '5': case '6': case '7': case '8': case '9':
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   712
                break;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   713
            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   714
            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   715
            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   716
            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   717
            case '\u0015': case '\u0016': case '\u0017':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   718
            case '\u0018': case '\u0019': case '\u001B':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   719
            case '\u007F':
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   720
                reader.scanChar();
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   721
                continue;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   722
            case '\u001A': // EOI is also a legal identifier part
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   723
                if (reader.bp >= reader.buflen) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   724
                    name = reader.name();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   725
                    tk = tokens.lookupKind(name);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   726
                    return;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   727
                }
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   728
                reader.scanChar();
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   729
                continue;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   730
            default:
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   731
                if (reader.ch < '\u0080') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   732
                    // all ASCII range chars already handled, above
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   733
                    isJavaIdentifierPart = false;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   734
                } else {
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   735
                    if (Character.isIdentifierIgnorable(reader.ch)) {
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   736
                        reader.scanChar();
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   737
                        continue;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   738
                    } else {
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   739
                        int codePoint = reader.peekSurrogates();
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   740
                        if (codePoint >= 0) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   741
                            if (isJavaIdentifierPart = Character.isJavaIdentifierPart(codePoint)) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   742
                                reader.putChar(true);
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   743
                            }
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   744
                        } else {
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   745
                            isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   746
                        }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   747
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   748
                }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   749
                if (!isJavaIdentifierPart) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   750
                    name = reader.name();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   751
                    tk = tokens.lookupKind(name);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   752
                    return;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   753
                }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   754
            }
14720
c24c61d0d9a6 7144981: javac should ignore ignorable characters in input
vromero
parents: 14370
diff changeset
   755
            reader.putChar(true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   756
        } while (true);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   757
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   758
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   759
    /** Return true if reader.ch can be part of an operator.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   760
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   761
    private boolean isSpecial(char ch) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   762
        switch (ch) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   763
        case '!': case '%': case '&': case '*': case '?':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   764
        case '+': case '-': case ':': case '<': case '=':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   765
        case '>': case '^': case '|': case '~':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   766
        case '@':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   767
            return true;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   768
        default:
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   769
            return false;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   770
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   771
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   772
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   773
    /** Read longest possible sequence of special characters and convert
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   774
     *  to token.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   775
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   776
    private void scanOperator() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   777
        while (true) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   778
            reader.putChar(false);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   779
            Name newname = reader.name();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   780
            TokenKind tk1 = tokens.lookupKind(newname);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   781
            if (tk1 == TokenKind.IDENTIFIER) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   782
                reader.sp--;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   783
                break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   784
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   785
            tk = tk1;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   786
            reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   787
            if (!isSpecial(reader.ch)) break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   788
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   789
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   790
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   791
    /** Read token.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   792
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   793
    public Token readToken() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   794
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   795
        reader.sp = 0;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   796
        name = null;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   797
        radix = 0;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   798
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   799
        int pos = 0;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   800
        int endPos = 0;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   801
        List<Comment> comments = null;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   802
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   803
        try {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   804
            loop: while (true) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   805
                pos = reader.bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   806
                switch (reader.ch) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   807
                case ' ': // (Spec 3.6)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   808
                case '\t': // (Spec 3.6)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   809
                case FF: // (Spec 3.6)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   810
                    do {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   811
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   812
                    } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   813
                    processWhiteSpace(pos, reader.bp);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   814
                    break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   815
                case LF: // (Spec 3.4)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   816
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   817
                    processLineTerminator(pos, reader.bp);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   818
                    break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   819
                case CR: // (Spec 3.4)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   820
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   821
                    if (reader.ch == LF) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   822
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   823
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   824
                    processLineTerminator(pos, reader.bp);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   825
                    break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   826
                case 'A': case 'B': case 'C': case 'D': case 'E':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   827
                case 'F': case 'G': case 'H': case 'I': case 'J':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   828
                case 'K': case 'L': case 'M': case 'N': case 'O':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   829
                case 'P': case 'Q': case 'R': case 'S': case 'T':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   830
                case 'U': case 'V': case 'W': case 'X': case 'Y':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   831
                case 'Z':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   832
                case 'a': case 'b': case 'c': case 'd': case 'e':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   833
                case 'f': case 'g': case 'h': case 'i': case 'j':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   834
                case 'k': case 'l': case 'm': case 'n': case 'o':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   835
                case 'p': case 'q': case 'r': case 's': case 't':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   836
                case 'u': case 'v': case 'w': case 'x': case 'y':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   837
                case 'z':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   838
                case '$': case '_':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   839
                    scanIdent();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   840
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   841
                case '0':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   842
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   843
                    if (reader.ch == 'x' || reader.ch == 'X') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   844
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   845
                        skipIllegalUnderscores();
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   846
                        scanNumber(pos, 16);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   847
                    } else if (reader.ch == 'b' || reader.ch == 'B') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   848
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   849
                        skipIllegalUnderscores();
34997
8174a7d851fb 8056897: Improve error recovery for empty binary and hexadecimal literals.
jlahoda
parents: 25874
diff changeset
   850
                        scanNumber(pos, 2);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   851
                    } else {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   852
                        reader.putChar('0');
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   853
                        if (reader.ch == '_') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   854
                            int savePos = reader.bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   855
                            do {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   856
                                reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   857
                            } while (reader.ch == '_');
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   858
                            if (reader.digit(pos, 10) < 0) {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   859
                                lexError(savePos, Errors.IllegalUnderscore);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   860
                            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   861
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   862
                        scanNumber(pos, 8);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   863
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   864
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   865
                case '1': case '2': case '3': case '4':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   866
                case '5': case '6': case '7': case '8': case '9':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   867
                    scanNumber(pos, 10);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   868
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   869
                case '.':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   870
                    reader.scanChar();
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   871
                    if (reader.digit(pos, 10) >= 0) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   872
                        reader.putChar('.');
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   873
                        scanFractionAndSuffix(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   874
                    } else if (reader.ch == '.') {
11383
ae090403accf 4881269: improve diagnostic for ill-formed tokens
jjg
parents: 11144
diff changeset
   875
                        int savePos = reader.bp;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   876
                        reader.putChar('.'); reader.putChar('.', true);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   877
                        if (reader.ch == '.') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   878
                            reader.scanChar();
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   879
                            reader.putChar('.');
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   880
                            tk = TokenKind.ELLIPSIS;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   881
                        } else {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   882
                            lexError(savePos, Errors.IllegalDot);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   883
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   884
                    } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   885
                        tk = TokenKind.DOT;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   886
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   887
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   888
                case ',':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   889
                    reader.scanChar(); tk = TokenKind.COMMA; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   890
                case ';':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   891
                    reader.scanChar(); tk = TokenKind.SEMI; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   892
                case '(':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   893
                    reader.scanChar(); tk = TokenKind.LPAREN; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   894
                case ')':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   895
                    reader.scanChar(); tk = TokenKind.RPAREN; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   896
                case '[':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   897
                    reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   898
                case ']':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   899
                    reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   900
                case '{':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   901
                    reader.scanChar(); tk = TokenKind.LBRACE; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   902
                case '}':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   903
                    reader.scanChar(); tk = TokenKind.RBRACE; break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   904
                case '/':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   905
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   906
                    if (reader.ch == '/') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   907
                        do {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   908
                            reader.scanCommentChar();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   909
                        } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   910
                        if (reader.bp < reader.buflen) {
13077
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
   911
                            comments = addComment(comments, processComment(pos, reader.bp, CommentStyle.LINE));
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   912
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   913
                        break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   914
                    } else if (reader.ch == '*') {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   915
                        boolean isEmpty = false;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   916
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   917
                        CommentStyle style;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   918
                        if (reader.ch == '*') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   919
                            style = CommentStyle.JAVADOC;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   920
                            reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   921
                            if (reader.ch == '/') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   922
                                isEmpty = true;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   923
                            }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   924
                        } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   925
                            style = CommentStyle.BLOCK;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   926
                        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   927
                        while (!isEmpty && reader.bp < reader.buflen) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   928
                            if (reader.ch == '*') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   929
                                reader.scanChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   930
                                if (reader.ch == '/') break;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   931
                            } else {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   932
                                reader.scanCommentChar();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   933
                            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   934
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   935
                        if (reader.ch == '/') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   936
                            reader.scanChar();
13077
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
   937
                            comments = addComment(comments, processComment(pos, reader.bp, style));
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   938
                            break;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   939
                        } else {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   940
                            lexError(pos, Errors.UnclosedComment);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   941
                            break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   942
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   943
                    } else if (reader.ch == '=') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   944
                        tk = TokenKind.SLASHEQ;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   945
                        reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   946
                    } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   947
                        tk = TokenKind.SLASH;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   948
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   949
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   950
                case '\'':
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   951
                    reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   952
                    if (reader.ch == '\'') {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   953
                        lexError(pos, Errors.EmptyCharLit);
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   954
                        reader.scanChar();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   955
                    } else {
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   956
                        if (isEOLN())
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   957
                            lexError(pos, Errors.IllegalLineEndInCharLit);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   958
                        scanLitChar(pos);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   959
                        if (reader.ch == '\'') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   960
                            reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   961
                            tk = TokenKind.CHARLITERAL;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   962
                        } else {
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
   963
                            lexError(pos, Errors.UnclosedCharLit);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   964
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   965
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   966
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   967
                case '\"':
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   968
                    scanString(pos);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   969
                    break loop;
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
   970
                default:
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   971
                    if (isSpecial(reader.ch)) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   972
                        scanOperator();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   973
                    } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   974
                        boolean isJavaIdentifierStart;
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   975
                        int codePoint = -1;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   976
                        if (reader.ch < '\u0080') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   977
                            // all ASCII range chars already handled, above
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   978
                            isJavaIdentifierStart = false;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   979
                        } else {
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   980
                            codePoint = reader.peekSurrogates();
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   981
                            if (codePoint >= 0) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   982
                                if (isJavaIdentifierStart = Character.isJavaIdentifierStart(codePoint)) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   983
                                    reader.putChar(true);
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   984
                                }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   985
                            } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   986
                                isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   987
                            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   988
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   989
                        if (isJavaIdentifierStart) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   990
                            scanIdent();
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   991
                        } else if (reader.digit(pos, 10) >= 0) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   992
                            scanNumber(pos, 10);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   993
                        } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   994
                            tk = TokenKind.EOF;
50898
12133a6e2613 8205418: Assorted improvements to source code model
jlahoda
parents: 49580
diff changeset
   995
                            pos = reader.realLength;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   996
                        } else {
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   997
                            String arg;
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   998
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
   999
                            if (codePoint >= 0) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1000
                                char high = reader.ch;
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1001
                                reader.scanChar();
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1002
                                arg = String.format("\\u%04x\\u%04x", (int) high, (int)reader.ch);
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1003
                            } else {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1004
                                arg = (32 < reader.ch && reader.ch < 127) ?
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1005
                                                String.format("%s", reader.ch) :
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1006
                                                String.format("\\u%04x", (int)reader.ch);
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 25445
diff changeset
  1007
                            }
48054
702043a4cdeb 8189749: Devise strategy for making source level checks more uniform
mcimadamore
parents: 47216
diff changeset
  1008
                            lexError(pos, Errors.IllegalChar(arg));
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1009
                            reader.scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1010
                        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1011
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1012
                    break loop;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1013
                }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1014
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1015
            endPos = reader.bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1016
            switch (tk.tag) {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1017
                case DEFAULT: return new Token(tk, pos, endPos, comments);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1018
                case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
55263
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1019
                case STRING: {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1020
                    // Get characters from string buffer.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1021
                    String string = reader.chars();
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1022
                    // If a text block.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1023
                    if (shouldStripIndent) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1024
                        // Verify that the incidental indentation is consistent.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1025
                        if (lint.isEnabled(LintCategory.TEXT_BLOCKS)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1026
                            Set<TextBlockSupport.WhitespaceChecks> checks =
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1027
                                    TextBlockSupport.checkWhitespace(string);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1028
                            if (checks.contains(TextBlockSupport.WhitespaceChecks.INCONSISTENT)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1029
                                lexWarning(LintCategory.TEXT_BLOCKS, pos,
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1030
                                        Warnings.InconsistentWhiteSpaceIndentation);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1031
                            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1032
                            if (checks.contains(TextBlockSupport.WhitespaceChecks.TRAILING)) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1033
                                lexWarning(LintCategory.TEXT_BLOCKS, pos,
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1034
                                        Warnings.TrailingWhiteSpaceWillBeRemoved);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1035
                            }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1036
                        }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1037
                        // Remove incidental indentation.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1038
                        string = TextBlockSupport.stripIndent(string);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1039
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1040
                    // Translate escape sequences if present.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1041
                    if (shouldTranslateEscapes) {
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1042
                        string = TextBlockSupport.translateEscapes(string);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1043
                    }
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1044
                    // Build string token.
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1045
                    return new StringToken(tk, pos, endPos, string, comments);
830ca7b43b95 8223967: Implement Text Blocks (Preview) in the Java compiler
jlaskey
parents: 53227
diff changeset
  1046
                }
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1047
                case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1048
                default: throw new AssertionError();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1049
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1050
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1051
        finally {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1052
            if (scannerDebug) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1053
                    System.out.println("nextToken(" + pos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1054
                                       + "," + endPos + ")=|" +
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1055
                                       new String(reader.getRawCharacters(pos, endPos))
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1056
                                       + "|");
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1057
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1058
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1059
    }
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1060
    //where
13077
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
  1061
        List<Comment> addComment(List<Comment> comments, Comment comment) {
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
  1062
            return comments == null ?
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
  1063
                    List.of(comment) :
16fb753bb5dc 7174143: encapsulate doc comment table
jjg
parents: 11383
diff changeset
  1064
                    comments.prepend(comment);
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1065
        }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1066
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1067
    /** Return the position where a lexical error occurred;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1068
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1069
    public int errPos() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1070
        return errPos;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1071
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1072
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1073
    /** Set the position where a lexical error occurred;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1074
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1075
    public void errPos(int pos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1076
        errPos = pos;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1077
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1078
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1079
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1080
     * Called when a complete comment has been scanned. pos and endPos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1081
     * will mark the comment boundary.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1082
     */
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1083
    protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1084
        if (scannerDebug)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1085
            System.out.println("processComment(" + pos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1086
                               + "," + endPos + "," + style + ")=|"
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1087
                               + new String(reader.getRawCharacters(pos, endPos))
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1088
                               + "|");
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1089
        char[] buf = reader.getRawCharacters(pos, endPos);
22163
3651128c74eb 8030244: Update langtools to use Diamond
briangoetz
parents: 16810
diff changeset
  1090
        return new BasicComment<>(new UnicodeReader(fac, buf, buf.length), style);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1091
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1092
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1093
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1094
     * Called when a complete whitespace run has been scanned. pos and endPos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1095
     * will mark the whitespace boundary.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1096
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1097
    protected void processWhiteSpace(int pos, int endPos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1098
        if (scannerDebug)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1099
            System.out.println("processWhitespace(" + pos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1100
                               + "," + endPos + ")=|" +
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1101
                               new String(reader.getRawCharacters(pos, endPos))
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1102
                               + "|");
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1103
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1104
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1105
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1106
     * Called when a line terminator has been processed.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1107
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1108
    protected void processLineTerminator(int pos, int endPos) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1109
        if (scannerDebug)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1110
            System.out.println("processTerminator(" + pos
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1111
                               + "," + endPos + ")=|" +
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1112
                               new String(reader.getRawCharacters(pos, endPos))
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1113
                               + "|");
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1114
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1115
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1116
    /** Build a map for translating between line numbers and
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1117
     * positions in the input.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1118
     *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1119
     * @return a LineMap */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1120
    public Position.LineMap getLineMap() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1121
        return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1122
    }
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1123
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1124
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1125
    /**
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1126
    * Scan a documentation comment; determine if a deprecated tag is present.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1127
    * Called once the initial /, * have been skipped, positioned at the second *
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1128
    * (which is treated as the beginning of the first line).
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1129
    * Stops positioned at the closing '/'.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1130
    */
16810
670d880283e2 8011676: Instances of Tokens.Comment should not be defined in inner classes
jjg
parents: 14801
diff changeset
  1131
    protected static class BasicComment<U extends UnicodeReader> implements Comment {
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1132
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1133
        CommentStyle cs;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1134
        U comment_reader;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1135
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1136
        protected boolean deprecatedFlag = false;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1137
        protected boolean scanned = false;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1138
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1139
        protected BasicComment(U comment_reader, CommentStyle cs) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1140
            this.comment_reader = comment_reader;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1141
            this.cs = cs;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1142
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1143
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1144
        public String getText() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1145
            return null;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1146
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1147
13078
11d5e1c88864 7178297: provide mapping from doc comment position to source file position
jjg
parents: 13077
diff changeset
  1148
        public int getSourcePos(int pos) {
11d5e1c88864 7178297: provide mapping from doc comment position to source file position
jjg
parents: 13077
diff changeset
  1149
            return -1;
11d5e1c88864 7178297: provide mapping from doc comment position to source file position
jjg
parents: 13077
diff changeset
  1150
        }
11d5e1c88864 7178297: provide mapping from doc comment position to source file position
jjg
parents: 13077
diff changeset
  1151
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1152
        public CommentStyle getStyle() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1153
            return cs;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1154
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1155
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1156
        public boolean isDeprecated() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1157
            if (!scanned && cs == CommentStyle.JAVADOC) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1158
                scanDocComment();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1159
            }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1160
            return deprecatedFlag;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1161
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1162
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1163
        @SuppressWarnings("fallthrough")
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1164
        protected void scanDocComment() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1165
            try {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1166
                boolean deprecatedPrefix = false;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1167
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1168
                comment_reader.bp += 3; // '/**'
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1169
                comment_reader.ch = comment_reader.buf[comment_reader.bp];
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1170
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1171
                forEachLine:
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1172
                while (comment_reader.bp < comment_reader.buflen) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1173
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1174
                    // Skip optional WhiteSpace at beginning of line
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1175
                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1176
                        comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1177
                    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1178
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1179
                    // Skip optional consecutive Stars
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1180
                    while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1181
                        comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1182
                        if (comment_reader.ch == '/') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1183
                            return;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1184
                        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1185
                    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1186
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1187
                    // Skip optional WhiteSpace after Stars
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1188
                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1189
                        comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1190
                    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1191
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1192
                    deprecatedPrefix = false;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1193
                    // At beginning of line in the JavaDoc sense.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1194
                    if (!deprecatedFlag) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1195
                        String deprecated = "@deprecated";
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1196
                        int i = 0;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1197
                        while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1198
                            comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1199
                            i++;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1200
                            if (i == deprecated.length()) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1201
                                deprecatedPrefix = true;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1202
                                break;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1203
                            }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1204
                        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1205
                    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1206
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1207
                    if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1208
                        if (Character.isWhitespace(comment_reader.ch)) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1209
                            deprecatedFlag = true;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1210
                        } else if (comment_reader.ch == '*') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1211
                            comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1212
                            if (comment_reader.ch == '/') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1213
                                deprecatedFlag = true;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1214
                                return;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1215
                            }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1216
                        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1217
                    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1218
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1219
                    // Skip rest of line
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1220
                    while (comment_reader.bp < comment_reader.buflen) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1221
                        switch (comment_reader.ch) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1222
                            case '*':
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1223
                                comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1224
                                if (comment_reader.ch == '/') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1225
                                    return;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1226
                                }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1227
                                break;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1228
                            case CR: // (Spec 3.4)
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1229
                                comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1230
                                if (comment_reader.ch != LF) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1231
                                    continue forEachLine;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1232
                                }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1233
                            /* fall through to LF case */
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1234
                            case LF: // (Spec 3.4)
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1235
                                comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1236
                                continue forEachLine;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1237
                            default:
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1238
                                comment_reader.scanCommentChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1239
                        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1240
                    } // rest of line
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1241
                } // forEachLine
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1242
                return;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1243
            } finally {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1244
                scanned = true;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1245
            }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1246
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
  1247
    }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
  1248
}