langtools/src/share/classes/com/sun/tools/javac/parser/Scanner.java
author jjg
Thu, 18 Nov 2010 16:13:11 -0800
changeset 7330 7c670eebe55c
parent 6716 71df48777dd1
child 8032 e1aa25ccdabb
permissions -rw-r--r--
6999438: remove support for exotic identifiers from JDK 7 Reviewed-by: mcimadamore
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10
06bc494ca11e Initial load
duke
parents:
diff changeset
     1
/*
6716
71df48777dd1 6877202: Elements.getDocComment() is not getting JavaDocComments
jjg
parents: 6592
diff changeset
     2
 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
10
06bc494ca11e Initial load
duke
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
06bc494ca11e Initial load
duke
parents:
diff changeset
     4
 *
06bc494ca11e Initial load
duke
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
06bc494ca11e Initial load
duke
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
5520
86e4b9a9da40 6943119: Rebrand source copyright notices
ohair
parents: 4072
diff changeset
     7
 * published by the Free Software Foundation.  Oracle designates this
10
06bc494ca11e Initial load
duke
parents:
diff changeset
     8
 * particular file as subject to the "Classpath" exception as provided
5520
86e4b9a9da40 6943119: Rebrand source copyright notices
ohair
parents: 4072
diff changeset
     9
 * by Oracle in the LICENSE file that accompanied this code.
10
06bc494ca11e Initial load
duke
parents:
diff changeset
    10
 *
06bc494ca11e Initial load
duke
parents:
diff changeset
    11
 * This code is distributed in the hope that it will be useful, but WITHOUT
06bc494ca11e Initial load
duke
parents:
diff changeset
    12
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
06bc494ca11e Initial load
duke
parents:
diff changeset
    13
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
06bc494ca11e Initial load
duke
parents:
diff changeset
    14
 * version 2 for more details (a copy is included in the LICENSE file that
06bc494ca11e Initial load
duke
parents:
diff changeset
    15
 * accompanied this code).
06bc494ca11e Initial load
duke
parents:
diff changeset
    16
 *
06bc494ca11e Initial load
duke
parents:
diff changeset
    17
 * You should have received a copy of the GNU General Public License version
06bc494ca11e Initial load
duke
parents:
diff changeset
    18
 * 2 along with this work; if not, write to the Free Software Foundation,
06bc494ca11e Initial load
duke
parents:
diff changeset
    19
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
06bc494ca11e Initial load
duke
parents:
diff changeset
    20
 *
5520
86e4b9a9da40 6943119: Rebrand source copyright notices
ohair
parents: 4072
diff changeset
    21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
86e4b9a9da40 6943119: Rebrand source copyright notices
ohair
parents: 4072
diff changeset
    22
 * or visit www.oracle.com if you need additional information or have any
86e4b9a9da40 6943119: Rebrand source copyright notices
ohair
parents: 4072
diff changeset
    23
 * questions.
10
06bc494ca11e Initial load
duke
parents:
diff changeset
    24
 */
06bc494ca11e Initial load
duke
parents:
diff changeset
    25
06bc494ca11e Initial load
duke
parents:
diff changeset
    26
package com.sun.tools.javac.parser;
06bc494ca11e Initial load
duke
parents:
diff changeset
    27
06bc494ca11e Initial load
duke
parents:
diff changeset
    28
import java.nio.*;
06bc494ca11e Initial load
duke
parents:
diff changeset
    29
731
1dd22bdb9ca5 6714364: refactor javac File handling code into new javac.file package
jjg
parents: 10
diff changeset
    30
import com.sun.tools.javac.code.Source;
1dd22bdb9ca5 6714364: refactor javac File handling code into new javac.file package
jjg
parents: 10
diff changeset
    31
import com.sun.tools.javac.file.JavacFileManager;
10
06bc494ca11e Initial load
duke
parents:
diff changeset
    32
import com.sun.tools.javac.util.*;
06bc494ca11e Initial load
duke
parents:
diff changeset
    33
06bc494ca11e Initial load
duke
parents:
diff changeset
    34
06bc494ca11e Initial load
duke
parents:
diff changeset
    35
import static com.sun.tools.javac.parser.Token.*;
06bc494ca11e Initial load
duke
parents:
diff changeset
    36
import static com.sun.tools.javac.util.LayoutCharacters.*;
06bc494ca11e Initial load
duke
parents:
diff changeset
    37
06bc494ca11e Initial load
duke
parents:
diff changeset
    38
/** The lexical analyzer maps an input stream consisting of
06bc494ca11e Initial load
duke
parents:
diff changeset
    39
 *  ASCII characters and Unicode escapes into a token sequence.
06bc494ca11e Initial load
duke
parents:
diff changeset
    40
 *
5847
1908176fd6e3 6944312: Potential rebranding issues in openjdk/langtools repository sources
jjg
parents: 5520
diff changeset
    41
 *  <p><b>This is NOT part of any supported API.
1908176fd6e3 6944312: Potential rebranding issues in openjdk/langtools repository sources
jjg
parents: 5520
diff changeset
    42
 *  If you write code that depends on this, you do so at your own risk.
10
06bc494ca11e Initial load
duke
parents:
diff changeset
    43
 *  This code and its internal interfaces are subject to change or
06bc494ca11e Initial load
duke
parents:
diff changeset
    44
 *  deletion without notice.</b>
06bc494ca11e Initial load
duke
parents:
diff changeset
    45
 */
06bc494ca11e Initial load
duke
parents:
diff changeset
    46
public class Scanner implements Lexer {
06bc494ca11e Initial load
duke
parents:
diff changeset
    47
06bc494ca11e Initial load
duke
parents:
diff changeset
    48
    private static boolean scannerDebug = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
    49
06bc494ca11e Initial load
duke
parents:
diff changeset
    50
    /* Output variables; set by nextToken():
06bc494ca11e Initial load
duke
parents:
diff changeset
    51
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    52
06bc494ca11e Initial load
duke
parents:
diff changeset
    53
    /** The token, set by nextToken().
06bc494ca11e Initial load
duke
parents:
diff changeset
    54
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    55
    private Token token;
06bc494ca11e Initial load
duke
parents:
diff changeset
    56
06bc494ca11e Initial load
duke
parents:
diff changeset
    57
    /** Allow hex floating-point literals.
06bc494ca11e Initial load
duke
parents:
diff changeset
    58
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    59
    private boolean allowHexFloats;
06bc494ca11e Initial load
duke
parents:
diff changeset
    60
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    61
    /** Allow binary literals.
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    62
     */
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    63
    private boolean allowBinaryLiterals;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    64
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    65
    /** Allow underscores in literals.
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    66
     */
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    67
    private boolean allowUnderscoresInLiterals;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    68
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    69
    /** The source language setting.
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    70
     */
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    71
    private Source source;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
    72
10
06bc494ca11e Initial load
duke
parents:
diff changeset
    73
    /** The token's position, 0-based offset from beginning of text.
06bc494ca11e Initial load
duke
parents:
diff changeset
    74
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    75
    private int pos;
06bc494ca11e Initial load
duke
parents:
diff changeset
    76
06bc494ca11e Initial load
duke
parents:
diff changeset
    77
    /** Character position just after the last character of the token.
06bc494ca11e Initial load
duke
parents:
diff changeset
    78
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    79
    private int endPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
    80
06bc494ca11e Initial load
duke
parents:
diff changeset
    81
    /** The last character position of the previous token.
06bc494ca11e Initial load
duke
parents:
diff changeset
    82
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    83
    private int prevEndPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
    84
06bc494ca11e Initial load
duke
parents:
diff changeset
    85
    /** The position where a lexical error occurred;
06bc494ca11e Initial load
duke
parents:
diff changeset
    86
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    87
    private int errPos = Position.NOPOS;
06bc494ca11e Initial load
duke
parents:
diff changeset
    88
06bc494ca11e Initial load
duke
parents:
diff changeset
    89
    /** The name of an identifier or token:
06bc494ca11e Initial load
duke
parents:
diff changeset
    90
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    91
    private Name name;
06bc494ca11e Initial load
duke
parents:
diff changeset
    92
06bc494ca11e Initial load
duke
parents:
diff changeset
    93
    /** The radix of a numeric literal token.
06bc494ca11e Initial load
duke
parents:
diff changeset
    94
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
    95
    private int radix;
06bc494ca11e Initial load
duke
parents:
diff changeset
    96
06bc494ca11e Initial load
duke
parents:
diff changeset
    97
    /** Has a @deprecated been encountered in last doc comment?
06bc494ca11e Initial load
duke
parents:
diff changeset
    98
     *  this needs to be reset by client.
06bc494ca11e Initial load
duke
parents:
diff changeset
    99
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   100
    protected boolean deprecatedFlag = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   101
06bc494ca11e Initial load
duke
parents:
diff changeset
   102
    /** A character buffer for literals.
06bc494ca11e Initial load
duke
parents:
diff changeset
   103
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   104
    private char[] sbuf = new char[128];
06bc494ca11e Initial load
duke
parents:
diff changeset
   105
    private int sp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   106
06bc494ca11e Initial load
duke
parents:
diff changeset
   107
    /** The input buffer, index of next chacter to be read,
06bc494ca11e Initial load
duke
parents:
diff changeset
   108
     *  index of one past last character in buffer.
06bc494ca11e Initial load
duke
parents:
diff changeset
   109
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   110
    private char[] buf;
06bc494ca11e Initial load
duke
parents:
diff changeset
   111
    private int bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   112
    private int buflen;
06bc494ca11e Initial load
duke
parents:
diff changeset
   113
    private int eofPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
   114
06bc494ca11e Initial load
duke
parents:
diff changeset
   115
    /** The current character.
06bc494ca11e Initial load
duke
parents:
diff changeset
   116
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   117
    private char ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   118
06bc494ca11e Initial load
duke
parents:
diff changeset
   119
    /** The buffer index of the last converted unicode character
06bc494ca11e Initial load
duke
parents:
diff changeset
   120
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   121
    private int unicodeConversionBp = -1;
06bc494ca11e Initial load
duke
parents:
diff changeset
   122
06bc494ca11e Initial load
duke
parents:
diff changeset
   123
    /** The log to be used for error reporting.
06bc494ca11e Initial load
duke
parents:
diff changeset
   124
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   125
    private final Log log;
06bc494ca11e Initial load
duke
parents:
diff changeset
   126
06bc494ca11e Initial load
duke
parents:
diff changeset
   127
    /** The name table. */
1260
a772ba9ba43d 6574134: Allow for alternative implementation of Name Table with garbage collection of name bytes
jjg
parents: 735
diff changeset
   128
    private final Names names;
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   129
06bc494ca11e Initial load
duke
parents:
diff changeset
   130
    /** The keyword table. */
06bc494ca11e Initial load
duke
parents:
diff changeset
   131
    private final Keywords keywords;
06bc494ca11e Initial load
duke
parents:
diff changeset
   132
06bc494ca11e Initial load
duke
parents:
diff changeset
   133
    /** Common code for constructors. */
6716
71df48777dd1 6877202: Elements.getDocComment() is not getting JavaDocComments
jjg
parents: 6592
diff changeset
   134
    private Scanner(ScannerFactory fac) {
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   135
        log = fac.log;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   136
        names = fac.names;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   137
        keywords = fac.keywords;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   138
        source = fac.source;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   139
        allowBinaryLiterals = source.allowBinaryLiterals();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   140
        allowHexFloats = source.allowHexFloats();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   141
        allowUnderscoresInLiterals = source.allowBinaryLiterals();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   142
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   143
06bc494ca11e Initial load
duke
parents:
diff changeset
   144
    private static final boolean hexFloatsWork = hexFloatsWork();
06bc494ca11e Initial load
duke
parents:
diff changeset
   145
    private static boolean hexFloatsWork() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   146
        try {
06bc494ca11e Initial load
duke
parents:
diff changeset
   147
            Float.valueOf("0x1.0p1");
06bc494ca11e Initial load
duke
parents:
diff changeset
   148
            return true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   149
        } catch (NumberFormatException ex) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   150
            return false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   151
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   152
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   153
06bc494ca11e Initial load
duke
parents:
diff changeset
   154
    /** Create a scanner from the input buffer.  buffer must implement
06bc494ca11e Initial load
duke
parents:
diff changeset
   155
     *  array() and compact(), and remaining() must be less than limit().
06bc494ca11e Initial load
duke
parents:
diff changeset
   156
     */
6716
71df48777dd1 6877202: Elements.getDocComment() is not getting JavaDocComments
jjg
parents: 6592
diff changeset
   157
    protected Scanner(ScannerFactory fac, CharBuffer buffer) {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   158
        this(fac, JavacFileManager.toArray(buffer), buffer.limit());
06bc494ca11e Initial load
duke
parents:
diff changeset
   159
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   160
06bc494ca11e Initial load
duke
parents:
diff changeset
   161
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
   162
     * Create a scanner from the input array.  This method might
06bc494ca11e Initial load
duke
parents:
diff changeset
   163
     * modify the array.  To avoid copying the input array, ensure
06bc494ca11e Initial load
duke
parents:
diff changeset
   164
     * that {@code inputLength < input.length} or
06bc494ca11e Initial load
duke
parents:
diff changeset
   165
     * {@code input[input.length -1]} is a white space character.
06bc494ca11e Initial load
duke
parents:
diff changeset
   166
     *
06bc494ca11e Initial load
duke
parents:
diff changeset
   167
     * @param fac the factory which created this Scanner
06bc494ca11e Initial load
duke
parents:
diff changeset
   168
     * @param input the input, might be modified
06bc494ca11e Initial load
duke
parents:
diff changeset
   169
     * @param inputLength the size of the input.
06bc494ca11e Initial load
duke
parents:
diff changeset
   170
     * Must be positive and less than or equal to input.length.
06bc494ca11e Initial load
duke
parents:
diff changeset
   171
     */
6716
71df48777dd1 6877202: Elements.getDocComment() is not getting JavaDocComments
jjg
parents: 6592
diff changeset
   172
    protected Scanner(ScannerFactory fac, char[] input, int inputLength) {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   173
        this(fac);
06bc494ca11e Initial load
duke
parents:
diff changeset
   174
        eofPos = inputLength;
06bc494ca11e Initial load
duke
parents:
diff changeset
   175
        if (inputLength == input.length) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   176
            if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   177
                inputLength--;
06bc494ca11e Initial load
duke
parents:
diff changeset
   178
            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   179
                char[] newInput = new char[inputLength + 1];
06bc494ca11e Initial load
duke
parents:
diff changeset
   180
                System.arraycopy(input, 0, newInput, 0, input.length);
06bc494ca11e Initial load
duke
parents:
diff changeset
   181
                input = newInput;
06bc494ca11e Initial load
duke
parents:
diff changeset
   182
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   183
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   184
        buf = input;
06bc494ca11e Initial load
duke
parents:
diff changeset
   185
        buflen = inputLength;
06bc494ca11e Initial load
duke
parents:
diff changeset
   186
        buf[buflen] = EOI;
06bc494ca11e Initial load
duke
parents:
diff changeset
   187
        bp = -1;
06bc494ca11e Initial load
duke
parents:
diff changeset
   188
        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   189
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   190
06bc494ca11e Initial load
duke
parents:
diff changeset
   191
    /** Report an error at the given position using the provided arguments.
06bc494ca11e Initial load
duke
parents:
diff changeset
   192
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   193
    private void lexError(int pos, String key, Object... args) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   194
        log.error(pos, key, args);
06bc494ca11e Initial load
duke
parents:
diff changeset
   195
        token = ERROR;
06bc494ca11e Initial load
duke
parents:
diff changeset
   196
        errPos = pos;
06bc494ca11e Initial load
duke
parents:
diff changeset
   197
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   198
06bc494ca11e Initial load
duke
parents:
diff changeset
   199
    /** Report an error at the current token position using the provided
06bc494ca11e Initial load
duke
parents:
diff changeset
   200
     *  arguments.
06bc494ca11e Initial load
duke
parents:
diff changeset
   201
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   202
    private void lexError(String key, Object... args) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   203
        lexError(pos, key, args);
06bc494ca11e Initial load
duke
parents:
diff changeset
   204
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   205
06bc494ca11e Initial load
duke
parents:
diff changeset
   206
    /** Convert an ASCII digit from its base (8, 10, or 16)
06bc494ca11e Initial load
duke
parents:
diff changeset
   207
     *  to its value.
06bc494ca11e Initial load
duke
parents:
diff changeset
   208
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   209
    private int digit(int base) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   210
        char c = ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   211
        int result = Character.digit(c, base);
06bc494ca11e Initial load
duke
parents:
diff changeset
   212
        if (result >= 0 && c > 0x7f) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   213
            lexError(pos+1, "illegal.nonascii.digit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   214
            ch = "0123456789abcdef".charAt(result);
06bc494ca11e Initial load
duke
parents:
diff changeset
   215
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   216
        return result;
06bc494ca11e Initial load
duke
parents:
diff changeset
   217
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   218
06bc494ca11e Initial load
duke
parents:
diff changeset
   219
    /** Convert unicode escape; bp points to initial '\' character
06bc494ca11e Initial load
duke
parents:
diff changeset
   220
     *  (Spec 3.3).
06bc494ca11e Initial load
duke
parents:
diff changeset
   221
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   222
    private void convertUnicode() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   223
        if (ch == '\\' && unicodeConversionBp != bp) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   224
            bp++; ch = buf[bp];
06bc494ca11e Initial load
duke
parents:
diff changeset
   225
            if (ch == 'u') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   226
                do {
06bc494ca11e Initial load
duke
parents:
diff changeset
   227
                    bp++; ch = buf[bp];
06bc494ca11e Initial load
duke
parents:
diff changeset
   228
                } while (ch == 'u');
06bc494ca11e Initial load
duke
parents:
diff changeset
   229
                int limit = bp + 3;
06bc494ca11e Initial load
duke
parents:
diff changeset
   230
                if (limit < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   231
                    int d = digit(16);
06bc494ca11e Initial load
duke
parents:
diff changeset
   232
                    int code = d;
06bc494ca11e Initial load
duke
parents:
diff changeset
   233
                    while (bp < limit && d >= 0) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   234
                        bp++; ch = buf[bp];
06bc494ca11e Initial load
duke
parents:
diff changeset
   235
                        d = digit(16);
06bc494ca11e Initial load
duke
parents:
diff changeset
   236
                        code = (code << 4) + d;
06bc494ca11e Initial load
duke
parents:
diff changeset
   237
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   238
                    if (d >= 0) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   239
                        ch = (char)code;
06bc494ca11e Initial load
duke
parents:
diff changeset
   240
                        unicodeConversionBp = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   241
                        return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   242
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   243
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   244
                lexError(bp, "illegal.unicode.esc");
06bc494ca11e Initial load
duke
parents:
diff changeset
   245
            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   246
                bp--;
06bc494ca11e Initial load
duke
parents:
diff changeset
   247
                ch = '\\';
06bc494ca11e Initial load
duke
parents:
diff changeset
   248
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   249
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   250
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   251
06bc494ca11e Initial load
duke
parents:
diff changeset
   252
    /** Read next character.
06bc494ca11e Initial load
duke
parents:
diff changeset
   253
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   254
    private void scanChar() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   255
        ch = buf[++bp];
06bc494ca11e Initial load
duke
parents:
diff changeset
   256
        if (ch == '\\') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   257
            convertUnicode();
06bc494ca11e Initial load
duke
parents:
diff changeset
   258
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   259
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   260
06bc494ca11e Initial load
duke
parents:
diff changeset
   261
    /** Read next character in comment, skipping over double '\' characters.
06bc494ca11e Initial load
duke
parents:
diff changeset
   262
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   263
    private void scanCommentChar() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   264
        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   265
        if (ch == '\\') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   266
            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   267
                bp++;
06bc494ca11e Initial load
duke
parents:
diff changeset
   268
            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   269
                convertUnicode();
06bc494ca11e Initial load
duke
parents:
diff changeset
   270
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   271
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   272
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   273
06bc494ca11e Initial load
duke
parents:
diff changeset
   274
    /** Append a character to sbuf.
06bc494ca11e Initial load
duke
parents:
diff changeset
   275
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   276
    private void putChar(char ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   277
        if (sp == sbuf.length) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   278
            char[] newsbuf = new char[sbuf.length * 2];
06bc494ca11e Initial load
duke
parents:
diff changeset
   279
            System.arraycopy(sbuf, 0, newsbuf, 0, sbuf.length);
06bc494ca11e Initial load
duke
parents:
diff changeset
   280
            sbuf = newsbuf;
06bc494ca11e Initial load
duke
parents:
diff changeset
   281
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   282
        sbuf[sp++] = ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   283
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   284
06bc494ca11e Initial load
duke
parents:
diff changeset
   285
    /** For debugging purposes: print character.
06bc494ca11e Initial load
duke
parents:
diff changeset
   286
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   287
    private void dch() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   288
        System.err.print(ch); System.out.flush();
06bc494ca11e Initial load
duke
parents:
diff changeset
   289
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   290
06bc494ca11e Initial load
duke
parents:
diff changeset
   291
    /** Read next character in character or string literal and copy into sbuf.
06bc494ca11e Initial load
duke
parents:
diff changeset
   292
     */
7330
7c670eebe55c 6999438: remove support for exotic identifiers from JDK 7
jjg
parents: 6716
diff changeset
   293
    private void scanLitChar() {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   294
        if (ch == '\\') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   295
            if (buf[bp+1] == '\\' && unicodeConversionBp != bp) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   296
                bp++;
06bc494ca11e Initial load
duke
parents:
diff changeset
   297
                putChar('\\');
06bc494ca11e Initial load
duke
parents:
diff changeset
   298
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   299
            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   300
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   301
                switch (ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   302
                case '0': case '1': case '2': case '3':
06bc494ca11e Initial load
duke
parents:
diff changeset
   303
                case '4': case '5': case '6': case '7':
06bc494ca11e Initial load
duke
parents:
diff changeset
   304
                    char leadch = ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   305
                    int oct = digit(8);
06bc494ca11e Initial load
duke
parents:
diff changeset
   306
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   307
                    if ('0' <= ch && ch <= '7') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   308
                        oct = oct * 8 + digit(8);
06bc494ca11e Initial load
duke
parents:
diff changeset
   309
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   310
                        if (leadch <= '3' && '0' <= ch && ch <= '7') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   311
                            oct = oct * 8 + digit(8);
06bc494ca11e Initial load
duke
parents:
diff changeset
   312
                            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   313
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   314
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   315
                    putChar((char)oct);
06bc494ca11e Initial load
duke
parents:
diff changeset
   316
                    break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   317
                case 'b':
06bc494ca11e Initial load
duke
parents:
diff changeset
   318
                    putChar('\b'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   319
                case 't':
06bc494ca11e Initial load
duke
parents:
diff changeset
   320
                    putChar('\t'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   321
                case 'n':
06bc494ca11e Initial load
duke
parents:
diff changeset
   322
                    putChar('\n'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   323
                case 'f':
06bc494ca11e Initial load
duke
parents:
diff changeset
   324
                    putChar('\f'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   325
                case 'r':
06bc494ca11e Initial load
duke
parents:
diff changeset
   326
                    putChar('\r'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   327
                case '\'':
06bc494ca11e Initial load
duke
parents:
diff changeset
   328
                    putChar('\''); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   329
                case '\"':
06bc494ca11e Initial load
duke
parents:
diff changeset
   330
                    putChar('\"'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   331
                case '\\':
06bc494ca11e Initial load
duke
parents:
diff changeset
   332
                    putChar('\\'); scanChar(); break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   333
                default:
06bc494ca11e Initial load
duke
parents:
diff changeset
   334
                    lexError(bp, "illegal.esc.char");
06bc494ca11e Initial load
duke
parents:
diff changeset
   335
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   336
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   337
        } else if (bp != buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   338
            putChar(ch); scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   339
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   340
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   341
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   342
    private void scanDigits(int digitRadix) {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   343
        char saveCh;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   344
        int savePos;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   345
        do {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   346
            if (ch != '_') {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   347
                putChar(ch);
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   348
            } else {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   349
                if (!allowUnderscoresInLiterals) {
6031
50004868a787 6964768: need test program to validate javac resource bundles
jjg
parents: 5847
diff changeset
   350
                    lexError("unsupported.underscore.lit", source.name);
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   351
                    allowUnderscoresInLiterals = true;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   352
                }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   353
            }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   354
            saveCh = ch;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   355
            savePos = bp;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   356
            scanChar();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   357
        } while (digit(digitRadix) >= 0 || ch == '_');
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   358
        if (saveCh == '_')
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   359
            lexError(savePos, "illegal.underscore");
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   360
    }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   361
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   362
    /** Read fractional part of hexadecimal floating point number.
06bc494ca11e Initial load
duke
parents:
diff changeset
   363
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   364
    private void scanHexExponentAndSuffix() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   365
        if (ch == 'p' || ch == 'P') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   366
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   367
            scanChar();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   368
            skipIllegalUnderscores();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   369
            if (ch == '+' || ch == '-') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   370
                putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   371
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   372
            }
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   373
            skipIllegalUnderscores();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   374
            if ('0' <= ch && ch <= '9') {
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   375
                scanDigits(10);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   376
                if (!allowHexFloats) {
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   377
                    lexError("unsupported.fp.lit", source.name);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   378
                    allowHexFloats = true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   379
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   380
                else if (!hexFloatsWork)
06bc494ca11e Initial load
duke
parents:
diff changeset
   381
                    lexError("unsupported.cross.fp.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   382
            } else
06bc494ca11e Initial load
duke
parents:
diff changeset
   383
                lexError("malformed.fp.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   384
        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   385
            lexError("malformed.fp.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   386
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   387
        if (ch == 'f' || ch == 'F') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   388
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   389
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   390
            token = FLOATLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   391
        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   392
            if (ch == 'd' || ch == 'D') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   393
                putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   394
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   395
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   396
            token = DOUBLELITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   397
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   398
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   399
06bc494ca11e Initial load
duke
parents:
diff changeset
   400
    /** Read fractional part of floating point number.
06bc494ca11e Initial load
duke
parents:
diff changeset
   401
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   402
    private void scanFraction() {
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   403
        skipIllegalUnderscores();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   404
        if ('0' <= ch && ch <= '9') {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   405
            scanDigits(10);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   406
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   407
        int sp1 = sp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   408
        if (ch == 'e' || ch == 'E') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   409
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   410
            scanChar();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   411
            skipIllegalUnderscores();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   412
            if (ch == '+' || ch == '-') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   413
                putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   414
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   415
            }
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   416
            skipIllegalUnderscores();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   417
            if ('0' <= ch && ch <= '9') {
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   418
                scanDigits(10);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   419
                return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   420
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   421
            lexError("malformed.fp.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   422
            sp = sp1;
06bc494ca11e Initial load
duke
parents:
diff changeset
   423
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   424
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   425
06bc494ca11e Initial load
duke
parents:
diff changeset
   426
    /** Read fractional part and 'd' or 'f' suffix of floating point number.
06bc494ca11e Initial load
duke
parents:
diff changeset
   427
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   428
    private void scanFractionAndSuffix() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   429
        this.radix = 10;
06bc494ca11e Initial load
duke
parents:
diff changeset
   430
        scanFraction();
06bc494ca11e Initial load
duke
parents:
diff changeset
   431
        if (ch == 'f' || ch == 'F') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   432
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   433
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   434
            token = FLOATLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   435
        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   436
            if (ch == 'd' || ch == 'D') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   437
                putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   438
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   439
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   440
            token = DOUBLELITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   441
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   442
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   443
06bc494ca11e Initial load
duke
parents:
diff changeset
   444
    /** Read fractional part and 'd' or 'f' suffix of floating point number.
06bc494ca11e Initial load
duke
parents:
diff changeset
   445
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   446
    private void scanHexFractionAndSuffix(boolean seendigit) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   447
        this.radix = 16;
06bc494ca11e Initial load
duke
parents:
diff changeset
   448
        assert ch == '.';
06bc494ca11e Initial load
duke
parents:
diff changeset
   449
        putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   450
        scanChar();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   451
        skipIllegalUnderscores();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   452
        if (digit(16) >= 0) {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   453
            seendigit = true;
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   454
            scanDigits(16);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   455
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   456
        if (!seendigit)
06bc494ca11e Initial load
duke
parents:
diff changeset
   457
            lexError("invalid.hex.number");
06bc494ca11e Initial load
duke
parents:
diff changeset
   458
        else
06bc494ca11e Initial load
duke
parents:
diff changeset
   459
            scanHexExponentAndSuffix();
06bc494ca11e Initial load
duke
parents:
diff changeset
   460
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   461
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   462
    private void skipIllegalUnderscores() {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   463
        if (ch == '_') {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   464
            lexError(bp, "illegal.underscore");
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   465
            while (ch == '_')
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   466
                scanChar();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   467
        }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   468
    }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   469
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   470
    /** Read a number.
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   471
     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   472
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   473
    private void scanNumber(int radix) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   474
        this.radix = radix;
06bc494ca11e Initial load
duke
parents:
diff changeset
   475
        // for octal, allow base-10 digit in case it's a float literal
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   476
        int digitRadix = (radix == 8 ? 10 : radix);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   477
        boolean seendigit = false;
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   478
        if (digit(digitRadix) >= 0) {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   479
            seendigit = true;
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   480
            scanDigits(digitRadix);
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   481
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   482
        if (radix == 16 && ch == '.') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   483
            scanHexFractionAndSuffix(seendigit);
06bc494ca11e Initial load
duke
parents:
diff changeset
   484
        } else if (seendigit && radix == 16 && (ch == 'p' || ch == 'P')) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   485
            scanHexExponentAndSuffix();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   486
        } else if (digitRadix == 10 && ch == '.') {
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   487
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   488
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   489
            scanFractionAndSuffix();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   490
        } else if (digitRadix == 10 &&
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   491
                   (ch == 'e' || ch == 'E' ||
06bc494ca11e Initial load
duke
parents:
diff changeset
   492
                    ch == 'f' || ch == 'F' ||
06bc494ca11e Initial load
duke
parents:
diff changeset
   493
                    ch == 'd' || ch == 'D')) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   494
            scanFractionAndSuffix();
06bc494ca11e Initial load
duke
parents:
diff changeset
   495
        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   496
            if (ch == 'l' || ch == 'L') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   497
                scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   498
                token = LONGLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   499
            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   500
                token = INTLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   501
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   502
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   503
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   504
06bc494ca11e Initial load
duke
parents:
diff changeset
   505
    /** Read an identifier.
06bc494ca11e Initial load
duke
parents:
diff changeset
   506
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   507
    private void scanIdent() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   508
        boolean isJavaIdentifierPart;
06bc494ca11e Initial load
duke
parents:
diff changeset
   509
        char high;
06bc494ca11e Initial load
duke
parents:
diff changeset
   510
        do {
06bc494ca11e Initial load
duke
parents:
diff changeset
   511
            if (sp == sbuf.length) putChar(ch); else sbuf[sp++] = ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   512
            // optimization, was: putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   513
06bc494ca11e Initial load
duke
parents:
diff changeset
   514
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   515
            switch (ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   516
            case 'A': case 'B': case 'C': case 'D': case 'E':
06bc494ca11e Initial load
duke
parents:
diff changeset
   517
            case 'F': case 'G': case 'H': case 'I': case 'J':
06bc494ca11e Initial load
duke
parents:
diff changeset
   518
            case 'K': case 'L': case 'M': case 'N': case 'O':
06bc494ca11e Initial load
duke
parents:
diff changeset
   519
            case 'P': case 'Q': case 'R': case 'S': case 'T':
06bc494ca11e Initial load
duke
parents:
diff changeset
   520
            case 'U': case 'V': case 'W': case 'X': case 'Y':
06bc494ca11e Initial load
duke
parents:
diff changeset
   521
            case 'Z':
06bc494ca11e Initial load
duke
parents:
diff changeset
   522
            case 'a': case 'b': case 'c': case 'd': case 'e':
06bc494ca11e Initial load
duke
parents:
diff changeset
   523
            case 'f': case 'g': case 'h': case 'i': case 'j':
06bc494ca11e Initial load
duke
parents:
diff changeset
   524
            case 'k': case 'l': case 'm': case 'n': case 'o':
06bc494ca11e Initial load
duke
parents:
diff changeset
   525
            case 'p': case 'q': case 'r': case 's': case 't':
06bc494ca11e Initial load
duke
parents:
diff changeset
   526
            case 'u': case 'v': case 'w': case 'x': case 'y':
06bc494ca11e Initial load
duke
parents:
diff changeset
   527
            case 'z':
06bc494ca11e Initial load
duke
parents:
diff changeset
   528
            case '$': case '_':
06bc494ca11e Initial load
duke
parents:
diff changeset
   529
            case '0': case '1': case '2': case '3': case '4':
06bc494ca11e Initial load
duke
parents:
diff changeset
   530
            case '5': case '6': case '7': case '8': case '9':
06bc494ca11e Initial load
duke
parents:
diff changeset
   531
            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
06bc494ca11e Initial load
duke
parents:
diff changeset
   532
            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
06bc494ca11e Initial load
duke
parents:
diff changeset
   533
            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
06bc494ca11e Initial load
duke
parents:
diff changeset
   534
            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
06bc494ca11e Initial load
duke
parents:
diff changeset
   535
            case '\u0015': case '\u0016': case '\u0017':
06bc494ca11e Initial load
duke
parents:
diff changeset
   536
            case '\u0018': case '\u0019': case '\u001B':
06bc494ca11e Initial load
duke
parents:
diff changeset
   537
            case '\u007F':
06bc494ca11e Initial load
duke
parents:
diff changeset
   538
                break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   539
            case '\u001A': // EOI is also a legal identifier part
06bc494ca11e Initial load
duke
parents:
diff changeset
   540
                if (bp >= buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   541
                    name = names.fromChars(sbuf, 0, sp);
06bc494ca11e Initial load
duke
parents:
diff changeset
   542
                    token = keywords.key(name);
06bc494ca11e Initial load
duke
parents:
diff changeset
   543
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   544
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   545
                break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   546
            default:
06bc494ca11e Initial load
duke
parents:
diff changeset
   547
                if (ch < '\u0080') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   548
                    // all ASCII range chars already handled, above
06bc494ca11e Initial load
duke
parents:
diff changeset
   549
                    isJavaIdentifierPart = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   550
                } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   551
                    high = scanSurrogates();
06bc494ca11e Initial load
duke
parents:
diff changeset
   552
                    if (high != 0) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   553
                        if (sp == sbuf.length) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   554
                            putChar(high);
06bc494ca11e Initial load
duke
parents:
diff changeset
   555
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   556
                            sbuf[sp++] = high;
06bc494ca11e Initial load
duke
parents:
diff changeset
   557
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   558
                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
06bc494ca11e Initial load
duke
parents:
diff changeset
   559
                            Character.toCodePoint(high, ch));
06bc494ca11e Initial load
duke
parents:
diff changeset
   560
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   561
                        isJavaIdentifierPart = Character.isJavaIdentifierPart(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   562
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   563
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   564
                if (!isJavaIdentifierPart) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   565
                    name = names.fromChars(sbuf, 0, sp);
06bc494ca11e Initial load
duke
parents:
diff changeset
   566
                    token = keywords.key(name);
06bc494ca11e Initial load
duke
parents:
diff changeset
   567
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   568
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   569
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   570
        } while (true);
06bc494ca11e Initial load
duke
parents:
diff changeset
   571
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   572
06bc494ca11e Initial load
duke
parents:
diff changeset
   573
    /** Are surrogates supported?
06bc494ca11e Initial load
duke
parents:
diff changeset
   574
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   575
    final static boolean surrogatesSupported = surrogatesSupported();
06bc494ca11e Initial load
duke
parents:
diff changeset
   576
    private static boolean surrogatesSupported() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   577
        try {
06bc494ca11e Initial load
duke
parents:
diff changeset
   578
            Character.isHighSurrogate('a');
06bc494ca11e Initial load
duke
parents:
diff changeset
   579
            return true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   580
        } catch (NoSuchMethodError ex) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   581
            return false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   582
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   583
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   584
06bc494ca11e Initial load
duke
parents:
diff changeset
   585
    /** Scan surrogate pairs.  If 'ch' is a high surrogate and
06bc494ca11e Initial load
duke
parents:
diff changeset
   586
     *  the next character is a low surrogate, then put the low
06bc494ca11e Initial load
duke
parents:
diff changeset
   587
     *  surrogate in 'ch', and return the high surrogate.
06bc494ca11e Initial load
duke
parents:
diff changeset
   588
     *  otherwise, just return 0.
06bc494ca11e Initial load
duke
parents:
diff changeset
   589
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   590
    private char scanSurrogates() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   591
        if (surrogatesSupported && Character.isHighSurrogate(ch)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   592
            char high = ch;
06bc494ca11e Initial load
duke
parents:
diff changeset
   593
06bc494ca11e Initial load
duke
parents:
diff changeset
   594
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   595
06bc494ca11e Initial load
duke
parents:
diff changeset
   596
            if (Character.isLowSurrogate(ch)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   597
                return high;
06bc494ca11e Initial load
duke
parents:
diff changeset
   598
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   599
06bc494ca11e Initial load
duke
parents:
diff changeset
   600
            ch = high;
06bc494ca11e Initial load
duke
parents:
diff changeset
   601
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   602
06bc494ca11e Initial load
duke
parents:
diff changeset
   603
        return 0;
06bc494ca11e Initial load
duke
parents:
diff changeset
   604
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   605
06bc494ca11e Initial load
duke
parents:
diff changeset
   606
    /** Return true if ch can be part of an operator.
06bc494ca11e Initial load
duke
parents:
diff changeset
   607
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   608
    private boolean isSpecial(char ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   609
        switch (ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   610
        case '!': case '%': case '&': case '*': case '?':
06bc494ca11e Initial load
duke
parents:
diff changeset
   611
        case '+': case '-': case ':': case '<': case '=':
06bc494ca11e Initial load
duke
parents:
diff changeset
   612
        case '>': case '^': case '|': case '~':
06bc494ca11e Initial load
duke
parents:
diff changeset
   613
        case '@':
06bc494ca11e Initial load
duke
parents:
diff changeset
   614
            return true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   615
        default:
06bc494ca11e Initial load
duke
parents:
diff changeset
   616
            return false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   617
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   618
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   619
06bc494ca11e Initial load
duke
parents:
diff changeset
   620
    /** Read longest possible sequence of special characters and convert
06bc494ca11e Initial load
duke
parents:
diff changeset
   621
     *  to token.
06bc494ca11e Initial load
duke
parents:
diff changeset
   622
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   623
    private void scanOperator() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   624
        while (true) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   625
            putChar(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   626
            Name newname = names.fromChars(sbuf, 0, sp);
06bc494ca11e Initial load
duke
parents:
diff changeset
   627
            if (keywords.key(newname) == IDENTIFIER) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   628
                sp--;
06bc494ca11e Initial load
duke
parents:
diff changeset
   629
                break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   630
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   631
            name = newname;
06bc494ca11e Initial load
duke
parents:
diff changeset
   632
            token = keywords.key(newname);
06bc494ca11e Initial load
duke
parents:
diff changeset
   633
            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   634
            if (!isSpecial(ch)) break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   635
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   636
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   637
06bc494ca11e Initial load
duke
parents:
diff changeset
   638
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
   639
     * Scan a documention comment; determine if a deprecated tag is present.
06bc494ca11e Initial load
duke
parents:
diff changeset
   640
     * Called once the initial /, * have been skipped, positioned at the second *
06bc494ca11e Initial load
duke
parents:
diff changeset
   641
     * (which is treated as the beginning of the first line).
06bc494ca11e Initial load
duke
parents:
diff changeset
   642
     * Stops positioned at the closing '/'.
06bc494ca11e Initial load
duke
parents:
diff changeset
   643
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   644
    @SuppressWarnings("fallthrough")
06bc494ca11e Initial load
duke
parents:
diff changeset
   645
    private void scanDocComment() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   646
        boolean deprecatedPrefix = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   647
06bc494ca11e Initial load
duke
parents:
diff changeset
   648
        forEachLine:
06bc494ca11e Initial load
duke
parents:
diff changeset
   649
        while (bp < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   650
06bc494ca11e Initial load
duke
parents:
diff changeset
   651
            // Skip optional WhiteSpace at beginning of line
06bc494ca11e Initial load
duke
parents:
diff changeset
   652
            while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   653
                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   654
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   655
06bc494ca11e Initial load
duke
parents:
diff changeset
   656
            // Skip optional consecutive Stars
06bc494ca11e Initial load
duke
parents:
diff changeset
   657
            while (bp < buflen && ch == '*') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   658
                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   659
                if (ch == '/') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   660
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   661
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   662
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   663
06bc494ca11e Initial load
duke
parents:
diff changeset
   664
            // Skip optional WhiteSpace after Stars
06bc494ca11e Initial load
duke
parents:
diff changeset
   665
            while (bp < buflen && (ch == ' ' || ch == '\t' || ch == FF)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   666
                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   667
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   668
06bc494ca11e Initial load
duke
parents:
diff changeset
   669
            deprecatedPrefix = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   670
            // At beginning of line in the JavaDoc sense.
06bc494ca11e Initial load
duke
parents:
diff changeset
   671
            if (bp < buflen && ch == '@' && !deprecatedFlag) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   672
                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   673
                if (bp < buflen && ch == 'd') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   674
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   675
                    if (bp < buflen && ch == 'e') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   676
                        scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   677
                        if (bp < buflen && ch == 'p') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   678
                            scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   679
                            if (bp < buflen && ch == 'r') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   680
                                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   681
                                if (bp < buflen && ch == 'e') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   682
                                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   683
                                    if (bp < buflen && ch == 'c') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   684
                                        scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   685
                                        if (bp < buflen && ch == 'a') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   686
                                            scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   687
                                            if (bp < buflen && ch == 't') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   688
                                                scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   689
                                                if (bp < buflen && ch == 'e') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   690
                                                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   691
                                                    if (bp < buflen && ch == 'd') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   692
                                                        deprecatedPrefix = true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   693
                                                        scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   694
                                                    }}}}}}}}}}}
06bc494ca11e Initial load
duke
parents:
diff changeset
   695
            if (deprecatedPrefix && bp < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   696
                if (Character.isWhitespace(ch)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   697
                    deprecatedFlag = true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   698
                } else if (ch == '*') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   699
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   700
                    if (ch == '/') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   701
                        deprecatedFlag = true;
06bc494ca11e Initial load
duke
parents:
diff changeset
   702
                        return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   703
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   704
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   705
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   706
06bc494ca11e Initial load
duke
parents:
diff changeset
   707
            // Skip rest of line
06bc494ca11e Initial load
duke
parents:
diff changeset
   708
            while (bp < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   709
                switch (ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   710
                case '*':
06bc494ca11e Initial load
duke
parents:
diff changeset
   711
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   712
                    if (ch == '/') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   713
                        return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   714
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   715
                    break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   716
                case CR: // (Spec 3.4)
06bc494ca11e Initial load
duke
parents:
diff changeset
   717
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   718
                    if (ch != LF) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   719
                        continue forEachLine;
06bc494ca11e Initial load
duke
parents:
diff changeset
   720
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   721
                    /* fall through to LF case */
06bc494ca11e Initial load
duke
parents:
diff changeset
   722
                case LF: // (Spec 3.4)
06bc494ca11e Initial load
duke
parents:
diff changeset
   723
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   724
                    continue forEachLine;
06bc494ca11e Initial load
duke
parents:
diff changeset
   725
                default:
06bc494ca11e Initial load
duke
parents:
diff changeset
   726
                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   727
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   728
            } // rest of line
06bc494ca11e Initial load
duke
parents:
diff changeset
   729
        } // forEachLine
06bc494ca11e Initial load
duke
parents:
diff changeset
   730
        return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   731
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   732
06bc494ca11e Initial load
duke
parents:
diff changeset
   733
    /** The value of a literal token, recorded as a string.
06bc494ca11e Initial load
duke
parents:
diff changeset
   734
     *  For integers, leading 0x and 'l' suffixes are suppressed.
06bc494ca11e Initial load
duke
parents:
diff changeset
   735
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   736
    public String stringVal() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   737
        return new String(sbuf, 0, sp);
06bc494ca11e Initial load
duke
parents:
diff changeset
   738
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   739
06bc494ca11e Initial load
duke
parents:
diff changeset
   740
    /** Read token.
06bc494ca11e Initial load
duke
parents:
diff changeset
   741
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   742
    public void nextToken() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   743
06bc494ca11e Initial load
duke
parents:
diff changeset
   744
        try {
06bc494ca11e Initial load
duke
parents:
diff changeset
   745
            prevEndPos = endPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
   746
            sp = 0;
06bc494ca11e Initial load
duke
parents:
diff changeset
   747
06bc494ca11e Initial load
duke
parents:
diff changeset
   748
            while (true) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   749
                pos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   750
                switch (ch) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   751
                case ' ': // (Spec 3.6)
06bc494ca11e Initial load
duke
parents:
diff changeset
   752
                case '\t': // (Spec 3.6)
06bc494ca11e Initial load
duke
parents:
diff changeset
   753
                case FF: // (Spec 3.6)
06bc494ca11e Initial load
duke
parents:
diff changeset
   754
                    do {
06bc494ca11e Initial load
duke
parents:
diff changeset
   755
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   756
                    } while (ch == ' ' || ch == '\t' || ch == FF);
06bc494ca11e Initial load
duke
parents:
diff changeset
   757
                    endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   758
                    processWhiteSpace();
06bc494ca11e Initial load
duke
parents:
diff changeset
   759
                    break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   760
                case LF: // (Spec 3.4)
06bc494ca11e Initial load
duke
parents:
diff changeset
   761
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   762
                    endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   763
                    processLineTerminator();
06bc494ca11e Initial load
duke
parents:
diff changeset
   764
                    break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   765
                case CR: // (Spec 3.4)
06bc494ca11e Initial load
duke
parents:
diff changeset
   766
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   767
                    if (ch == LF) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   768
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   769
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   770
                    endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   771
                    processLineTerminator();
06bc494ca11e Initial load
duke
parents:
diff changeset
   772
                    break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   773
                case 'A': case 'B': case 'C': case 'D': case 'E':
06bc494ca11e Initial load
duke
parents:
diff changeset
   774
                case 'F': case 'G': case 'H': case 'I': case 'J':
06bc494ca11e Initial load
duke
parents:
diff changeset
   775
                case 'K': case 'L': case 'M': case 'N': case 'O':
06bc494ca11e Initial load
duke
parents:
diff changeset
   776
                case 'P': case 'Q': case 'R': case 'S': case 'T':
06bc494ca11e Initial load
duke
parents:
diff changeset
   777
                case 'U': case 'V': case 'W': case 'X': case 'Y':
06bc494ca11e Initial load
duke
parents:
diff changeset
   778
                case 'Z':
06bc494ca11e Initial load
duke
parents:
diff changeset
   779
                case 'a': case 'b': case 'c': case 'd': case 'e':
06bc494ca11e Initial load
duke
parents:
diff changeset
   780
                case 'f': case 'g': case 'h': case 'i': case 'j':
06bc494ca11e Initial load
duke
parents:
diff changeset
   781
                case 'k': case 'l': case 'm': case 'n': case 'o':
06bc494ca11e Initial load
duke
parents:
diff changeset
   782
                case 'p': case 'q': case 'r': case 's': case 't':
06bc494ca11e Initial load
duke
parents:
diff changeset
   783
                case 'u': case 'v': case 'w': case 'x': case 'y':
06bc494ca11e Initial load
duke
parents:
diff changeset
   784
                case 'z':
06bc494ca11e Initial load
duke
parents:
diff changeset
   785
                case '$': case '_':
06bc494ca11e Initial load
duke
parents:
diff changeset
   786
                    scanIdent();
06bc494ca11e Initial load
duke
parents:
diff changeset
   787
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   788
                case '0':
06bc494ca11e Initial load
duke
parents:
diff changeset
   789
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   790
                    if (ch == 'x' || ch == 'X') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   791
                        scanChar();
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   792
                        skipIllegalUnderscores();
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   793
                        if (ch == '.') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   794
                            scanHexFractionAndSuffix(false);
06bc494ca11e Initial load
duke
parents:
diff changeset
   795
                        } else if (digit(16) < 0) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   796
                            lexError("invalid.hex.number");
06bc494ca11e Initial load
duke
parents:
diff changeset
   797
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   798
                            scanNumber(16);
06bc494ca11e Initial load
duke
parents:
diff changeset
   799
                        }
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   800
                    } else if (ch == 'b' || ch == 'B') {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   801
                        if (!allowBinaryLiterals) {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   802
                            lexError("unsupported.binary.lit", source.name);
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   803
                            allowBinaryLiterals = true;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   804
                        }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   805
                        scanChar();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   806
                        skipIllegalUnderscores();
4072
70eaf9773f81 6891079: Compiler allows invalid binary literals 0b and oBL
jjg
parents: 3895
diff changeset
   807
                        if (digit(2) < 0) {
70eaf9773f81 6891079: Compiler allows invalid binary literals 0b and oBL
jjg
parents: 3895
diff changeset
   808
                            lexError("invalid.binary.number");
70eaf9773f81 6891079: Compiler allows invalid binary literals 0b and oBL
jjg
parents: 3895
diff changeset
   809
                        } else {
70eaf9773f81 6891079: Compiler allows invalid binary literals 0b and oBL
jjg
parents: 3895
diff changeset
   810
                            scanNumber(2);
70eaf9773f81 6891079: Compiler allows invalid binary literals 0b and oBL
jjg
parents: 3895
diff changeset
   811
                        }
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   812
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   813
                        putChar('0');
3895
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   814
                        if (ch == '_') {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   815
                            int savePos = bp;
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   816
                            do {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   817
                                scanChar();
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   818
                            } while (ch == '_');
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   819
                            if (digit(10) < 0) {
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   820
                                lexError(savePos, "illegal.underscore");
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   821
                            }
3b3c2a1e5e8a 6860965: Project Coin: binary literals
jjg
parents: 2723
diff changeset
   822
                        }
10
06bc494ca11e Initial load
duke
parents:
diff changeset
   823
                        scanNumber(8);
06bc494ca11e Initial load
duke
parents:
diff changeset
   824
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   825
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   826
                case '1': case '2': case '3': case '4':
06bc494ca11e Initial load
duke
parents:
diff changeset
   827
                case '5': case '6': case '7': case '8': case '9':
06bc494ca11e Initial load
duke
parents:
diff changeset
   828
                    scanNumber(10);
06bc494ca11e Initial load
duke
parents:
diff changeset
   829
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   830
                case '.':
06bc494ca11e Initial load
duke
parents:
diff changeset
   831
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   832
                    if ('0' <= ch && ch <= '9') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   833
                        putChar('.');
06bc494ca11e Initial load
duke
parents:
diff changeset
   834
                        scanFractionAndSuffix();
06bc494ca11e Initial load
duke
parents:
diff changeset
   835
                    } else if (ch == '.') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   836
                        putChar('.'); putChar('.');
06bc494ca11e Initial load
duke
parents:
diff changeset
   837
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   838
                        if (ch == '.') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   839
                            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   840
                            putChar('.');
06bc494ca11e Initial load
duke
parents:
diff changeset
   841
                            token = ELLIPSIS;
06bc494ca11e Initial load
duke
parents:
diff changeset
   842
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   843
                            lexError("malformed.fp.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   844
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   845
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   846
                        token = DOT;
06bc494ca11e Initial load
duke
parents:
diff changeset
   847
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   848
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   849
                case ',':
06bc494ca11e Initial load
duke
parents:
diff changeset
   850
                    scanChar(); token = COMMA; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   851
                case ';':
06bc494ca11e Initial load
duke
parents:
diff changeset
   852
                    scanChar(); token = SEMI; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   853
                case '(':
06bc494ca11e Initial load
duke
parents:
diff changeset
   854
                    scanChar(); token = LPAREN; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   855
                case ')':
06bc494ca11e Initial load
duke
parents:
diff changeset
   856
                    scanChar(); token = RPAREN; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   857
                case '[':
06bc494ca11e Initial load
duke
parents:
diff changeset
   858
                    scanChar(); token = LBRACKET; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   859
                case ']':
06bc494ca11e Initial load
duke
parents:
diff changeset
   860
                    scanChar(); token = RBRACKET; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   861
                case '{':
06bc494ca11e Initial load
duke
parents:
diff changeset
   862
                    scanChar(); token = LBRACE; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   863
                case '}':
06bc494ca11e Initial load
duke
parents:
diff changeset
   864
                    scanChar(); token = RBRACE; return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   865
                case '/':
06bc494ca11e Initial load
duke
parents:
diff changeset
   866
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   867
                    if (ch == '/') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   868
                        do {
06bc494ca11e Initial load
duke
parents:
diff changeset
   869
                            scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   870
                        } while (ch != CR && ch != LF && bp < buflen);
06bc494ca11e Initial load
duke
parents:
diff changeset
   871
                        if (bp < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   872
                            endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   873
                            processComment(CommentStyle.LINE);
06bc494ca11e Initial load
duke
parents:
diff changeset
   874
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   875
                        break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   876
                    } else if (ch == '*') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   877
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   878
                        CommentStyle style;
06bc494ca11e Initial load
duke
parents:
diff changeset
   879
                        if (ch == '*') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   880
                            style = CommentStyle.JAVADOC;
06bc494ca11e Initial load
duke
parents:
diff changeset
   881
                            scanDocComment();
06bc494ca11e Initial load
duke
parents:
diff changeset
   882
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   883
                            style = CommentStyle.BLOCK;
06bc494ca11e Initial load
duke
parents:
diff changeset
   884
                            while (bp < buflen) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   885
                                if (ch == '*') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   886
                                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   887
                                    if (ch == '/') break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   888
                                } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   889
                                    scanCommentChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   890
                                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   891
                            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   892
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   893
                        if (ch == '/') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   894
                            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   895
                            endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   896
                            processComment(style);
06bc494ca11e Initial load
duke
parents:
diff changeset
   897
                            break;
06bc494ca11e Initial load
duke
parents:
diff changeset
   898
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   899
                            lexError("unclosed.comment");
06bc494ca11e Initial load
duke
parents:
diff changeset
   900
                            return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   901
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   902
                    } else if (ch == '=') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   903
                        name = names.slashequals;
06bc494ca11e Initial load
duke
parents:
diff changeset
   904
                        token = SLASHEQ;
06bc494ca11e Initial load
duke
parents:
diff changeset
   905
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   906
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   907
                        name = names.slash;
06bc494ca11e Initial load
duke
parents:
diff changeset
   908
                        token = SLASH;
06bc494ca11e Initial load
duke
parents:
diff changeset
   909
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   910
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   911
                case '\'':
06bc494ca11e Initial load
duke
parents:
diff changeset
   912
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   913
                    if (ch == '\'') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   914
                        lexError("empty.char.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   915
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   916
                        if (ch == CR || ch == LF)
06bc494ca11e Initial load
duke
parents:
diff changeset
   917
                            lexError(pos, "illegal.line.end.in.char.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   918
                        scanLitChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   919
                        if (ch == '\'') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   920
                            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   921
                            token = CHARLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   922
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   923
                            lexError(pos, "unclosed.char.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   924
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   925
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   926
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   927
                case '\"':
06bc494ca11e Initial load
duke
parents:
diff changeset
   928
                    scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   929
                    while (ch != '\"' && ch != CR && ch != LF && bp < buflen)
06bc494ca11e Initial load
duke
parents:
diff changeset
   930
                        scanLitChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   931
                    if (ch == '\"') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   932
                        token = STRINGLITERAL;
06bc494ca11e Initial load
duke
parents:
diff changeset
   933
                        scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   934
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   935
                        lexError(pos, "unclosed.str.lit");
06bc494ca11e Initial load
duke
parents:
diff changeset
   936
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   937
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   938
                default:
06bc494ca11e Initial load
duke
parents:
diff changeset
   939
                    if (isSpecial(ch)) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   940
                        scanOperator();
06bc494ca11e Initial load
duke
parents:
diff changeset
   941
                    } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   942
                        boolean isJavaIdentifierStart;
06bc494ca11e Initial load
duke
parents:
diff changeset
   943
                        if (ch < '\u0080') {
06bc494ca11e Initial load
duke
parents:
diff changeset
   944
                            // all ASCII range chars already handled, above
06bc494ca11e Initial load
duke
parents:
diff changeset
   945
                            isJavaIdentifierStart = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
   946
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   947
                            char high = scanSurrogates();
06bc494ca11e Initial load
duke
parents:
diff changeset
   948
                            if (high != 0) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   949
                                if (sp == sbuf.length) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   950
                                    putChar(high);
06bc494ca11e Initial load
duke
parents:
diff changeset
   951
                                } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   952
                                    sbuf[sp++] = high;
06bc494ca11e Initial load
duke
parents:
diff changeset
   953
                                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   954
06bc494ca11e Initial load
duke
parents:
diff changeset
   955
                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
06bc494ca11e Initial load
duke
parents:
diff changeset
   956
                                    Character.toCodePoint(high, ch));
06bc494ca11e Initial load
duke
parents:
diff changeset
   957
                            } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   958
                                isJavaIdentifierStart = Character.isJavaIdentifierStart(ch);
06bc494ca11e Initial load
duke
parents:
diff changeset
   959
                            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   960
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   961
                        if (isJavaIdentifierStart) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   962
                            scanIdent();
06bc494ca11e Initial load
duke
parents:
diff changeset
   963
                        } else if (bp == buflen || ch == EOI && bp+1 == buflen) { // JLS 3.5
06bc494ca11e Initial load
duke
parents:
diff changeset
   964
                            token = EOF;
06bc494ca11e Initial load
duke
parents:
diff changeset
   965
                            pos = bp = eofPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
   966
                        } else {
06bc494ca11e Initial load
duke
parents:
diff changeset
   967
                            lexError("illegal.char", String.valueOf((int)ch));
06bc494ca11e Initial load
duke
parents:
diff changeset
   968
                            scanChar();
06bc494ca11e Initial load
duke
parents:
diff changeset
   969
                        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   970
                    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   971
                    return;
06bc494ca11e Initial load
duke
parents:
diff changeset
   972
                }
06bc494ca11e Initial load
duke
parents:
diff changeset
   973
            }
06bc494ca11e Initial load
duke
parents:
diff changeset
   974
        } finally {
06bc494ca11e Initial load
duke
parents:
diff changeset
   975
            endPos = bp;
06bc494ca11e Initial load
duke
parents:
diff changeset
   976
            if (scannerDebug)
06bc494ca11e Initial load
duke
parents:
diff changeset
   977
                System.out.println("nextToken(" + pos
06bc494ca11e Initial load
duke
parents:
diff changeset
   978
                                   + "," + endPos + ")=|" +
06bc494ca11e Initial load
duke
parents:
diff changeset
   979
                                   new String(getRawCharacters(pos, endPos))
06bc494ca11e Initial load
duke
parents:
diff changeset
   980
                                   + "|");
06bc494ca11e Initial load
duke
parents:
diff changeset
   981
        }
06bc494ca11e Initial load
duke
parents:
diff changeset
   982
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   983
06bc494ca11e Initial load
duke
parents:
diff changeset
   984
    /** Return the current token, set by nextToken().
06bc494ca11e Initial load
duke
parents:
diff changeset
   985
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   986
    public Token token() {
06bc494ca11e Initial load
duke
parents:
diff changeset
   987
        return token;
06bc494ca11e Initial load
duke
parents:
diff changeset
   988
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   989
06bc494ca11e Initial load
duke
parents:
diff changeset
   990
    /** Sets the current token.
06bc494ca11e Initial load
duke
parents:
diff changeset
   991
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
   992
    public void token(Token token) {
06bc494ca11e Initial load
duke
parents:
diff changeset
   993
        this.token = token;
06bc494ca11e Initial load
duke
parents:
diff changeset
   994
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
   995
06bc494ca11e Initial load
duke
parents:
diff changeset
   996
    /** Return the current token's position: a 0-based
06bc494ca11e Initial load
duke
parents:
diff changeset
   997
     *  offset from beginning of the raw input stream
06bc494ca11e Initial load
duke
parents:
diff changeset
   998
     *  (before unicode translation)
06bc494ca11e Initial load
duke
parents:
diff changeset
   999
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1000
    public int pos() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1001
        return pos;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1002
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1003
06bc494ca11e Initial load
duke
parents:
diff changeset
  1004
    /** Return the last character position of the current token.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1005
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1006
    public int endPos() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1007
        return endPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1008
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1009
06bc494ca11e Initial load
duke
parents:
diff changeset
  1010
    /** Return the last character position of the previous token.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1011
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1012
    public int prevEndPos() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1013
        return prevEndPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1014
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1015
06bc494ca11e Initial load
duke
parents:
diff changeset
  1016
    /** Return the position where a lexical error occurred;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1017
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1018
    public int errPos() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1019
        return errPos;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1020
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1021
06bc494ca11e Initial load
duke
parents:
diff changeset
  1022
    /** Set the position where a lexical error occurred;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1023
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1024
    public void errPos(int pos) {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1025
        errPos = pos;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1026
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1027
06bc494ca11e Initial load
duke
parents:
diff changeset
  1028
    /** Return the name of an identifier or token for the current token.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1029
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1030
    public Name name() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1031
        return name;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1032
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1033
06bc494ca11e Initial load
duke
parents:
diff changeset
  1034
    /** Return the radix of a numeric literal token.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1035
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1036
    public int radix() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1037
        return radix;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1038
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1039
06bc494ca11e Initial load
duke
parents:
diff changeset
  1040
    /** Has a @deprecated been encountered in last doc comment?
06bc494ca11e Initial load
duke
parents:
diff changeset
  1041
     *  This needs to be reset by client with resetDeprecatedFlag.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1042
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1043
    public boolean deprecatedFlag() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1044
        return deprecatedFlag;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1045
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1046
06bc494ca11e Initial load
duke
parents:
diff changeset
  1047
    public void resetDeprecatedFlag() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1048
        deprecatedFlag = false;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1049
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1050
06bc494ca11e Initial load
duke
parents:
diff changeset
  1051
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1052
     * Returns the documentation string of the current token.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1053
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1054
    public String docComment() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1055
        return null;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1056
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1057
06bc494ca11e Initial load
duke
parents:
diff changeset
  1058
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1059
     * Returns a copy of the input buffer, up to its inputLength.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1060
     * Unicode escape sequences are not translated.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1061
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1062
    public char[] getRawCharacters() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1063
        char[] chars = new char[buflen];
06bc494ca11e Initial load
duke
parents:
diff changeset
  1064
        System.arraycopy(buf, 0, chars, 0, buflen);
06bc494ca11e Initial load
duke
parents:
diff changeset
  1065
        return chars;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1066
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1067
06bc494ca11e Initial load
duke
parents:
diff changeset
  1068
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1069
     * Returns a copy of a character array subset of the input buffer.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1070
     * The returned array begins at the <code>beginIndex</code> and
06bc494ca11e Initial load
duke
parents:
diff changeset
  1071
     * extends to the character at index <code>endIndex - 1</code>.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1072
     * Thus the length of the substring is <code>endIndex-beginIndex</code>.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1073
     * This behavior is like
06bc494ca11e Initial load
duke
parents:
diff changeset
  1074
     * <code>String.substring(beginIndex, endIndex)</code>.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1075
     * Unicode escape sequences are not translated.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1076
     *
06bc494ca11e Initial load
duke
parents:
diff changeset
  1077
     * @param beginIndex the beginning index, inclusive.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1078
     * @param endIndex the ending index, exclusive.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1079
     * @throws IndexOutOfBounds if either offset is outside of the
06bc494ca11e Initial load
duke
parents:
diff changeset
  1080
     *         array bounds
06bc494ca11e Initial load
duke
parents:
diff changeset
  1081
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1082
    public char[] getRawCharacters(int beginIndex, int endIndex) {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1083
        int length = endIndex - beginIndex;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1084
        char[] chars = new char[length];
06bc494ca11e Initial load
duke
parents:
diff changeset
  1085
        System.arraycopy(buf, beginIndex, chars, 0, length);
06bc494ca11e Initial load
duke
parents:
diff changeset
  1086
        return chars;
06bc494ca11e Initial load
duke
parents:
diff changeset
  1087
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1088
06bc494ca11e Initial load
duke
parents:
diff changeset
  1089
    public enum CommentStyle {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1090
        LINE,
06bc494ca11e Initial load
duke
parents:
diff changeset
  1091
        BLOCK,
06bc494ca11e Initial load
duke
parents:
diff changeset
  1092
        JAVADOC,
06bc494ca11e Initial load
duke
parents:
diff changeset
  1093
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1094
06bc494ca11e Initial load
duke
parents:
diff changeset
  1095
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1096
     * Called when a complete comment has been scanned. pos and endPos
06bc494ca11e Initial load
duke
parents:
diff changeset
  1097
     * will mark the comment boundary.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1098
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1099
    protected void processComment(CommentStyle style) {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1100
        if (scannerDebug)
06bc494ca11e Initial load
duke
parents:
diff changeset
  1101
            System.out.println("processComment(" + pos
06bc494ca11e Initial load
duke
parents:
diff changeset
  1102
                               + "," + endPos + "," + style + ")=|"
06bc494ca11e Initial load
duke
parents:
diff changeset
  1103
                               + new String(getRawCharacters(pos, endPos))
06bc494ca11e Initial load
duke
parents:
diff changeset
  1104
                               + "|");
06bc494ca11e Initial load
duke
parents:
diff changeset
  1105
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1106
06bc494ca11e Initial load
duke
parents:
diff changeset
  1107
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1108
     * Called when a complete whitespace run has been scanned. pos and endPos
06bc494ca11e Initial load
duke
parents:
diff changeset
  1109
     * will mark the whitespace boundary.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1110
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1111
    protected void processWhiteSpace() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1112
        if (scannerDebug)
06bc494ca11e Initial load
duke
parents:
diff changeset
  1113
            System.out.println("processWhitespace(" + pos
06bc494ca11e Initial load
duke
parents:
diff changeset
  1114
                               + "," + endPos + ")=|" +
06bc494ca11e Initial load
duke
parents:
diff changeset
  1115
                               new String(getRawCharacters(pos, endPos))
06bc494ca11e Initial load
duke
parents:
diff changeset
  1116
                               + "|");
06bc494ca11e Initial load
duke
parents:
diff changeset
  1117
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1118
06bc494ca11e Initial load
duke
parents:
diff changeset
  1119
    /**
06bc494ca11e Initial load
duke
parents:
diff changeset
  1120
     * Called when a line terminator has been processed.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1121
     */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1122
    protected void processLineTerminator() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1123
        if (scannerDebug)
06bc494ca11e Initial load
duke
parents:
diff changeset
  1124
            System.out.println("processTerminator(" + pos
06bc494ca11e Initial load
duke
parents:
diff changeset
  1125
                               + "," + endPos + ")=|" +
06bc494ca11e Initial load
duke
parents:
diff changeset
  1126
                               new String(getRawCharacters(pos, endPos))
06bc494ca11e Initial load
duke
parents:
diff changeset
  1127
                               + "|");
06bc494ca11e Initial load
duke
parents:
diff changeset
  1128
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1129
06bc494ca11e Initial load
duke
parents:
diff changeset
  1130
    /** Build a map for translating between line numbers and
06bc494ca11e Initial load
duke
parents:
diff changeset
  1131
     * positions in the input.
06bc494ca11e Initial load
duke
parents:
diff changeset
  1132
     *
06bc494ca11e Initial load
duke
parents:
diff changeset
  1133
     * @return a LineMap */
06bc494ca11e Initial load
duke
parents:
diff changeset
  1134
    public Position.LineMap getLineMap() {
06bc494ca11e Initial load
duke
parents:
diff changeset
  1135
        return Position.makeLineMap(buf, buflen, false);
06bc494ca11e Initial load
duke
parents:
diff changeset
  1136
    }
06bc494ca11e Initial load
duke
parents:
diff changeset
  1137
06bc494ca11e Initial load
duke
parents:
diff changeset
  1138
}