langtools/src/share/classes/com/sun/tools/javac/parser/UnicodeReader.java
author jlahoda
Mon, 14 Jul 2014 12:02:58 +0200
changeset 25600 7f93cb0536fd
parent 15385 ee1eebe7e210
permissions -rw-r--r--
8048805: Request to investigate and update lexer error recovery in javac 8046620: Further investigation needed for few error messages for negative unicode tests in langtools regression ws 8048803: javac should report complete character code in the error messages Summary: Improving error reporting and recovery in the lexer Reviewed-by: jjg, mcimadamore Contributed-by: jan.lahoda@oracle.com, sonali.goel@oracle.com
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     1
/*
15385
ee1eebe7e210 8006775: JSR 308: Compiler changes in JDK8
jjg
parents: 14259
diff changeset
     2
 * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     4
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     7
 * published by the Free Software Foundation.  Oracle designates this
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     8
 * particular file as subject to the "Classpath" exception as provided
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
     9
 * by Oracle in the LICENSE file that accompanied this code.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    10
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    11
 * This code is distributed in the hope that it will be useful, but WITHOUT
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    12
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    13
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    14
 * version 2 for more details (a copy is included in the LICENSE file that
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    15
 * accompanied this code).
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    16
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    17
 * You should have received a copy of the GNU General Public License version
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    18
 * 2 along with this work; if not, write to the Free Software Foundation,
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    19
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    20
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    22
 * or visit www.oracle.com if you need additional information or have any
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    23
 * questions.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    24
 */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    25
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    26
package com.sun.tools.javac.parser;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    27
14049
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
    28
import java.nio.CharBuffer;
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
    29
import java.util.Arrays;
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
    30
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    31
import com.sun.tools.javac.file.JavacFileManager;
14049
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
    32
import com.sun.tools.javac.util.ArrayUtils;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    33
import com.sun.tools.javac.util.Log;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    34
import com.sun.tools.javac.util.Name;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    35
import com.sun.tools.javac.util.Names;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    36
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    37
import static com.sun.tools.javac.util.LayoutCharacters.*;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    38
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    39
/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    40
 * characters contained in the input stream, handling unicode escape accordingly.
15385
ee1eebe7e210 8006775: JSR 308: Compiler changes in JDK8
jjg
parents: 14259
diff changeset
    41
 * Additionally, it provides features for saving chars into a buffer and to retrieve
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    42
 * them at a later stage.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    43
 *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    44
 *  <p><b>This is NOT part of any supported API.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    45
 *  If you write code that depends on this, you do so at your own risk.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    46
 *  This code and its internal interfaces are subject to change or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    47
 *  deletion without notice.</b>
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    48
 */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    49
public class UnicodeReader {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    50
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    51
    /** The input buffer, index of next character to be read,
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    52
     *  index of one past last character in buffer.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    53
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    54
    protected char[] buf;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    55
    protected int bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    56
    protected final int buflen;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    57
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    58
    /** The current character.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    59
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    60
    protected char ch;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    61
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    62
    /** The buffer index of the last converted unicode character
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    63
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    64
    protected int unicodeConversionBp = -1;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    65
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    66
    protected Log log;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    67
    protected Names names;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    68
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    69
    /** A character buffer for saved chars.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    70
     */
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    71
    protected char[] sbuf = new char[128];
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    72
    protected int sp;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    73
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    74
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    75
     * Create a scanner from the input array.  This method might
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    76
     * modify the array.  To avoid copying the input array, ensure
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    77
     * that {@code inputLength < input.length} or
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    78
     * {@code input[input.length -1]} is a white space character.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    79
     *
14259
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
    80
     * @param sf the factory which created this Scanner
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
    81
     * @param buffer the input, might be modified
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    82
     * Must be positive and less than or equal to input.length.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    83
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    84
    protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    85
        this(sf, JavacFileManager.toArray(buffer), buffer.limit());
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    86
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    87
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    88
    protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    89
        log = sf.log;
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
    90
        names = sf.names;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    91
        if (inputLength == input.length) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    92
            if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    93
                inputLength--;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    94
            } else {
14049
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
    95
                input = Arrays.copyOf(input, inputLength + 1);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    96
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    97
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    98
        buf = input;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
    99
        buflen = inputLength;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   100
        buf[buflen] = EOI;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   101
        bp = -1;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   102
        scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   103
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   104
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   105
    /** Read next character.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   106
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   107
    protected void scanChar() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   108
        if (bp < buflen) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   109
            ch = buf[++bp];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   110
            if (ch == '\\') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   111
                convertUnicode();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   112
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   113
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   114
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   115
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   116
    /** Read next character in comment, skipping over double '\' characters.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   117
     */
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   118
    protected void scanCommentChar() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   119
        scanChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   120
        if (ch == '\\') {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   121
            if (peekChar() == '\\' && !isUnicode()) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   122
                skipChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   123
            } else {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   124
                convertUnicode();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   125
            }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   126
        }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   127
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   128
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   129
    /** Append a character to sbuf.
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   130
     */
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   131
    protected void putChar(char ch, boolean scan) {
14049
3207422a0f9b 7193657: provide internal ArrayUtils class to simplify common usage of arrays in javac
jjg
parents: 10948
diff changeset
   132
        sbuf = ArrayUtils.ensureCapacity(sbuf, sp);
10948
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   133
        sbuf[sp++] = ch;
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   134
        if (scan)
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   135
            scanChar();
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   136
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   137
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   138
    protected void putChar(char ch) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   139
        putChar(ch, false);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   140
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   141
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   142
    protected void putChar(boolean scan) {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   143
        putChar(ch, scan);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   144
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   145
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   146
    Name name() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   147
        return names.fromChars(sbuf, 0, sp);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   148
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   149
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   150
    String chars() {
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   151
        return new String(sbuf, 0, sp);
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   152
    }
063463f6535f 7104201: Refactor DocCommentScanner
mcimadamore
parents: 10815
diff changeset
   153
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   154
    /** Convert unicode escape; bp points to initial '\' character
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   155
     *  (Spec 3.3).
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   156
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   157
    protected void convertUnicode() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   158
        if (ch == '\\' && unicodeConversionBp != bp) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   159
            bp++; ch = buf[bp];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   160
            if (ch == 'u') {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   161
                do {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   162
                    bp++; ch = buf[bp];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   163
                } while (ch == 'u');
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   164
                int limit = bp + 3;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   165
                if (limit < buflen) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   166
                    int d = digit(bp, 16);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   167
                    int code = d;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   168
                    while (bp < limit && d >= 0) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   169
                        bp++; ch = buf[bp];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   170
                        d = digit(bp, 16);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   171
                        code = (code << 4) + d;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   172
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   173
                    if (d >= 0) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   174
                        ch = (char)code;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   175
                        unicodeConversionBp = bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   176
                        return;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   177
                    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   178
                }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   179
                log.error(bp, "illegal.unicode.esc");
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   180
            } else {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   181
                bp--;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   182
                ch = '\\';
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   183
            }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   184
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   185
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   186
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   187
    /** Are surrogates supported?
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   188
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   189
    final static boolean surrogatesSupported = surrogatesSupported();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   190
    private static boolean surrogatesSupported() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   191
        try {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   192
            Character.isHighSurrogate('a');
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   193
            return true;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   194
        } catch (NoSuchMethodError ex) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   195
            return false;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   196
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   197
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   198
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   199
    /** Scan surrogate pairs.  If 'ch' is a high surrogate and
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   200
     *  the next character is a low surrogate, returns the code point
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   201
     *  constructed from these surrogates. Otherwise, returns -1.
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   202
     *  This method will not consume any of the characters.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   203
     */
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   204
    protected int peekSurrogates() {
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   205
        if (surrogatesSupported && Character.isHighSurrogate(ch)) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   206
            char high = ch;
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   207
            int prevBP = bp;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   208
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   209
            scanChar();
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   210
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   211
            char low = ch;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   212
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   213
            ch = high;
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   214
            bp = prevBP;
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   215
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   216
            if (Character.isLowSurrogate(low)) {
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   217
                return Character.toCodePoint(high, low);
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   218
            }
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   219
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   220
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   221
        return -1;
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   222
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   223
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   224
    /** Convert an ASCII digit from its base (8, 10, or 16)
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   225
     *  to its value.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   226
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   227
    protected int digit(int pos, int base) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   228
        char c = ch;
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   229
        if ('0' <= c && c <= '9')
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   230
            return Character.digit(c, base); //a fast common case
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   231
        int codePoint = peekSurrogates();
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   232
        int result = codePoint >= 0 ? Character.digit(codePoint, base) : Character.digit(c, base);
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   233
        if (result >= 0 && c > 0x7f) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   234
            log.error(pos + 1, "illegal.nonascii.digit");
25600
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   235
            if (codePoint >= 0)
7f93cb0536fd 8048805: Request to investigate and update lexer error recovery in javac
jlahoda
parents: 15385
diff changeset
   236
                scanChar();
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   237
            ch = "0123456789abcdef".charAt(result);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   238
        }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   239
        return result;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   240
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   241
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   242
    protected boolean isUnicode() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   243
        return unicodeConversionBp == bp;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   244
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   245
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   246
    protected void skipChar() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   247
        bp++;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   248
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   249
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   250
    protected char peekChar() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   251
        return buf[bp + 1];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   252
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   253
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   254
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   255
     * Returns a copy of the input buffer, up to its inputLength.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   256
     * Unicode escape sequences are not translated.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   257
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   258
    public char[] getRawCharacters() {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   259
        char[] chars = new char[buflen];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   260
        System.arraycopy(buf, 0, chars, 0, buflen);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   261
        return chars;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   262
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   263
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   264
    /**
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   265
     * Returns a copy of a character array subset of the input buffer.
14259
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
   266
     * The returned array begins at the {@code beginIndex} and
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
   267
     * extends to the character at index {@code endIndex - 1}.
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
   268
     * Thus the length of the substring is {@code endIndex-beginIndex}.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   269
     * This behavior is like
14259
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
   270
     * {@code String.substring(beginIndex, endIndex)}.
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   271
     * Unicode escape sequences are not translated.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   272
     *
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   273
     * @param beginIndex the beginning index, inclusive.
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   274
     * @param endIndex the ending index, exclusive.
14259
fb94a1df0d53 8000208: fix langtools javadoc comment issues
jjg
parents: 14049
diff changeset
   275
     * @throws ArrayIndexOutOfBoundsException if either offset is outside of the
10815
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   276
     *         array bounds
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   277
     */
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   278
    public char[] getRawCharacters(int beginIndex, int endIndex) {
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   279
        int length = endIndex - beginIndex;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   280
        char[] chars = new char[length];
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   281
        System.arraycopy(buf, beginIndex, chars, 0, length);
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   282
        return chars;
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   283
    }
a719aa5f1631 7096014: Javac tokens should retain state
mcimadamore
parents:
diff changeset
   284
}