nashorn/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/parser/Token.java
author mhaupt
Wed, 28 Oct 2015 10:54:05 +0100
changeset 33414 2e284c36d51f
parent 25865 d38d876f1654
child 33890 2e8c1be40a52
permissions -rw-r--r--
8134941: Implement ES6 template literal support Reviewed-by: attila, hannesw Contributed-by: andreas.woess@oracle.com
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
16147
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     1
/*
33414
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
     2
 * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
16147
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     3
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     4
 *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     5
 * This code is free software; you can redistribute it and/or modify it
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     6
 * under the terms of the GNU General Public License version 2 only, as
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     7
 * published by the Free Software Foundation.  Oracle designates this
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     8
 * particular file as subject to the "Classpath" exception as provided
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
     9
 * by Oracle in the LICENSE file that accompanied this code.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    10
 *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    11
 * This code is distributed in the hope that it will be useful, but WITHOUT
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    12
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    13
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    14
 * version 2 for more details (a copy is included in the LICENSE file that
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    15
 * accompanied this code).
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    16
 *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    17
 * You should have received a copy of the GNU General Public License version
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    18
 * 2 along with this work; if not, write to the Free Software Foundation,
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    19
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    20
 *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    21
 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    22
 * or visit www.oracle.com if you need additional information or have any
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    23
 * questions.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    24
 */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    25
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    26
package jdk.nashorn.internal.parser;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    27
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    28
import static jdk.nashorn.internal.parser.TokenKind.LITERAL;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    29
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    30
import jdk.nashorn.internal.runtime.Source;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    31
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    32
/**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    33
 * Basic parse/lex unit.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    34
 *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    35
 */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    36
public class Token {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    37
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    38
    private Token() {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    39
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    40
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    41
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    42
     * Create a compact form of token information.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    43
     * @param type     Type of token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    44
     * @param position Start position of the token in the source.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    45
     * @param length   Length of the token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    46
     * @return Token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    47
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    48
    public static long toDesc(final TokenType type, final int position, final int length) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    49
        return (long)position << 32 |
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    50
               (long)length   << 8  |
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    51
               type.ordinal();
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    52
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    53
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    54
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    55
     * Extract token position from a token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    56
     * @param token Token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    57
     * @return Start position of the token in the source.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    58
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    59
    public static int descPosition(final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    60
        return (int)(token >>> 32);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    61
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    62
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    63
    /**
24994
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    64
     * Normally returns the token itself, except in case of string tokens
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    65
     * which report their position past their opening delimiter and thus
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    66
     * need to have position and length adjusted.
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    67
     *
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    68
     * @param token Token descriptor.
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    69
     * @return same or adjusted token.
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    70
     */
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    71
    public static long withDelimiter(final long token) {
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    72
        final TokenType tokenType = Token.descType(token);
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    73
        switch(tokenType) {
33414
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    74
            case STRING:
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    75
            case ESCSTRING:
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    76
            case EXECSTRING:
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    77
            case TEMPLATE:
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    78
            case TEMPLATE_TAIL: {
24994
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    79
                final int start = Token.descPosition(token) - 1;
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    80
                final int len = Token.descLength(token) + 2;
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    81
                return toDesc(tokenType, start, len);
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    82
            }
33414
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    83
            case TEMPLATE_HEAD:
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    84
            case TEMPLATE_MIDDLE: {
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    85
                final int start = Token.descPosition(token) - 1;
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    86
                final int len = Token.descLength(token) + 3;
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    87
                return toDesc(tokenType, start, len);
2e284c36d51f 8134941: Implement ES6 template literal support
mhaupt
parents: 25865
diff changeset
    88
            }
24994
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    89
            default: {
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    90
                return token;
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    91
            }
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    92
        }
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    93
    }
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    94
92a19723a5ac 8047035: (function() "hello")() crashes in Lexer with jdk9
sundar
parents: 16151
diff changeset
    95
    /**
16147
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    96
     * Extract token length from a token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    97
     * @param token Token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    98
     * @return Length of the token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
    99
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   100
    public static int descLength(final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   101
        return (int)token >>> 8;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   102
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   103
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   104
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   105
     * Extract token type from a token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   106
     * @param token Token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   107
     * @return Type of token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   108
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   109
    public static TokenType descType(final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   110
        return TokenType.getValues()[(int)token & 0xff];
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   111
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   112
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   113
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   114
     * Change the token to use a new type.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   115
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   116
     * @param token   The original token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   117
     * @param newType The new token type.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   118
     * @return The recast token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   119
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   120
    public static long recast(final long token, final TokenType newType) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   121
        return token & ~0xFFL | newType.ordinal();
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   122
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   123
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   124
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   125
     * Return a string representation of a token.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   126
     * @param source  Token source.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   127
     * @param token   Token descriptor.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   128
     * @param verbose True to include details.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   129
     * @return String representation.
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   130
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   131
    public static String toString(final Source source, final long token, final boolean verbose) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   132
        final TokenType type = Token.descType(token);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   133
        String result;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   134
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   135
        if (source != null && type.getKind() == LITERAL) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   136
            result = source.getString(token);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   137
        } else {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   138
            result = type.getNameOrType();
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   139
        }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   140
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   141
        if (verbose) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   142
            final int position = Token.descPosition(token);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   143
            final int length = Token.descLength(token);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   144
            result += " (" + position + ", " + length + ")";
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   145
        }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   146
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   147
        return result;
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   148
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   149
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   150
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   151
     * String conversion of token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   152
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   153
     * @param source the source
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   154
     * @param token  the token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   155
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   156
     * @return token as string
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   157
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   158
    public static String toString(final Source source, final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   159
        return Token.toString(source, token, false);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   160
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   161
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   162
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   163
     * String conversion of token - version without source given
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   164
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   165
     * @param token  the token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   166
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   167
     * @return token as string
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   168
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   169
    public static String toString(final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   170
        return Token.toString(null, token, false);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   171
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   172
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   173
    /**
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   174
     * Static hash code computation function token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   175
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   176
     * @param token a token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   177
     *
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   178
     * @return hash code for token
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   179
     */
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   180
    public static int hashCode(final long token) {
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   181
        return (int)(token ^ token >>> 32);
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   182
    }
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   183
e63b63819133 8005403: Open-source Nashorn
jlaskey
parents:
diff changeset
   184
}