1 /*

     2  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.  Oracle designates this

     8  * particular file as subject to the "Classpath" exception as provided

     9  * by Oracle in the LICENSE file that accompanied this code.

    10  *

    11  * This code is distributed in the hope that it will be useful, but WITHOUT

    12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14  * version 2 for more details (a copy is included in the LICENSE file that

    15  * accompanied this code).

    16  *

    17  * You should have received a copy of the GNU General Public License version

    18  * 2 along with this work; if not, write to the Free Software Foundation,

    19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20  *

    21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22  * or visit www.oracle.com if you need additional information or have any

    23  * questions.

    24  */

    25 

    26 package com.sun.tools.javac.parser;

    27 

    28 import java.util.Locale;

    29 

    30 import com.sun.tools.javac.api.Formattable;

    31 import com.sun.tools.javac.api.Messages;

    32 import com.sun.tools.javac.parser.Tokens.Token.Tag;

    33 import com.sun.tools.javac.util.List;

    34 import com.sun.tools.javac.util.Name;

    35 import com.sun.tools.javac.util.Context;

    36 import com.sun.tools.javac.util.Filter;

    37 import com.sun.tools.javac.util.ListBuffer;

    38 import com.sun.tools.javac.util.Names;

    39 

    40 /** A class that defines codes/utilities for Java source tokens

    41  *  returned from lexical analysis.

    42  *

    43  *  <p><b>This is NOT part of any supported API.

    44  *  If you write code that depends on this, you do so at your own risk.

    45  *  This code and its internal interfaces are subject to change or

    46  *  deletion without notice.</b>

    47  */

    48 public class Tokens {

    49 

    50     private final Names names;

    51 

    52     /**

    53      * Keyword array. Maps name indices to Token.

    54      */

    55     private final TokenKind[] key;

    56 

    57     /**  The number of the last entered keyword.

    58      */

    59     private int maxKey = 0;

    60 

    61     /** The names of all tokens.

    62      */

    63     private Name[] tokenName = new Name[TokenKind.values().length];

    64 

    65     public static final Context.Key<Tokens> tokensKey = new Context.Key<>();

    66 

    67     public static Tokens instance(Context context) {

    68         Tokens instance = context.get(tokensKey);

    69         if (instance == null)

    70             instance = new Tokens(context);

    71         return instance;

    72     }

    73 

    74     protected Tokens(Context context) {

    75         context.put(tokensKey, this);

    76         names = Names.instance(context);

    77         for (TokenKind t : TokenKind.values()) {

    78             if (t.name != null)

    79                 enterKeyword(t.name, t);

    80             else

    81                 tokenName[t.ordinal()] = null;

    82         }

    83 

    84         key = new TokenKind[maxKey+1];

    85         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;

    86         for (TokenKind t : TokenKind.values()) {

    87             if (t.name != null)

    88             key[tokenName[t.ordinal()].getIndex()] = t;

    89         }

    90     }

    91 

    92     private void enterKeyword(String s, TokenKind token) {

    93         Name n = names.fromString(s);

    94         tokenName[token.ordinal()] = n;

    95         if (n.getIndex() > maxKey) maxKey = n.getIndex();

    96     }

    97 

    98     /**

    99      * Create a new token given a name; if the name corresponds to a token name,

   100      * a new token of the corresponding kind is returned; otherwise, an

   101      * identifier token is returned.

   102      */

   103     TokenKind lookupKind(Name name) {

   104         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];

   105     }

   106 

   107     TokenKind lookupKind(String name) {

   108         return lookupKind(names.fromString(name));

   109     }

   110 

   111     /**

   112      * This enum defines all tokens used by the javac scanner. A token is

   113      * optionally associated with a name.

   114      */

   115     public enum TokenKind implements Formattable, Filter<TokenKind> {

   116         EOF(),

   117         ERROR(),

   118         IDENTIFIER(Tag.NAMED),

   119         ABSTRACT("abstract"),

   120         ASSERT("assert", Tag.NAMED),

   121         BOOLEAN("boolean", Tag.NAMED),

   122         BREAK("break"),

   123         BYTE("byte", Tag.NAMED),

   124         CASE("case"),

   125         CATCH("catch"),

   126         CHAR("char", Tag.NAMED),

   127         CLASS("class"),

   128         CONST("const"),

   129         CONTINUE("continue"),

   130         DEFAULT("default"),

   131         DO("do"),

   132         DOUBLE("double", Tag.NAMED),

   133         ELSE("else"),

   134         ENUM("enum", Tag.NAMED),

   135         EXTENDS("extends"),

   136         FINAL("final"),

   137         FINALLY("finally"),

   138         FLOAT("float", Tag.NAMED),

   139         FOR("for"),

   140         GOTO("goto"),

   141         IF("if"),

   142         IMPLEMENTS("implements"),

   143         IMPORT("import"),

   144         INSTANCEOF("instanceof"),

   145         INT("int", Tag.NAMED),

   146         INTERFACE("interface"),

   147         LONG("long", Tag.NAMED),

   148         NATIVE("native"),

   149         NEW("new"),

   150         PACKAGE("package"),

   151         PRIVATE("private"),

   152         PROTECTED("protected"),

   153         PUBLIC("public"),

   154         RETURN("return"),

   155         SHORT("short", Tag.NAMED),

   156         STATIC("static"),

   157         STRICTFP("strictfp"),

   158         SUPER("super", Tag.NAMED),

   159         SWITCH("switch"),

   160         SYNCHRONIZED("synchronized"),

   161         THIS("this", Tag.NAMED),

   162         THROW("throw"),

   163         THROWS("throws"),

   164         TRANSIENT("transient"),

   165         TRY("try"),

   166         VOID("void", Tag.NAMED),

   167         VOLATILE("volatile"),

   168         WHILE("while"),

   169         INTLITERAL(Tag.NUMERIC),

   170         LONGLITERAL(Tag.NUMERIC),

   171         FLOATLITERAL(Tag.NUMERIC),

   172         DOUBLELITERAL(Tag.NUMERIC),

   173         CHARLITERAL(Tag.NUMERIC),

   174         STRINGLITERAL(Tag.STRING),

   175         TRUE("true", Tag.NAMED),

   176         FALSE("false", Tag.NAMED),

   177         NULL("null", Tag.NAMED),

   178         UNDERSCORE("_", Tag.NAMED),

   179         ARROW("->"),

   180         COLCOL("::"),

   181         LPAREN("("),

   182         RPAREN(")"),

   183         LBRACE("{"),

   184         RBRACE("}"),

   185         LBRACKET("["),

   186         RBRACKET("]"),

   187         SEMI(";"),

   188         COMMA(","),

   189         DOT("."),

   190         ELLIPSIS("..."),

   191         EQ("="),

   192         GT(">"),

   193         LT("<"),

   194         BANG("!"),

   195         TILDE("~"),

   196         QUES("?"),

   197         COLON(":"),

   198         EQEQ("=="),

   199         LTEQ("<="),

   200         GTEQ(">="),

   201         BANGEQ("!="),

   202         AMPAMP("&&"),

   203         BARBAR("||"),

   204         PLUSPLUS("++"),

   205         SUBSUB("--"),

   206         PLUS("+"),

   207         SUB("-"),

   208         STAR("*"),

   209         SLASH("/"),

   210         AMP("&"),

   211         BAR("|"),

   212         CARET("^"),

   213         PERCENT("%"),

   214         LTLT("<<"),

   215         GTGT(">>"),

   216         GTGTGT(">>>"),

   217         PLUSEQ("+="),

   218         SUBEQ("-="),

   219         STAREQ("*="),

   220         SLASHEQ("/="),

   221         AMPEQ("&="),

   222         BAREQ("|="),

   223         CARETEQ("^="),

   224         PERCENTEQ("%="),

   225         LTLTEQ("<<="),

   226         GTGTEQ(">>="),

   227         GTGTGTEQ(">>>="),

   228         MONKEYS_AT("@"),

   229         CUSTOM;

   230 

   231         public final String name;

   232         public final Tag tag;

   233 

   234         TokenKind() {

   235             this(null, Tag.DEFAULT);

   236         }

   237 

   238         TokenKind(String name) {

   239             this(name, Tag.DEFAULT);

   240         }

   241 

   242         TokenKind(Tag tag) {

   243             this(null, tag);

   244         }

   245 

   246         TokenKind(String name, Tag tag) {

   247             this.name = name;

   248             this.tag = tag;

   249         }

   250 

   251         public String toString() {

   252             switch (this) {

   253             case IDENTIFIER:

   254                 return "token.identifier";

   255             case CHARLITERAL:

   256                 return "token.character";

   257             case STRINGLITERAL:

   258                 return "token.string";

   259             case INTLITERAL:

   260                 return "token.integer";

   261             case LONGLITERAL:

   262                 return "token.long-integer";

   263             case FLOATLITERAL:

   264                 return "token.float";

   265             case DOUBLELITERAL:

   266                 return "token.double";

   267             case ERROR:

   268                 return "token.bad-symbol";

   269             case EOF:

   270                 return "token.end-of-input";

   271             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:

   272             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:

   273                 return "'" + name + "'";

   274             default:

   275                 return name;

   276             }

   277         }

   278 

   279         public String getKind() {

   280             return "Token";

   281         }

   282 

   283         public String toString(Locale locale, Messages messages) {

   284             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());

   285         }

   286 

   287         @Override

   288         public boolean accepts(TokenKind that) {

   289             return this == that;

   290         }

   291     }

   292 

   293     public interface Comment {

   294 

   295         enum CommentStyle {

   296             LINE,

   297             BLOCK,

   298             JAVADOC,

   299         }

   300 

   301         String getText();

   302         int getSourcePos(int index);

   303         CommentStyle getStyle();

   304         boolean isDeprecated();

   305     }

   306 

   307     /**

   308      * This is the class representing a javac token. Each token has several fields

   309      * that are set by the javac lexer (i.e. start/end position, string value, etc).

   310      */

   311     public static class Token {

   312 

   313         /** tags constants **/

   314         enum Tag {

   315             DEFAULT,

   316             NAMED,

   317             STRING,

   318             NUMERIC

   319         }

   320 

   321         /** The token kind */

   322         public final TokenKind kind;

   323 

   324         /** The start position of this token */

   325         public final int pos;

   326 

   327         /** The end position of this token */

   328         public final int endPos;

   329 

   330         /** Comment reader associated with this token */

   331         public final List<Comment> comments;

   332 

   333         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {

   334             this.kind = kind;

   335             this.pos = pos;

   336             this.endPos = endPos;

   337             this.comments = comments;

   338             checkKind();

   339         }

   340 

   341         Token[] split(Tokens tokens) {

   342             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {

   343                 throw new AssertionError("Cant split" + kind);

   344             }

   345 

   346             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));

   347             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));

   348 

   349             if (t1 == null || t2 == null) {

   350                 throw new AssertionError("Cant split - bad subtokens");

   351             }

   352             return new Token[] {

   353                 new Token(t1, pos, pos + t1.name.length(), comments),

   354                 new Token(t2, pos + t1.name.length(), endPos, null)

   355             };

   356         }

   357 

   358         protected void checkKind() {

   359             if (kind.tag != Tag.DEFAULT) {

   360                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   361             }

   362         }

   363 

   364         public Name name() {

   365             throw new UnsupportedOperationException();

   366         }

   367 

   368         public String stringVal() {

   369             throw new UnsupportedOperationException();

   370         }

   371 

   372         public int radix() {

   373             throw new UnsupportedOperationException();

   374         }

   375 

   376         /**

   377          * Preserve classic semantics - if multiple javadocs are found on the token

   378          * the last one is returned

   379          */

   380         public Comment comment(Comment.CommentStyle style) {

   381             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);

   382             return comments.isEmpty() ?

   383                     null :

   384                     comments.head;

   385         }

   386 

   387         /**

   388          * Preserve classic semantics - deprecated should be set if at least one

   389          * javadoc comment attached to this token contains the '@deprecated' string

   390          */

   391         public boolean deprecatedFlag() {

   392             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {

   393                 if (c.isDeprecated()) {

   394                     return true;

   395                 }

   396             }

   397             return false;

   398         }

   399 

   400         private List<Comment> getComments(Comment.CommentStyle style) {

   401             if (comments == null) {

   402                 return List.nil();

   403             } else {

   404                 ListBuffer<Comment> buf = new ListBuffer<>();

   405                 for (Comment c : comments) {

   406                     if (c.getStyle() == style) {

   407                         buf.add(c);

   408                     }

   409                 }

   410                 return buf.toList();

   411             }

   412         }

   413     }

   414 

   415     final static class NamedToken extends Token {

   416         /** The name of this token */

   417         public final Name name;

   418 

   419         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {

   420             super(kind, pos, endPos, comments);

   421             this.name = name;

   422         }

   423 

   424         protected void checkKind() {

   425             if (kind.tag != Tag.NAMED) {

   426                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);

   427             }

   428         }

   429 

   430         @Override

   431         public Name name() {

   432             return name;

   433         }

   434     }

   435 

   436     static class StringToken extends Token {

   437         /** The string value of this token */

   438         public final String stringVal;

   439 

   440         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {

   441             super(kind, pos, endPos, comments);

   442             this.stringVal = stringVal;

   443         }

   444 

   445         protected void checkKind() {

   446             if (kind.tag != Tag.STRING) {

   447                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);

   448             }

   449         }

   450 

   451         @Override

   452         public String stringVal() {

   453             return stringVal;

   454         }

   455     }

   456 

   457     final static class NumericToken extends StringToken {

   458         /** The 'radix' value of this token */

   459         public final int radix;

   460 

   461         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {

   462             super(kind, pos, endPos, stringVal, comments);

   463             this.radix = radix;

   464         }

   465 

   466         protected void checkKind() {

   467             if (kind.tag != Tag.NUMERIC) {

   468                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);

   469             }

   470         }

   471 

   472         @Override

   473         public int radix() {

   474             return radix;

   475         }

   476     }

   477 

   478     public static final Token DUMMY =

   479                 new Token(TokenKind.ERROR, 0, 0, null);

   480 }