langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java
changeset 14720 c24c61d0d9a6
parent 14370 eefd0e6642a8
child 14801 d66cab4ef397
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Tue Nov 27 13:55:10 2012 -0800
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java	Fri Nov 23 15:13:45 2012 +0000
@@ -348,8 +348,8 @@
     private void scanIdent() {
         boolean isJavaIdentifierPart;
         char high;
+        reader.putChar(true);
         do {
-            reader.putChar(true);
             switch (reader.ch) {
             case 'A': case 'B': case 'C': case 'D': case 'E':
             case 'F': case 'G': case 'H': case 'I': case 'J':
@@ -366,6 +366,7 @@
             case '$': case '_':
             case '0': case '1': case '2': case '3': case '4':
             case '5': case '6': case '7': case '8': case '9':
+                break;
             case '\u0000': case '\u0001': case '\u0002': case '\u0003':
             case '\u0004': case '\u0005': case '\u0006': case '\u0007':
             case '\u0008': case '\u000E': case '\u000F': case '\u0010':
@@ -373,26 +374,33 @@
             case '\u0015': case '\u0016': case '\u0017':
             case '\u0018': case '\u0019': case '\u001B':
             case '\u007F':
-                break;
+                reader.scanChar();
+                continue;
             case '\u001A': // EOI is also a legal identifier part
                 if (reader.bp >= reader.buflen) {
                     name = reader.name();
                     tk = tokens.lookupKind(name);
                     return;
                 }
-                break;
+                reader.scanChar();
+                continue;
             default:
                 if (reader.ch < '\u0080') {
                     // all ASCII range chars already handled, above
                     isJavaIdentifierPart = false;
                 } else {
-                    high = reader.scanSurrogates();
-                    if (high != 0) {
-                        reader.putChar(high);
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(
-                            Character.toCodePoint(high, reader.ch));
+                    if (Character.isIdentifierIgnorable(reader.ch)) {
+                        reader.scanChar();
+                        continue;
                     } else {
-                        isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                        high = reader.scanSurrogates();
+                        if (high != 0) {
+                            reader.putChar(high);
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(
+                                Character.toCodePoint(high, reader.ch));
+                        } else {
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                        }
                     }
                 }
                 if (!isJavaIdentifierPart) {
@@ -401,6 +409,7 @@
                     return;
                 }
             }
+            reader.putChar(true);
         } while (true);
     }