7144981: javac should ignore ignorable characters in input
Reviewed-by: jjg, mcimadamore
--- a/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Tue Nov 27 13:55:10 2012 -0800
+++ b/langtools/src/share/classes/com/sun/tools/javac/parser/JavaTokenizer.java Fri Nov 23 15:13:45 2012 +0000
@@ -348,8 +348,8 @@
private void scanIdent() {
boolean isJavaIdentifierPart;
char high;
+ reader.putChar(true);
do {
- reader.putChar(true);
switch (reader.ch) {
case 'A': case 'B': case 'C': case 'D': case 'E':
case 'F': case 'G': case 'H': case 'I': case 'J':
@@ -366,6 +366,7 @@
case '$': case '_':
case '0': case '1': case '2': case '3': case '4':
case '5': case '6': case '7': case '8': case '9':
+ break;
case '\u0000': case '\u0001': case '\u0002': case '\u0003':
case '\u0004': case '\u0005': case '\u0006': case '\u0007':
case '\u0008': case '\u000E': case '\u000F': case '\u0010':
@@ -373,26 +374,33 @@
case '\u0015': case '\u0016': case '\u0017':
case '\u0018': case '\u0019': case '\u001B':
case '\u007F':
- break;
+ reader.scanChar();
+ continue;
case '\u001A': // EOI is also a legal identifier part
if (reader.bp >= reader.buflen) {
name = reader.name();
tk = tokens.lookupKind(name);
return;
}
- break;
+ reader.scanChar();
+ continue;
default:
if (reader.ch < '\u0080') {
// all ASCII range chars already handled, above
isJavaIdentifierPart = false;
} else {
- high = reader.scanSurrogates();
- if (high != 0) {
- reader.putChar(high);
- isJavaIdentifierPart = Character.isJavaIdentifierPart(
- Character.toCodePoint(high, reader.ch));
+ if (Character.isIdentifierIgnorable(reader.ch)) {
+ reader.scanChar();
+ continue;
} else {
- isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+ high = reader.scanSurrogates();
+ if (high != 0) {
+ reader.putChar(high);
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(
+ Character.toCodePoint(high, reader.ch));
+ } else {
+ isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+ }
}
}
if (!isJavaIdentifierPart) {
@@ -401,6 +409,7 @@
return;
}
}
+ reader.putChar(true);
} while (true);
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/langtools/test/tools/javac/7144981/IgnoreIgnorableCharactersInInput.java Fri Nov 23 15:13:45 2012 +0000
@@ -0,0 +1,92 @@
+
+/*
+ * @test /nodynamiccopyright/
+ * @bug 7144981
+ * @summary javac should ignore ignorable characters in input
+ * @run main IgnoreIgnorableCharactersInInput
+ */
+
+import com.sun.source.util.JavacTask;
+import java.io.File;
+import java.net.URI;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TreeSet;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.ToolProvider;
+
+public class IgnoreIgnorableCharactersInInput {
+
+ public static void main(String... args) throws Exception {
+ new IgnoreIgnorableCharactersInInput().run();
+ }
+
+ void run() throws Exception {
+ JavaCompiler comp = ToolProvider.getSystemJavaCompiler();
+ File classesDir = new File(System.getProperty("user.dir"), "classes");
+ classesDir.mkdirs();
+ JavaSource[] sources = new JavaSource[]{
+ new JavaSource("TestOneIgnorableChar", "AA\\u0000BB"),
+ new JavaSource("TestMultipleIgnorableChar", "AA\\u0000\\u0000\\u0000BB")};
+ JavacTask ct = (JavacTask)comp.getTask(null, null, null,
+ Arrays.asList("-d", classesDir.getPath()),
+ null, Arrays.asList(sources));
+ try {
+ if (!ct.call()) {
+ throw new AssertionError("Error thrown when compiling test cases");
+ }
+ } catch (Throwable ex) {
+ throw new AssertionError("Error thrown when compiling test cases");
+ }
+ check(classesDir,
+ "TestOneIgnorableChar.class",
+ "TestOneIgnorableChar$AABB.class",
+ "TestMultipleIgnorableChar.class",
+ "TestMultipleIgnorableChar$AABB.class");
+ if (errors > 0)
+ throw new AssertionError("There are some errors in the test check the error output");
+ }
+
+ /**
+ * Check that a directory contains the expected files.
+ */
+ void check(File dir, String... paths) {
+ Set<String> found = new TreeSet<String>(Arrays.asList(dir.list()));
+ Set<String> expect = new TreeSet<String>(Arrays.asList(paths));
+ if (found.equals(expect))
+ return;
+ for (String f: found) {
+ if (!expect.contains(f))
+ error("Unexpected file found: " + f);
+ }
+ for (String e: expect) {
+ if (!found.contains(e))
+ error("Expected file not found: " + e);
+ }
+ }
+
+ int errors;
+
+ void error(String msg) {
+ System.err.println(msg);
+ errors++;
+ }
+
+ class JavaSource extends SimpleJavaFileObject {
+
+ String internalSource =
+ "public class #O {public class #I {} }";
+ public JavaSource(String outerClassName, String innerClassName) {
+ super(URI.create(outerClassName + ".java"), JavaFileObject.Kind.SOURCE);
+ internalSource =
+ internalSource.replace("#O", outerClassName).replace("#I", innerClassName);
+ }
+
+ @Override
+ public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+ return internalSource;
+ }
+ }
+}