--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/jdk/sun/nio/cs/FindEncoderBugs.java Tue Sep 12 19:03:39 2017 +0200
@@ -0,0 +1,527 @@
+/*
+ * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 6233345 6381699 6381702 6381705 6381706
+ * @summary Encode many char sequences in many ways
+ * @library /test/lib
+ * @build jdk.test.lib.RandomFactory
+ * @run main/timeout=1200 FindEncoderBugs
+ * @author Martin Buchholz
+ * @key randomness
+ */
+
+import java.util.*;
+import java.util.regex.*;
+import java.nio.*;
+import java.nio.charset.*;
+import jdk.test.lib.RandomFactory;
+
+public class FindEncoderBugs {
+
+ static boolean isBroken(String csn) {
+ if (csn.equals("x-COMPOUND_TEXT")) return true;
+ return false;
+ }
+
+ static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
+ List<T> list = new ArrayList<T>(c);
+ Collections.sort(list);
+ return list;
+ }
+
+ static class TooManyFailures extends RuntimeException {
+ private static final long serialVersionUID = 0L;
+ }
+
+ static String string(byte[] a) {
+ final StringBuilder sb = new StringBuilder();
+ for (byte b : a) {
+ if (sb.length() != 0) sb.append(' ');
+ sb.append(String.format("%02x", b & 0xff));
+ }
+ return sb.toString();
+ }
+
+ static String string(char[] a) {
+ final StringBuilder sb = new StringBuilder();
+ for (char c : a) {
+ if (sb.length() != 0) sb.append(' ');
+ sb.append(String.format("\\u%04x", (int) c));
+ }
+ return sb.toString();
+ }
+
+ static class Reporter {
+ // Some machinery to make sure only a small number of errors
+ // that are "too similar" are reported.
+ static class Counts extends HashMap<String, Long> {
+ private static final long serialVersionUID = -1;
+ long inc(String signature) {
+ Long count = get(signature);
+ if (count == null) count = 0L;
+ put(signature, count+1);
+ return count+1;
+ }
+ }
+
+ final Counts failureCounts = new Counts();
+ final static long maxFailures = 2;
+
+ final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
+ final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
+
+ boolean bug(String format, Object... args) {
+ String signature = String.format(format, args);
+ // signature = hideBytes.matcher(signature).replaceAll("\"??\"");
+ // signature = hideChars.matcher(signature).replaceAll("\\u????");
+ failed++;
+ if (failureCounts.inc(signature) <= maxFailures) {
+ System.out.printf(format, args);
+ System.out.println();
+ return true;
+ }
+ return false;
+ }
+
+ void summarize() {
+ for (String key : sort(failureCounts.keySet()))
+ System.out.printf("-----%n%s%nfailures=%d%n",
+ key, failureCounts.get(key));
+ }
+ }
+
+ static final Reporter reporter = new Reporter();
+
+ static class Result {
+ final int limit;
+ final int ipos;
+ final boolean direct;
+ final char[] ia;
+ final byte[] oa;
+ final CoderResult cr;
+
+ private static byte[] toByteArray(ByteBuffer bb) {
+ byte[] bytes = new byte[bb.position()];
+ for (int i = 0; i < bytes.length; i++)
+ bytes[i] = bb.get(i);
+ return bytes;
+ }
+
+ Result(CharBuffer ib, ByteBuffer ob, CoderResult cr) {
+ ipos = ib.position();
+ ia = toArray(ib);
+ oa = toArray(ob);
+ direct = ib.isDirect();
+ limit = ob.limit();
+ this.cr = cr;
+ }
+
+ static char[] toArray(CharBuffer b) {
+ int pos = b.position();
+ char[] a = new char[b.limit()];
+ b.position(0);
+ b.get(a);
+ b.position(pos);
+ return a;
+ }
+
+ static byte[] toArray(ByteBuffer b) {
+ byte[] a = new byte[b.position()];
+ b.position(0);
+ b.get(a);
+ return a;
+ }
+
+ static boolean eq(Result x, Result y) {
+ return x == y ||
+ (x != null && y != null &&
+ (Arrays.equals(x.oa, y.oa) &&
+ x.ipos == y.ipos &&
+ x.cr == y.cr));
+ }
+
+ public String toString() {
+ return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
+ string(ia), ipos, ia.length,
+ cr, string(oa), oa.length, limit,
+ (direct ? " (direct)" : ""));
+ }
+ }
+
+ static class CharsetTester {
+ private final Charset cs;
+ private final boolean hasBom;
+ private static final int maxFailures = 5;
+ private int failures = 0;
+ // private static final long maxCharsetFailures = Long.MAX_VALUE;
+ private static final long maxCharsetFailures = 10000L;
+ private final long failed0 = failed;
+
+ // legend: r=regular d=direct In=Input Ou=Output
+ static final int maxBufSize = 20;
+ static final CharBuffer[] rInBuffers = new CharBuffer[maxBufSize];
+ static final CharBuffer[] dInBuffers = new CharBuffer[maxBufSize];
+
+ static final ByteBuffer[] rOuBuffers = new ByteBuffer[maxBufSize];
+ static final ByteBuffer[] dOuBuffers = new ByteBuffer[maxBufSize];
+ static {
+ for (int i = 0; i < maxBufSize; i++) {
+ rInBuffers[i] = CharBuffer.allocate(i);
+ dInBuffers[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
+ rOuBuffers[i] = ByteBuffer.allocate(i);
+ dOuBuffers[i] = ByteBuffer.allocateDirect(i);
+ }
+ }
+
+ CharsetTester(Charset cs) {
+ this.cs = cs;
+ this.hasBom =
+ cs.name().matches(".*BOM.*") ||
+ cs.name().equals("UTF-16");
+ }
+
+ static boolean bug(String format, Object... args) {
+ return reporter.bug(format, args);
+ }
+
+ static boolean hasBom(byte[] a) {
+ switch (a.length) {
+ case 2: case 4:
+ int sum = 0;
+ for (byte x : a)
+ sum += x;
+ return sum == (byte) 0xfe + (byte) 0xff;
+ default: return false;
+ }
+ }
+
+ void testSurrogates() {
+ int failures = 0;
+ for (int i = 0; i < 10; i++) {
+ Result r = test(new char[] { randomHighSurrogate() });
+ if (r == null) break;
+ if (! (r.cr.isUnderflow() &&
+ r.ipos == 0))
+ bug("Lone high surrogate not UNDERFLOW: %s %s",
+ cs, r);
+ }
+ for (int i = 0; i < 10; i++) {
+ Result r = test(new char[] { randomLowSurrogate() });
+ if (r == null) break;
+ if (! (r.cr.isMalformed() && r.cr.length() == 1))
+ bug("Lone low surrogate not MALFORMED[1]: %s %s",
+ cs, r);
+ }
+ char[] chars = new char[2];
+ for (int i = 0; i < 10; i++) {
+ chars[0] = randomLowSurrogate(); // Always illegal
+ chars[1] = randomChar();
+ Result r = test(chars);
+ if (r == null) break;
+ if (! (r.cr.isMalformed() &&
+ r.cr.length() == 1 &&
+ (r.ipos == 0 || (hasBom && hasBom(r.oa))))) {
+ if (failures++ > 5) return;
+ bug("Unpaired low surrogate not MALFORMED[1]: %s %s",
+ cs, r);
+ }
+ }
+ for (int i = 0; i < 10; i++) {
+ chars[0] = randomHighSurrogate();
+ do {
+ chars[1] = randomChar();
+ } while (Character.isLowSurrogate(chars[1]));
+ Result r = test(chars);
+ if (r == null) break;
+ if (! (r.cr.isMalformed() &&
+ r.cr.length() == 1 &&
+ (r.ipos == 0 || (hasBom && hasBom(r.oa))))) {
+ if (failures++ > 5) return;
+ bug("Unpaired high surrogate not MALFORMED[1]: %s %s",
+ cs, r);
+ }
+ }
+ for (int i = 0; i < 1000; i++) {
+ chars[0] = randomHighSurrogate();
+ chars[1] = randomLowSurrogate();
+ Result r = test(chars);
+ if (r == null) break;
+ if (! ((r.cr.isUnmappable() &&
+ r.cr.length() == 2 &&
+ r.oa.length == 0)
+ ||
+ (r.cr.isUnderflow() &&
+ r.oa.length > 0 &&
+ r.ipos == 2))) {
+ if (failures++ > 5) return;
+ bug("Legal supplementary character bug: %s %s",
+ cs, r);
+ }
+ }
+ }
+
+// if (! (r.cr.isMalformed() &&
+// r.cr.length() == 1 &&
+// (rob.position() == 0 || hasBom(rob)))) {
+// if (failures++ > 5) return;
+// bug("Unpaired surrogate not malformed: %s %s",
+// cs, r);
+// }
+// }
+
+// dib.clear(); dib.put(chars); dib.flip();
+// rib.position(0);
+// rob.clear(); rob.limit(lim);
+// for (CharBuffer ib : new CharBuffer[] { rib, dib }) {
+// Result r = recode(ib, rob);
+// if (! (r.cr.isMalformed() &&
+// r.cr.length() == 1 &&
+// (rob.position() == 0 || hasBom(rob)))) {
+// if (failures++ > 5) return;
+// bug("Unpaired surrogate not malformed: %s %s",
+// cs, r);
+// }
+// }
+// //}
+// for (int i = 0; i < 10000; i++) {
+// chars[0] = randomHighSurrogate();
+// chars[1] = randomLowSurrogate();
+// dib.clear(); dib.put(chars); dib.flip();
+// rib.position(0);
+// rob.clear(); rob.limit(lim);
+// for (CharBuffer ib : new CharBuffer[] { rib, dib }) {
+// Result r = recode(ib, rob);
+// if (! ((r.cr.isUnmappable() &&
+// r.cr.length() == 2 &&
+// rob.position() == 0)
+// ||
+// (r.cr.isUnderflow() &&
+// rob.position() > 0 &&
+// ib.position() == 2))) {
+// if (failures++ > 5) return;
+// bug("Legal supplementary character bug: %s %s",
+// cs, r);
+// }
+// }
+// }
+// }
+// }
+
+ Result recode(CharBuffer ib, ByteBuffer ob) {
+ try {
+ byte canary = 22;
+ ib.clear(); // Prepare to read
+ ob.clear(); // Prepare to write
+ for (int i = 0; i < ob.limit(); i++)
+ ob.put(i, canary);
+ CharsetEncoder coder = cs.newEncoder();
+ CoderResult cr = coder.encode(ib, ob, false);
+ equal(ib.limit(), ib.capacity());
+ equal(ob.limit(), ob.capacity());
+ Result r = new Result(ib, ob, cr);
+ if (cr.isError())
+ check(cr.length() > 0);
+ if (cr.isOverflow() && ob.remaining() > 10)
+ bug("OVERFLOW, but there's lots of room: %s %s",
+ cs, r);
+// if (cr.isOverflow() && ib.remaining() == 0 && ! hasBom)
+// bug("OVERFLOW, yet remaining() == 0: %s %s",
+// cs, r);
+ if (cr.isError() && ib.remaining() < cr.length())
+ bug("remaining() < CoderResult.length(): %s %s",
+ cs, r);
+// if (ib.position() == 0
+// && ob.position() > 0
+// && ! hasBom(r.oa))
+// bug("output only if input consumed: %s %s",
+// cs, r);
+ CoderResult cr2 = coder.encode(ib, ob, false);
+ if (ib.position() != r.ipos ||
+ ob.position() != r.oa.length ||
+ cr != cr2)
+ bug("Coding operation not idempotent: %s%n %s%n %s",
+ cs, r, new Result(ib, ob, cr2));
+ if (ob.position() < ob.limit() &&
+ ob.get(ob.position()) != canary)
+ bug("Buffer overrun: %s %s %s",
+ cs, r, ob.get(ob.position()));
+ return r;
+ } catch (Throwable t) {
+ if (bug("Unexpected exception: %s %s %s",
+ cs, t.getClass().getSimpleName(),
+ new Result(ib, ob, null)))
+ t.printStackTrace();
+ return null;
+ }
+ }
+
+ Result recode2(char[] ia, int n) {
+ int len = ia.length;
+ CharBuffer rib = CharBuffer.wrap(ia);
+ CharBuffer dib = dInBuffers[len];
+ dib.clear(); dib.put(ia); dib.clear();
+ ByteBuffer rob = rOuBuffers[n];
+ ByteBuffer dob = dOuBuffers[n];
+ equal(rob.limit(), n);
+ equal(dob.limit(), n);
+ check(dib.isDirect());
+ check(dob.isDirect());
+ Result r1 = recode(rib, rob);
+ Result r2 = recode(dib, dob);
+ if (r1 != null && r2 != null && ! Result.eq(r1, r2))
+ bug("Results differ for direct buffers: %s%n %s%n %s",
+ cs, r1, r2);
+ return r1;
+ }
+
+ Result test(char[] ia) {
+ if (failed - failed0 >= maxCharsetFailures)
+ throw new TooManyFailures();
+
+ Result roomy = recode2(ia, maxBufSize - 1);
+ if (roomy == null) return roomy;
+ int olen = roomy.oa.length;
+ if (olen > 0) {
+ if (roomy.ipos == roomy.ia.length) {
+ Result perfectFit = recode2(ia, olen);
+ if (! Result.eq(roomy, perfectFit))
+ bug("Results differ: %s%n %s%n %s",
+ cs, roomy, perfectFit);
+ }
+ for (int i = 0; i < olen; i++) {
+ Result claustrophobic = recode2(ia, i);
+ if (claustrophobic == null) return roomy;
+ if (roomy.cr.isUnderflow() &&
+ ! claustrophobic.cr.isOverflow())
+ bug("Expected OVERFLOW: %s%n %s%n %s",
+ cs, roomy, claustrophobic);
+ }
+ }
+ return roomy;
+ }
+
+ void testExhaustively(char[] prefix, int n) {
+ int len = prefix.length;
+ char[] ia = Arrays.copyOf(prefix, len + 1);
+ for (int i = 0; i < 0x10000; i++) {
+ ia[len] = (char) i;
+ if (n == 1)
+ test(ia);
+ else
+ testExhaustively(ia, n - 1);
+ }
+ }
+
+ void testRandomly(char[] prefix, int n) {
+ int len = prefix.length;
+ char[] ia = Arrays.copyOf(prefix, len + n);
+ for (int i = 0; i < 10000; i++) {
+ for (int j = 0; j < n; j++)
+ ia[len + j] = randomChar();
+ test(ia);
+ }
+ }
+
+ void testPrefix(char[] prefix) {
+ if (prefix.length > 0)
+ System.out.printf("Testing prefix %s%n", string(prefix));
+
+ test(prefix);
+
+ testExhaustively(prefix, 1);
+ // Can you spare a year of CPU time?
+ //testExhaustively(prefix, 2);
+
+ testRandomly(prefix, 2);
+ testRandomly(prefix, 3);
+ }
+ }
+
+ private final static Random rnd = RandomFactory.getRandom();
+ private static char randomChar() {
+ return (char) rnd.nextInt(Character.MAX_VALUE);
+ }
+ private static char randomHighSurrogate() {
+ return (char) (Character.MIN_HIGH_SURROGATE + rnd.nextInt(1024));
+ }
+ private static char randomLowSurrogate() {
+ return (char) (Character.MIN_LOW_SURROGATE + rnd.nextInt(1024));
+ }
+
+ private static void testCharset(Charset cs) throws Throwable {
+ if (! cs.canEncode())
+ return;
+
+ final String csn = cs.name();
+
+ if (isBroken(csn)) {
+ System.out.printf("Skipping possibly broken charset %s%n", csn);
+ return;
+ }
+ System.out.println(csn);
+
+ CharsetTester tester = new CharsetTester(cs);
+
+ tester.testSurrogates();
+
+ tester.testPrefix(new char[] {});
+
+ if (csn.equals("x-ISCII91")) {
+ System.out.println("More ISCII testing...");
+ new CharsetTester(cs).testPrefix(new char[]{'\u094d'}); // Halant
+ new CharsetTester(cs).testPrefix(new char[]{'\u093c'}); // Nukta
+ }
+ }
+
+ private static void realMain(String[] args) {
+ for (Charset cs : sort(Charset.availableCharsets().values())) {
+ try {
+ testCharset(cs);
+ } catch (TooManyFailures e) {
+ System.out.printf("Too many failures for %s%n", cs);
+ } catch (Throwable t) {
+ unexpected(t);
+ }
+ }
+ reporter.summarize();
+ }
+
+ //--------------------- Infrastructure ---------------------------
+ static volatile long passed = 0, failed = 0;
+ static void pass() {passed++;}
+ static void fail() {failed++; Thread.dumpStack();}
+ static void fail(String format, Object... args) {
+ System.out.println(String.format(format, args)); failed++;}
+ static void fail(String msg) {System.out.println(msg); fail();}
+ static void unexpected(Throwable t) {failed++; t.printStackTrace();}
+ static void check(boolean cond) {if (cond) pass(); else fail();}
+ static void equal(Object x, Object y) {
+ if (x == null ? y == null : x.equals(y)) pass();
+ else fail(x + " not equal to " + y);}
+ public static void main(String[] args) throws Throwable {
+ try {realMain(args);} catch (Throwable t) {unexpected(t);}
+ System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
+ if (failed > 0) throw new AssertionError("Some tests failed");}
+}