4206909: want java.util.zip to work for interactive use (Z_SYNC_FLUSH)
Summary: Add sync_flush option into Deflater/DefalterOutputStream
Reviewed-by: martin, alanb
--- a/jdk/src/share/classes/java/util/zip/Deflater.java Wed Oct 21 11:40:40 2009 -0700
+++ b/jdk/src/share/classes/java/util/zip/Deflater.java Wed Oct 21 11:50:25 2009 -0700
@@ -122,6 +122,33 @@
*/
public static final int DEFAULT_STRATEGY = 0;
+ /**
+ * Compression flush mode used to achieve best compression result.
+ *
+ * @see Deflater#deflate(byte[], int, int, int)
+ * @since 1.7
+ */
+ public static final int NO_FLUSH = 0;
+
+ /**
+ * Compression flush mode used to flush out all pending output; may
+ * degrade compression for some compression algorithms.
+ *
+ * @see Deflater#deflate(byte[], int, int, int)
+ * @since 1.7
+ */
+ public static final int SYNC_FLUSH = 2;
+
+ /**
+ * Compression flush mode used to flush out all pending output and
+ * reset the deflater. Using this mode too often can seriously degrade
+ * compression.
+ *
+ * @see Deflater#deflate(byte[], int, int, int)
+ * @since 1.7
+ */
+ public static final int FULL_FLUSH = 3;
+
static {
/* Zip library is loaded from System.initializeSystemClass */
initIDs();
@@ -289,35 +316,100 @@
}
/**
- * Fills specified buffer with compressed data. Returns actual number
- * of bytes of compressed data. A return value of 0 indicates that
- * needsInput() should be called in order to determine if more input
- * data is required.
+ * Compresses the input data and fills specified buffer with compressed
+ * data. Returns actual number of bytes of compressed data. A return value
+ * of 0 indicates that {@link needsInput() needsInput} should be called
+ * in order to determine if more input data is required.
+ *
+ * <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
+ * An invocation of this method of the form {@code deflater.deflate(b, off, len)}
+ * yields the same result as the invocation of
+ * {@code deflater.deflate(b, off, len, Deflater.NO_FLUSH)}.
+ *
* @param b the buffer for the compressed data
* @param off the start offset of the data
* @param len the maximum number of bytes of compressed data
- * @return the actual number of bytes of compressed data
+ * @return the actual number of bytes of compressed data written to the
+ * output buffer
+ */
+ public int deflate(byte[] b, int off, int len) {
+ return deflateBytes(b, off, len, NO_FLUSH);
+ }
+
+ /**
+ * Compresses the input data and fills specified buffer with compressed
+ * data. Returns actual number of bytes of compressed data. A return value
+ * of 0 indicates that {@link needsInput() needsInput} should be called
+ * in order to determine if more input data is required.
+ *
+ * <p>This method uses {@link #NO_FLUSH} as its compression flush mode.
+ * An invocation of this method of the form {@code deflater.deflate(b)}
+ * yields the same result as the invocation of
+ * {@code deflater.deflate(b, 0, b.length, Deflater.NO_FLUSH)}.
+ *
+ * @param b the buffer for the compressed data
+ * @return the actual number of bytes of compressed data written to the
+ * output buffer
*/
- public synchronized int deflate(byte[] b, int off, int len) {
+ public int deflate(byte[] b) {
+ return deflate(b, 0, b.length, NO_FLUSH);
+ }
+
+ /**
+ * Compresses the input data and fills the specified buffer with compressed
+ * data. Returns actual number of bytes of data compressed.
+ *
+ * <p>Compression flush mode is one of the following three modes:
+ *
+ * <ul>
+ * <li>{@link #NO_FLUSH}: allows the deflater to decide how much data
+ * to accumulate, before producing output, in order to achieve the best
+ * compression (should be used in normal use scenario). A return value
+ * of 0 in this flush mode indicates that {@link #needsInput()} should
+ * be called in order to determine if more input data is required.
+ *
+ * <li>{@link #SYNC_FLUSH}: all pending output in the deflater is flushed,
+ * to the specified output buffer, so that an inflater that works on
+ * compressed data can get all input data available so far (In particular
+ * the {@link #needsInput()} returns {@code true} after this invocation
+ * if enough output space is provided). Flushing with {@link #SYNC_FLUSH}
+ * may degrade compression for some compression algorithms and so it
+ * should be used only when necessary.
+ *
+ * <li>{@link #FULL_FLUSH}: all pending output is flushed out as with
+ * {@link #SYNC_FLUSH}. The compression state is reset so that the inflater
+ * that works on the compressed output data can restart from this point
+ * if previous compressed data has been damaged or if random access is
+ * desired. Using {@link #FULL_FLUSH} too often can seriously degrade
+ * compression.
+ * </ul>
+ *
+ * <p>In the case of {@link #FULL_FLUSH} or {@link #SYNC_FLUSH}, if
+ * the return value is {@code len}, the space available in output
+ * buffer {@code b}, this method should be invoked again with the same
+ * {@code flush} parameter and more output space.
+ *
+ * @param b the buffer for the compressed data
+ * @param off the start offset of the data
+ * @param len the maximum number of bytes of compressed data
+ * @param flush the compression flush mode
+ * @return the actual number of bytes of compressed data written to
+ * the output buffer
+ *
+ * @throws IllegalArgumentException if the flush mode is invalid
+ * @since 1.7
+ */
+ public synchronized int deflate(byte[] b, int off, int len, int flush) {
if (b == null) {
throw new NullPointerException();
}
if (off < 0 || len < 0 || off > b.length - len) {
throw new ArrayIndexOutOfBoundsException();
}
- return deflateBytes(b, off, len);
- }
-
- /**
- * Fills specified buffer with compressed data. Returns actual number
- * of bytes of compressed data. A return value of 0 indicates that
- * needsInput() should be called in order to determine if more input
- * data is required.
- * @param b the buffer for the compressed data
- * @return the actual number of bytes of compressed data
- */
- public int deflate(byte[] b) {
- return deflate(b, 0, b.length);
+ if (flush == NO_FLUSH || flush == SYNC_FLUSH ||
+ flush == FULL_FLUSH)
+ return deflateBytes(b, off, len, flush);
+ throw new IllegalArgumentException();
}
/**
@@ -420,7 +512,7 @@
private native static long init(int level, int strategy, boolean nowrap);
private native static void setDictionary(long strm, byte[] b, int off,
int len);
- private native int deflateBytes(byte[] b, int off, int len);
+ private native int deflateBytes(byte[] b, int off, int len, int flush);
private native static int getAdler(long strm);
private native static long getBytesRead(long strm);
private native static long getBytesWritten(long strm);
--- a/jdk/src/share/classes/java/util/zip/DeflaterOutputStream.java Wed Oct 21 11:40:40 2009 -0700
+++ b/jdk/src/share/classes/java/util/zip/DeflaterOutputStream.java Wed Oct 21 11:50:25 2009 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright 1996-2006 Sun Microsystems, Inc. All Rights Reserved.
+ * Copyright 1996-2009 Sun Microsystems, Inc. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -56,15 +56,29 @@
private boolean closed = false;
+ private final boolean syncFlush;
+
/**
- * Creates a new output stream with the specified compressor and
- * buffer size.
+ * Creates a new output stream with the specified compressor,
+ * buffer size and flush mode.
+
* @param out the output stream
* @param def the compressor ("deflater")
* @param size the output buffer size
- * @exception IllegalArgumentException if size is <= 0
+ * @param syncFlush
+ * if {@code true} the {@link flush()} method of this
+ * instance flushes the compressor with flush mode
+ * {@link Deflater#SYNC_FLUSH} before flushing the output
+ * stream, otherwise only flushes the output stream
+ *
+ * @throws IllegalArgumentException if size is <= 0
+ *
+ * @since 1.7
*/
- public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
+ public DeflaterOutputStream(OutputStream out,
+ Deflater def,
+ int size,
+ boolean syncFlush) {
super(out);
if (out == null || def == null) {
throw new NullPointerException();
@@ -72,27 +86,93 @@
throw new IllegalArgumentException("buffer size <= 0");
}
this.def = def;
- buf = new byte[size];
+ this.buf = new byte[size];
+ this.syncFlush = syncFlush;
+ }
+
+
+ /**
+ * Creates a new output stream with the specified compressor and
+ * buffer size.
+ *
+ * <p>The new output stream instance is created as if by invoking
+ * the 4-argument constructor DeflaterOutputStream(out, def, size, false).
+ *
+ * @param out the output stream
+ * @param def the compressor ("deflater")
+ * @param size the output buffer size
+ * @exception IllegalArgumentException if size is <= 0
+ */
+ public DeflaterOutputStream(OutputStream out, Deflater def, int size) {
+ this(out, def, size, false);
}
/**
+ * Creates a new output stream with the specified compressor, flush
+ * mode and a default buffer size.
+ *
+ * @param out the output stream
+ * @param def the compressor ("deflater")
+ * @param syncFlush
+ * if {@code true} the {@link flush()} method of this
+ * instance flushes the compressor with flush mode
+ * {@link Deflater#SYNC_FLUSH} before flushing the output
+ * stream, otherwise only flushes the output stream
+ *
+ * @since 1.7
+ */
+ public DeflaterOutputStream(OutputStream out,
+ Deflater def,
+ boolean syncFlush) {
+ this(out, def, 512, syncFlush);
+ }
+
+
+ /**
* Creates a new output stream with the specified compressor and
* a default buffer size.
+ *
+ * <p>The new output stream instance is created as if by invoking
+ * the 3-argument constructor DeflaterOutputStream(out, def, false).
+ *
* @param out the output stream
* @param def the compressor ("deflater")
*/
public DeflaterOutputStream(OutputStream out, Deflater def) {
- this(out, def, 512);
+ this(out, def, 512, false);
}
boolean usesDefaultDeflater = false;
+
+ /**
+ * Creates a new output stream with a default compressor, a default
+ * buffer size and the specified flush mode.
+ *
+ * @param out the output stream
+ * @param syncFlush
+ * if {@code true} the {@link flush()} method of this
+ * instance flushes the compressor with flush mode
+ * {@link Deflater#SYNC_FLUSH} before flushing the output
+ * stream, otherwise only flushes the output stream
+ *
+ * @since 1.7
+ */
+ public DeflaterOutputStream(OutputStream out, boolean syncFlush) {
+ this(out, new Deflater(), 512, syncFlush);
+ usesDefaultDeflater = true;
+ }
+
/**
* Creates a new output stream with a default compressor and buffer size.
+ *
+ * <p>The new output stream instance is created as if by invoking
+ * the 2-argument constructor DeflaterOutputStream(out, false).
+ *
* @param out the output stream
*/
public DeflaterOutputStream(OutputStream out) {
- this(out, new Deflater());
+ this(out, false);
usesDefaultDeflater = true;
}
@@ -178,4 +258,32 @@
out.write(buf, 0, len);
}
}
+
+ /**
+ * Flushes the compressed output stream.
+ *
+ * If {@link DeflaterOutputStream(OutputStream, Deflater, int, boolean)
+ * syncFlush} is {@code true} when this compressed output stream is
+ * constructed this method flushes the underlying {@code compressor}
+ * first with the flush mode {@link Deflater#SYNC_FLUSH} to force
+ * all pending data to be flushed out to the output stream and then
+ * flushes the output stream. Otherwise this method only flushes the
+ * output stream without flushing the {@code compressor}.
+ *
+ * @throws IOException if an I/O error has occurred
+ *
+ * @since 1.7
+ */
+ public void flush() throws IOException {
+ if (syncFlush && !def.finished()) {
+ int len = 0;
+ while ((len = def.deflate(buf, 0, buf.length, Deflater.SYNC_FLUSH)) > 0)
+ {
+ out.write(buf, 0, len);
+ if (len < buf.length)
+ break;
+ }
+ }
+ out.flush();
+ }
}
--- a/jdk/src/share/native/java/util/zip/Deflater.c Wed Oct 21 11:40:40 2009 -0700
+++ b/jdk/src/share/native/java/util/zip/Deflater.c Wed Oct 21 11:50:25 2009 -0700
@@ -118,7 +118,7 @@
JNIEXPORT jint JNICALL
Java_java_util_zip_Deflater_deflateBytes(JNIEnv *env, jobject this,
- jarray b, jint off, jint len)
+ jarray b, jint off, jint len, jint flush)
{
z_stream *strm = jlong_to_ptr((*env)->GetLongField(env, this, strmID));
@@ -197,7 +197,7 @@
strm->next_out = (Bytef *) out_buf;
strm->avail_in = this_len;
strm->avail_out = len;
- res = deflate(strm, finish ? Z_FINISH : Z_NO_FLUSH);
+ res = deflate(strm, finish ? Z_FINISH : flush);
if (res == Z_STREAM_END || res == Z_OK) {
(*env)->SetByteArrayRegion(env, b, off, len - strm->avail_out, out_buf);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/java/util/zip/InflateIn_DeflateOut.java Wed Oct 21 11:50:25 2009 -0700
@@ -0,0 +1,251 @@
+/*
+ * Copyright 2009 Google, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/**
+ * @test
+ * @bug 4206909
+ * @summary Test basic functionality of DeflaterOutputStream and InflaterInputStream including flush
+ */
+
+import java.io.*;
+import java.util.*;
+import java.util.zip.*;
+
+public class InflateIn_DeflateOut {
+
+ private static class PairedInputStream extends ByteArrayInputStream {
+ private PairedOutputStream out = null;
+ private Random random;
+
+ public PairedInputStream() {
+ // The ByteArrayInputStream needs to start with a buffer, but we
+ // need to set it to have no data
+ super(new byte[1]);
+ count = 0;
+ pos = 0;
+ random = new Random(new Date().getTime());
+ }
+
+ public void setPairedOutputStream(PairedOutputStream out) {
+ this.out = out;
+ }
+
+ private void maybeFlushPair() {
+ if (random.nextInt(100) < 10) {
+ out.flush();
+ }
+ }
+
+ public int read() {
+ maybeFlushPair();
+ return super.read();
+ }
+
+ public int read(byte b[], int off, int len) {
+ maybeFlushPair();
+ return super.read(b, off, len);
+ }
+
+ public void addBytes(byte[] bytes, int len) {
+ int oldavail = count - pos;
+ int newcount = oldavail + len;
+ byte[] newbuf = new byte[newcount];
+ System.arraycopy(buf, pos, newbuf, 0, oldavail);
+ System.arraycopy(bytes, 0, newbuf, oldavail, len);
+ pos = 0;
+ count = newcount;
+ buf = newbuf;
+ }
+ }
+
+ private static class PairedOutputStream extends ByteArrayOutputStream {
+ private PairedInputStream pairedStream = null;
+
+ public PairedOutputStream(PairedInputStream inputPair) {
+ super();
+ this.pairedStream = inputPair;
+ }
+
+ public void flush() {
+ if (count > 0) {
+ pairedStream.addBytes(buf, count);
+ reset();
+ }
+ }
+
+ public void close() {
+ flush();
+ }
+ }
+
+ private static boolean readFully(InputStream in, byte[] buf, int length)
+ throws IOException {
+ int pos = 0;
+ int n;
+ while ((n = in.read(buf, pos, length - pos)) > 0) {
+ pos += n;
+ if (pos == length) return true;
+ }
+ return false;
+ }
+
+ private static boolean readLineIfAvailable(InputStream in, StringBuilder sb)
+ throws IOException {
+ try {
+ while (in.available() > 0) {
+ int i = in.read();
+ if (i < 0) break;
+ char c = (char) (((byte) i) & 0xff);
+ sb.append(c);
+ if (c == '\n') return true;
+ }
+ } catch (EOFException e) {
+ // empty
+ }
+ return false;
+ }
+
+ /** Check that written, closed and read */
+ private static void WriteCloseRead() throws Throwable {
+ Random random = new Random(new Date().getTime());
+
+ PairedInputStream pis = new PairedInputStream();
+ InflaterInputStream iis = new InflaterInputStream(pis);
+
+ PairedOutputStream pos = new PairedOutputStream(pis);
+ pis.setPairedOutputStream(pos);
+ DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+ byte[] data = new byte[random.nextInt(1024 * 1024)];
+ byte[] buf = new byte[data.length];
+ random.nextBytes(data);
+
+ dos.write(data);
+ dos.close();
+ check(readFully(iis, buf, buf.length));
+ check(Arrays.equals(data, buf));
+ }
+
+ /** Check that written, flushed and read */
+ private static void WriteFlushRead() throws Throwable {
+ Random random = new Random(new Date().getTime());
+
+ PairedInputStream pis = new PairedInputStream();
+ InflaterInputStream iis = new InflaterInputStream(pis);
+
+ PairedOutputStream pos = new PairedOutputStream(pis);
+ pis.setPairedOutputStream(pos);
+ DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+ // Large writes
+ for (int x = 0; x < 200 ; x++) {
+ // byte[] data = new byte[random.nextInt(1024 * 1024)];
+ byte[] data = new byte[1024];
+ byte[] buf = new byte[data.length];
+ random.nextBytes(data);
+
+ dos.write(data);
+ dos.flush();
+ check(readFully(iis, buf, buf.length));
+ check(Arrays.equals(data, buf));
+ }
+
+ // Small writes
+ for (int x = 0; x < 2000 ; x++) {
+ byte[] data = new byte[random.nextInt(20) + 10];
+ byte[] buf = new byte[data.length];
+ random.nextBytes(data);
+
+ dos.write(data);
+ dos.flush();
+ if (!readFully(iis, buf, buf.length)) {
+ fail("Didn't read full buffer of " + buf.length);
+ }
+ check(Arrays.equals(data, buf));
+ }
+
+ String quit = "QUIT\r\n";
+
+ // Close it out
+ dos.write(quit.getBytes());
+ dos.close();
+
+ StringBuilder sb = new StringBuilder();
+ check(readLineIfAvailable(iis, sb));
+ equal(sb.toString(), quit);
+ }
+
+ /** Validate that we need to use flush at least once on a line
+ * oriented protocol */
+ private static void LineOrientedProtocol() throws Throwable {
+ PairedInputStream pis = new PairedInputStream();
+ InflaterInputStream iis = new InflaterInputStream(pis);
+
+ PairedOutputStream pos = new PairedOutputStream(pis);
+ pis.setPairedOutputStream(pos);
+ DeflaterOutputStream dos = new DeflaterOutputStream(pos, true);
+
+ boolean flushed = false;
+ int count = 0;
+
+ // Do at least a certain number of lines, but too many without a
+ // flush means this test isn't testing anything
+ while ((count < 10 && flushed) || (count < 1000 && !flushed)) {
+ String command = "PING " + count + "\r\n";
+ dos.write(command.getBytes());
+
+ StringBuilder buf = new StringBuilder();
+ if (!readLineIfAvailable(iis, buf)) {
+ flushed = true;
+ dos.flush();
+ check(readLineIfAvailable(iis, buf));
+ }
+ equal(buf.toString(), command);
+ count++;
+ }
+ check(flushed);
+ }
+
+ public static void realMain(String[] args) throws Throwable {
+ WriteCloseRead();
+
+ WriteFlushRead();
+
+ LineOrientedProtocol();
+ }
+
+ //--------------------- Infrastructure ---------------------------
+ static volatile int passed = 0, failed = 0;
+ static void pass() {passed++;}
+ static void fail() {failed++; Thread.dumpStack();}
+ static void fail(String msg) {System.out.println(msg); fail();}
+ static void unexpected(Throwable t) {failed++; t.printStackTrace();}
+ static void check(boolean cond) {if (cond) pass(); else fail();}
+ static void equal(Object x, Object y) {
+ if (x == null ? y == null : x.equals(y)) pass();
+ else fail(x + " not equal to " + y);}
+ public static void main(String[] args) throws Throwable {
+ try {realMain(args);} catch (Throwable t) {unexpected(t);}
+ System.out.println("\nPassed = " + passed + " failed = " + failed);
+ if (failed > 0) throw new AssertionError("Some tests failed");}
+}