8034802: (zipfs) newFileSystem throws UOE when the zip file is located in a custom file system
Reviewed-by: xiaofeya, clanger
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ByteArrayChannel.java Tue Sep 18 10:43:01 2018 -0700
@@ -0,0 +1,252 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation. Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.nio.zipfs;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.NonWritableChannelException;
+import java.nio.channels.SeekableByteChannel;
+import java.util.Arrays;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class ByteArrayChannel implements SeekableByteChannel {
+
+ private final ReadWriteLock rwlock = new ReentrantReadWriteLock();
+ private byte buf[];
+
+ /*
+ * The current position of this channel.
+ */
+ private int pos;
+
+ /*
+ * The index that is one greater than the last valid byte in the channel.
+ */
+ private int last;
+
+ private boolean closed;
+ private boolean readonly;
+
+ /*
+ * Creates a {@code ByteArrayChannel} with size {@code sz}.
+ */
+ ByteArrayChannel(int sz, boolean readonly) {
+ this.buf = new byte[sz];
+ this.pos = this.last = 0;
+ this.readonly = readonly;
+ }
+
+ /*
+ * Creates a ByteArrayChannel with its 'pos' at 0 and its 'last' at buf's end.
+ * Note: no defensive copy of the 'buf', used directly.
+ */
+ ByteArrayChannel(byte[] buf, boolean readonly) {
+ this.buf = buf;
+ this.pos = 0;
+ this.last = buf.length;
+ this.readonly = readonly;
+ }
+
+ @Override
+ public boolean isOpen() {
+ return !closed;
+ }
+
+ @Override
+ public long position() throws IOException {
+ beginRead();
+ try {
+ ensureOpen();
+ return pos;
+ } finally {
+ endRead();
+ }
+ }
+
+ @Override
+ public SeekableByteChannel position(long pos) throws IOException {
+ beginWrite();
+ try {
+ ensureOpen();
+ if (pos < 0 || pos >= Integer.MAX_VALUE)
+ throw new IllegalArgumentException("Illegal position " + pos);
+ this.pos = Math.min((int)pos, last);
+ return this;
+ } finally {
+ endWrite();
+ }
+ }
+
+ @Override
+ public int read(ByteBuffer dst) throws IOException {
+ beginWrite();
+ try {
+ ensureOpen();
+ if (pos == last)
+ return -1;
+ int n = Math.min(dst.remaining(), last - pos);
+ dst.put(buf, pos, n);
+ pos += n;
+ return n;
+ } finally {
+ endWrite();
+ }
+ }
+
+ @Override
+ public SeekableByteChannel truncate(long size) throws IOException {
+ if (readonly)
+ throw new NonWritableChannelException();
+ ensureOpen();
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int write(ByteBuffer src) throws IOException {
+ if (readonly)
+ throw new NonWritableChannelException();
+ beginWrite();
+ try {
+ ensureOpen();
+ int n = src.remaining();
+ ensureCapacity(pos + n);
+ src.get(buf, pos, n);
+ pos += n;
+ if (pos > last) {
+ last = pos;
+ }
+ return n;
+ } finally {
+ endWrite();
+ }
+ }
+
+ @Override
+ public long size() throws IOException {
+ beginRead();
+ try {
+ ensureOpen();
+ return last;
+ } finally {
+ endRead();
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed)
+ return;
+ beginWrite();
+ try {
+ closed = true;
+ buf = null;
+ pos = 0;
+ last = 0;
+ } finally {
+ endWrite();
+ }
+ }
+
+ /**
+ * Creates a newly allocated byte array. Its size is the current
+ * size of this channel and the valid contents of the buffer
+ * have been copied into it.
+ *
+ * @return the current contents of this channel, as a byte array.
+ */
+ public byte[] toByteArray() {
+ beginRead();
+ try {
+ // avoid copy if last == bytes.length?
+ return Arrays.copyOf(buf, last);
+ } finally {
+ endRead();
+ }
+ }
+
+ private void ensureOpen() throws IOException {
+ if (closed)
+ throw new ClosedChannelException();
+ }
+
+ private final void beginWrite() {
+ rwlock.writeLock().lock();
+ }
+
+ private final void endWrite() {
+ rwlock.writeLock().unlock();
+ }
+
+ private final void beginRead() {
+ rwlock.readLock().lock();
+ }
+
+ private final void endRead() {
+ rwlock.readLock().unlock();
+ }
+
+ private void ensureCapacity(int minCapacity) {
+ // overflow-conscious code
+ if (minCapacity - buf.length > 0) {
+ grow(minCapacity);
+ }
+ }
+
+ /**
+ * The maximum size of array to allocate.
+ * Some VMs reserve some header words in an array.
+ * Attempts to allocate larger arrays may result in
+ * OutOfMemoryError: Requested array size exceeds VM limit
+ */
+ private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
+
+ /**
+ * Increases the capacity to ensure that it can hold at least the
+ * number of elements specified by the minimum capacity argument.
+ *
+ * @param minCapacity the desired minimum capacity
+ */
+ private void grow(int minCapacity) {
+ // overflow-conscious code
+ int oldCapacity = buf.length;
+ int newCapacity = oldCapacity << 1;
+ if (newCapacity - minCapacity < 0)
+ newCapacity = minCapacity;
+ if (newCapacity - MAX_ARRAY_SIZE > 0)
+ newCapacity = hugeCapacity(minCapacity);
+ buf = Arrays.copyOf(buf, newCapacity);
+ }
+
+ private static int hugeCapacity(int minCapacity) {
+ if (minCapacity < 0) // overflow
+ throw new OutOfMemoryError();
+ return (minCapacity > MAX_ARRAY_SIZE) ?
+ Integer.MAX_VALUE :
+ MAX_ARRAY_SIZE;
+ }
+}
--- a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java Tue Sep 18 09:44:20 2018 -0700
+++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystem.java Tue Sep 18 10:43:01 2018 -0700
@@ -30,6 +30,7 @@
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.File;
+import java.io.FilterOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -70,33 +71,34 @@
private final ZipFileSystemProvider provider;
private final Path zfpath;
final ZipCoder zc;
+ private final ZipPath rootdir;
+ private boolean readOnly = false; // readonly file system
+
+ // configurable by env map
private final boolean noExtt; // see readExtra()
- private final ZipPath rootdir;
- // configurable by env map
private final boolean useTempFile; // use a temp file for newOS, default
// is to use BAOS for better performance
- private boolean readOnly = false; // readonly file system
private static final boolean isWindows = AccessController.doPrivileged(
(PrivilegedAction<Boolean>) () -> System.getProperty("os.name")
.startsWith("Windows"));
private final boolean forceEnd64;
+ private final int defaultMethod; // METHOD_STORED if "noCompression=true"
+ // METHOD_DEFLATED otherwise
ZipFileSystem(ZipFileSystemProvider provider,
Path zfpath,
Map<String, ?> env) throws IOException
{
- // create a new zip if not exists
- boolean createNew = "true".equals(env.get("create"));
// default encoding for name/comment
String nameEncoding = env.containsKey("encoding") ?
(String)env.get("encoding") : "UTF-8";
this.noExtt = "false".equals(env.get("zipinfo-time"));
- this.useTempFile = TRUE.equals(env.get("useTempFile"));
- this.forceEnd64 = "true".equals(env.get("forceZIP64End"));
- this.provider = provider;
- this.zfpath = zfpath;
+ this.useTempFile = isTrue(env, "useTempFile");
+ this.forceEnd64 = isTrue(env, "forceZIP64End");
+ this.defaultMethod = isTrue(env, "noCompression") ? METHOD_STORED: METHOD_DEFLATED;
if (Files.notExists(zfpath)) {
- if (createNew) {
+ // create a new zip if not exists
+ if (isTrue(env, "create")) {
try (OutputStream os = Files.newOutputStream(zfpath, CREATE_NEW, WRITE)) {
new END().write(os, 0, forceEnd64);
}
@@ -122,6 +124,13 @@
}
throw x;
}
+ this.provider = provider;
+ this.zfpath = zfpath;
+ }
+
+ // returns true if there is a name=true/"true" setting in env
+ private static boolean isTrue(Map<String, ?> env, String name) {
+ return "true".equals(env.get(name)) || TRUE.equals(env.get(name));
}
@Override
@@ -254,22 +263,23 @@
try {
if (!isOpen)
return;
- isOpen = false; // set closed
+ isOpen = false; // set closed
} finally {
endWrite();
}
- if (!streams.isEmpty()) { // unlock and close all remaining streams
+ if (!streams.isEmpty()) { // unlock and close all remaining streams
Set<InputStream> copy = new HashSet<>(streams);
for (InputStream is: copy)
is.close();
}
- beginWrite(); // lock and sync
+ beginWrite(); // lock and sync
try {
AccessController.doPrivileged((PrivilegedExceptionAction<Void>) () -> {
sync(); return null;
});
- ch.close(); // close the ch just in case no update
- } catch (PrivilegedActionException e) { // and sync dose not close the ch
+ ch.close(); // close the ch just in case no update
+ // and sync didn't close the ch
+ } catch (PrivilegedActionException e) {
throw (IOException)e.getException();
} finally {
endWrite();
@@ -316,8 +326,8 @@
IndexNode inode = getInode(path);
if (inode == null)
return null;
- e = new Entry(inode.name, inode.isdir); // pseudo directory
- e.method = METHOD_STORED; // STORED for dir
+ // pseudo directory, uses METHOD_STORED
+ e = new Entry(inode.name, inode.isdir, METHOD_STORED);
e.mtime = e.atime = e.ctime = zfsDefaultTimeStamp;
}
} finally {
@@ -425,8 +435,7 @@
if (dir.length == 0 || exists(dir)) // root dir, or exiting dir
throw new FileAlreadyExistsException(getString(dir));
checkParents(dir);
- Entry e = new Entry(dir, Entry.NEW, true);
- e.method = METHOD_STORED; // STORED for dir
+ Entry e = new Entry(dir, Entry.NEW, true, METHOD_STORED);
update(e);
} finally {
endWrite();
@@ -467,7 +476,7 @@
checkParents(dst);
}
Entry u = new Entry(eSrc, Entry.COPY); // copy eSrc entry
- u.name(dst); // change name
+ u.name(dst); // change name
if (eSrc.type == Entry.NEW || eSrc.type == Entry.FILECH)
{
u.type = eSrc.type; // make it the same type
@@ -527,7 +536,7 @@
if (hasAppend) {
InputStream is = getInputStream(e);
OutputStream os = getOutputStream(new Entry(e, Entry.NEW));
- copyStream(is, os);
+ is.transferTo(os);
is.close();
return os;
}
@@ -536,7 +545,7 @@
if (!hasCreate && !hasCreateNew)
throw new NoSuchFileException(getString(path));
checkParents(path);
- return getOutputStream(new Entry(path, Entry.NEW, false));
+ return getOutputStream(new Entry(path, Entry.NEW, false, defaultMethod));
}
} finally {
endRead();
@@ -572,6 +581,37 @@
throw new IllegalArgumentException("APPEND + TRUNCATE_EXISTING not allowed");
}
+
+ // Returns an output SeekableByteChannel for either
+ // (1) writing the contents of a new entry, if the entry doesn't exit, or
+ // (2) updating/replacing the contents of an existing entry.
+ // Note: The content is not compressed.
+ private class EntryOutputChannel extends ByteArrayChannel {
+ Entry e;
+
+ EntryOutputChannel(Entry e) throws IOException {
+ super(e.size > 0? (int)e.size : 8192, false);
+ this.e = e;
+ if (e.mtime == -1)
+ e.mtime = System.currentTimeMillis();
+ if (e.method == -1)
+ e.method = defaultMethod;
+ // store size, compressed size, and crc-32 in datadescriptor
+ e.flag = FLAG_DATADESCR;
+ if (zc.isUTF8())
+ e.flag |= FLAG_USE_UTF8;
+ }
+
+ @Override
+ public void close() throws IOException {
+ e.bytes = toByteArray();
+ e.size = e.bytes.length;
+ e.crc = -1;
+ super.close();
+ update(e);
+ }
+ }
+
// Returns a Writable/ReadByteChannel for now. Might consdier to use
// newFileChannel() instead, which dump the entry data into a regular
// file on the default file system and create a FileChannel on top of
@@ -585,57 +625,36 @@
if (options.contains(StandardOpenOption.WRITE) ||
options.contains(StandardOpenOption.APPEND)) {
checkWritable();
- beginRead();
+ beginRead(); // only need a readlock, the "update()" will obtain
+ // thewritelock when the channel is closed
try {
- final WritableByteChannel wbc = Channels.newChannel(
- newOutputStream(path, options.toArray(new OpenOption[0])));
- long leftover = 0;
- if (options.contains(StandardOpenOption.APPEND)) {
- Entry e = getEntry(path);
- if (e != null && e.size >= 0)
- leftover = e.size;
- }
- final long offset = leftover;
- return new SeekableByteChannel() {
- long written = offset;
- public boolean isOpen() {
- return wbc.isOpen();
- }
-
- public long position() throws IOException {
- return written;
- }
-
- public SeekableByteChannel position(long pos)
- throws IOException
- {
- throw new UnsupportedOperationException();
+ ensureOpen();
+ Entry e = getEntry(path);
+ if (e != null) {
+ if (e.isDir() || options.contains(CREATE_NEW))
+ throw new FileAlreadyExistsException(getString(path));
+ SeekableByteChannel sbc =
+ new EntryOutputChannel(new Entry(e, Entry.NEW));
+ if (options.contains(APPEND)) {
+ try (InputStream is = getInputStream(e)) { // copyover
+ byte[] buf = new byte[8192];
+ ByteBuffer bb = ByteBuffer.wrap(buf);
+ int n;
+ while ((n = is.read(buf)) != -1) {
+ bb.position(0);
+ bb.limit(n);
+ sbc.write(bb);
+ }
+ }
}
-
- public int read(ByteBuffer dst) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- public SeekableByteChannel truncate(long size)
- throws IOException
- {
- throw new UnsupportedOperationException();
- }
+ return sbc;
+ }
+ if (!options.contains(CREATE) && !options.contains(CREATE_NEW))
+ throw new NoSuchFileException(getString(path));
+ checkParents(path);
+ return new EntryOutputChannel(
+ new Entry(path, Entry.NEW, false, defaultMethod));
- public int write(ByteBuffer src) throws IOException {
- int n = wbc.write(src);
- written += n;
- return n;
- }
-
- public long size() throws IOException {
- return written;
- }
-
- public void close() throws IOException {
- wbc.close();
- }
- };
} finally {
endRead();
}
@@ -646,51 +665,10 @@
Entry e = getEntry(path);
if (e == null || e.isDir())
throw new NoSuchFileException(getString(path));
- final ReadableByteChannel rbc =
- Channels.newChannel(getInputStream(e));
- final long size = e.size;
- return new SeekableByteChannel() {
- long read = 0;
- public boolean isOpen() {
- return rbc.isOpen();
- }
-
- public long position() throws IOException {
- return read;
- }
-
- public SeekableByteChannel position(long pos)
- throws IOException
- {
- throw new UnsupportedOperationException();
- }
-
- public int read(ByteBuffer dst) throws IOException {
- int n = rbc.read(dst);
- if (n > 0) {
- read += n;
- }
- return n;
- }
-
- public SeekableByteChannel truncate(long size)
- throws IOException
- {
- throw new NonWritableChannelException();
- }
-
- public int write (ByteBuffer src) throws IOException {
- throw new NonWritableChannelException();
- }
-
- public long size() throws IOException {
- return size;
- }
-
- public void close() throws IOException {
- rbc.close();
- }
- };
+ try (InputStream is = getInputStream(e)) {
+ // TBD: if (e.size < NNNNN);
+ return new ByteArrayChannel(is.readAllBytes(), true);
+ }
} finally {
endRead();
}
@@ -846,10 +824,6 @@
private Set<InputStream> streams =
Collections.synchronizedSet(new HashSet<InputStream>());
- // the ex-channel and ex-path that need to close when their outstanding
- // input streams are all closed by the obtainers.
- private Set<ExChannelCloser> exChClosers = new HashSet<>();
-
private Set<Path> tmppaths = Collections.synchronizedSet(new HashSet<Path>());
private Path getTempPathForEntry(byte[] path) throws IOException {
Path tmpPath = createTempFileInSameDirectoryAs(zfpath);
@@ -1087,8 +1061,9 @@
if (pos + CENHDR + nlen > limit) {
zerror("invalid CEN header (bad header size)");
}
- IndexNode inode = new IndexNode(cen, nlen, pos);
+ IndexNode inode = new IndexNode(cen, pos, nlen);
inodes.put(inode, inode);
+
// skip ext and comment
pos += (CENHDR + nlen + elen + clen);
}
@@ -1205,19 +1180,37 @@
return written;
}
+ private long writeEntry(Entry e, OutputStream os, byte[] buf)
+ throws IOException {
+
+ if (e.bytes == null && e.file == null) // dir, 0-length data
+ return 0;
+
+ long written = 0;
+ try (OutputStream os2 = e.method == METHOD_STORED ?
+ new EntryOutputStreamCRC32(e, os) : new EntryOutputStreamDef(e, os)) {
+ if (e.bytes != null) { // in-memory
+ os2.write(e.bytes, 0, e.bytes.length);
+ } else if (e.file != null) { // tmp file
+ if (e.type == Entry.NEW || e.type == Entry.FILECH) {
+ try (InputStream is = Files.newInputStream(e.file)) {
+ is.transferTo(os2);
+ }
+ }
+ Files.delete(e.file);
+ tmppaths.remove(e.file);
+ }
+ }
+ written += e.csize;
+ if ((e.flag & FLAG_DATADESCR) != 0) {
+ written += e.writeEXT(os);
+ }
+ return written;
+ }
+
// sync the zip file system, if there is any udpate
private void sync() throws IOException {
- // System.out.printf("->sync(%s) starting....!%n", toString());
- // check ex-closer
- if (!exChClosers.isEmpty()) {
- for (ExChannelCloser ecc : exChClosers) {
- if (ecc.streams.isEmpty()) {
- ecc.ch.close();
- Files.delete(ecc.path);
- exChClosers.remove(ecc);
- }
- }
- }
+
if (!hasUpdate)
return;
Path tmpFile = createTempFileInSameDirectoryAs(zfpath);
@@ -1243,34 +1236,7 @@
} else { // NEW, FILECH or CEN
e.locoff = written;
written += e.writeLOC(os); // write loc header
- if (e.bytes != null) { // in-memory, deflated
- os.write(e.bytes); // already
- written += e.bytes.length;
- } else if (e.file != null) { // tmp file
- try (InputStream is = Files.newInputStream(e.file)) {
- int n;
- if (e.type == Entry.NEW) { // deflated already
- while ((n = is.read(buf)) != -1) {
- os.write(buf, 0, n);
- written += n;
- }
- } else if (e.type == Entry.FILECH) {
- // the data are not deflated, use ZEOS
- try (OutputStream os2 = new EntryOutputStream(e, os)) {
- while ((n = is.read(buf)) != -1) {
- os2.write(buf, 0, n);
- }
- }
- written += e.csize;
- if ((e.flag & FLAG_DATADESCR) != 0)
- written += e.writeEXT(os);
- }
- }
- Files.delete(e.file);
- tmppaths.remove(e.file);
- } else {
- // dir, 0-length data
- }
+ written += writeEntry(e, os, buf);
}
elist.add(e);
} catch (IOException x) {
@@ -1303,27 +1269,9 @@
end.cenlen = written - end.cenoff;
end.write(os, written, forceEnd64);
}
- if (!streams.isEmpty()) {
- //
- // TBD: ExChannelCloser should not be necessary if we only
- // sync when being closed, all streams should have been
- // closed already. Keep the logic here for now.
- //
- // There are outstanding input streams open on existing "ch",
- // so, don't close the "cha" and delete the "file for now, let
- // the "ex-channel-closer" to handle them
- ExChannelCloser ecc = new ExChannelCloser(
- createTempFileInSameDirectoryAs(zfpath),
- ch,
- streams);
- Files.move(zfpath, ecc.path, REPLACE_EXISTING);
- exChClosers.add(ecc);
- streams = Collections.synchronizedSet(new HashSet<InputStream>());
- } else {
- ch.close();
- Files.delete(zfpath);
- }
+ ch.close();
+ Files.delete(zfpath);
Files.move(tmpFile, zfpath, REPLACE_EXISTING);
hasUpdate = false; // clear
}
@@ -1361,16 +1309,6 @@
}
}
- private static void copyStream(InputStream is, OutputStream os)
- throws IOException
- {
- byte[] copyBuf = new byte[8192];
- int n;
- while ((n = is.read(copyBuf)) != -1) {
- os.write(copyBuf, 0, n);
- }
- }
-
// Returns an out stream for either
// (1) writing the contents of a new entry, if the entry exits, or
// (2) updating/replacing the contents of the specified existing entry.
@@ -1379,9 +1317,9 @@
if (e.mtime == -1)
e.mtime = System.currentTimeMillis();
if (e.method == -1)
- e.method = METHOD_DEFLATED; // TBD: use default method
- // store size, compressed size, and crc-32 in LOC header
- e.flag = 0;
+ e.method = defaultMethod;
+ // store size, compressed size, and crc-32 in datadescr
+ e.flag = FLAG_DATADESCR;
if (zc.isUTF8())
e.flag |= FLAG_USE_UTF8;
OutputStream os;
@@ -1394,16 +1332,130 @@
return new EntryOutputStream(e, os);
}
+ private class EntryOutputStream extends FilterOutputStream {
+ private Entry e;
+ private long written;
+ private boolean isClosed;
+
+ EntryOutputStream(Entry e, OutputStream os) throws IOException {
+ super(os);
+ this.e = Objects.requireNonNull(e, "Zip entry is null");
+ // this.written = 0;
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ out.write(b);
+ written += 1;
+ }
+
+ @Override
+ public synchronized void write(byte b[], int off, int len)
+ throws IOException {
+ out.write(b, off, len);
+ written += len;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (isClosed) {
+ return;
+ }
+ isClosed = true;
+ e.size = written;
+ if (out instanceof ByteArrayOutputStream)
+ e.bytes = ((ByteArrayOutputStream)out).toByteArray();
+ super.close();
+ update(e);
+ }
+ }
+
+ // Wrapper output stream class to write out a "stored" entry.
+ // (1) this class does not close the underlying out stream when
+ // being closed.
+ // (2) no need to be "synchronized", only used by sync()
+ private class EntryOutputStreamCRC32 extends FilterOutputStream {
+ private Entry e;
+ private CRC32 crc;
+ private long written;
+ private boolean isClosed;
+
+ EntryOutputStreamCRC32(Entry e, OutputStream os) throws IOException {
+ super(os);
+ this.e = Objects.requireNonNull(e, "Zip entry is null");
+ this.crc = new CRC32();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ crc.update(b);
+ written += 1;
+ }
+
+ @Override
+ public void write(byte b[], int off, int len)
+ throws IOException {
+ out.write(b, off, len);
+ crc.update(b, off, len);
+ written += len;
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (isClosed)
+ return;
+ isClosed = true;
+ e.size = e.csize = written;
+ e.size = crc.getValue();
+ }
+ }
+
+ // Wrapper output stream class to write out a "deflated" entry.
+ // (1) this class does not close the underlying out stream when
+ // being closed.
+ // (2) no need to be "synchronized", only used by sync()
+ private class EntryOutputStreamDef extends DeflaterOutputStream {
+ private CRC32 crc;
+ private Entry e;
+ private boolean isClosed;
+
+ EntryOutputStreamDef(Entry e, OutputStream os) throws IOException {
+ super(os, getDeflater());
+ this.e = Objects.requireNonNull(e, "Zip entry is null");
+ this.crc = new CRC32();
+ }
+
+ @Override
+ public void write(byte b[], int off, int len)
+ throws IOException {
+ super.write(b, off, len);
+ crc.update(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (isClosed)
+ return;
+ isClosed = true;
+ finish();
+ e.size = def.getBytesRead();
+ e.csize = def.getBytesWritten();
+ e.crc = crc.getValue();
+ }
+ }
+
private InputStream getInputStream(Entry e)
throws IOException
{
InputStream eis = null;
if (e.type == Entry.NEW) {
+ // now bytes & file is uncompressed.
if (e.bytes != null)
- eis = new ByteArrayInputStream(e.bytes);
+ return new ByteArrayInputStream(e.bytes);
else if (e.file != null)
- eis = Files.newInputStream(e.file);
+ return Files.newInputStream(e.file);
else
throw new ZipException("update entry data is missing");
} else if (e.type == Entry.FILECH) {
@@ -1569,87 +1621,6 @@
}
}
- class EntryOutputStream extends DeflaterOutputStream
- {
- private CRC32 crc;
- private Entry e;
- private long written;
- private boolean isClosed = false;
-
- EntryOutputStream(Entry e, OutputStream os)
- throws IOException
- {
- super(os, getDeflater());
- if (e == null)
- throw new NullPointerException("Zip entry is null");
- this.e = e;
- crc = new CRC32();
- }
-
- @Override
- public synchronized void write(byte b[], int off, int len)
- throws IOException
- {
- if (e.type != Entry.FILECH) // only from sync
- ensureOpen();
- if (isClosed) {
- throw new IOException("Stream closed");
- }
- if (off < 0 || len < 0 || off > b.length - len) {
- throw new IndexOutOfBoundsException();
- } else if (len == 0) {
- return;
- }
- switch (e.method) {
- case METHOD_DEFLATED:
- super.write(b, off, len);
- break;
- case METHOD_STORED:
- written += len;
- out.write(b, off, len);
- break;
- default:
- throw new ZipException("invalid compression method");
- }
- crc.update(b, off, len);
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (isClosed) {
- return;
- }
- isClosed = true;
- // TBD ensureOpen();
- switch (e.method) {
- case METHOD_DEFLATED:
- finish();
- e.size = def.getBytesRead();
- e.csize = def.getBytesWritten();
- e.crc = crc.getValue();
- break;
- case METHOD_STORED:
- // we already know that both e.size and e.csize are the same
- e.size = e.csize = written;
- e.crc = crc.getValue();
- break;
- default:
- throw new ZipException("invalid compression method");
- }
- //crc.reset();
- if (out instanceof ByteArrayOutputStream)
- e.bytes = ((ByteArrayOutputStream)out).toByteArray();
-
- if (e.type == Entry.FILECH) {
- releaseDeflater(def);
- return;
- }
- super.close();
- releaseDeflater(def);
- update(e);
- }
- }
-
static void zerror(String msg) throws ZipException {
throw new ZipException(msg);
}
@@ -1806,7 +1777,7 @@
}
// constructor for cenInit() (1) remove tailing '/' (2) pad leading '/'
- IndexNode(byte[] cen, int nlen, int pos) {
+ IndexNode(byte[] cen, int pos, int nlen) {
int noff = pos + CENHDR;
if (cen[noff + nlen - 1] == '/') {
isdir = true;
@@ -1902,18 +1873,18 @@
Entry() {}
- Entry(byte[] name, boolean isdir) {
+ Entry(byte[] name, boolean isdir, int method) {
name(name);
this.isdir = isdir;
this.mtime = this.ctime = this.atime = System.currentTimeMillis();
this.crc = 0;
this.size = 0;
this.csize = 0;
- this.method = METHOD_DEFLATED;
+ this.method = method;
}
- Entry(byte[] name, int type, boolean isdir) {
- this(name, isdir);
+ Entry(byte[] name, int type, boolean isdir, int method) {
+ this(name, isdir, method);
this.type = type;
}
@@ -1941,9 +1912,8 @@
}
Entry (byte[] name, Path file, int type) {
- this(name, type, false);
+ this(name, type, false, METHOD_STORED);
this.file = file;
- this.method = METHOD_STORED;
}
int version() throws ZipException {
@@ -2422,6 +2392,7 @@
public String toString() {
StringBuilder sb = new StringBuilder(1024);
Formatter fm = new Formatter(sb);
+ fm.format(" name : %s%n", new String(name));
fm.format(" creationTime : %tc%n", creationTime().toMillis());
fm.format(" lastAccessTime : %tc%n", lastAccessTime().toMillis());
fm.format(" lastModifiedTime: %tc%n", lastModifiedTime().toMillis());
@@ -2439,20 +2410,6 @@
}
}
- private static class ExChannelCloser {
- Path path;
- SeekableByteChannel ch;
- Set<InputStream> streams;
- ExChannelCloser(Path path,
- SeekableByteChannel ch,
- Set<InputStream> streams)
- {
- this.path = path;
- this.ch = ch;
- this.streams = streams;
- }
- }
-
// ZIP directory has two issues:
// (1) ZIP spec does not require the ZIP file to include
// directory entry
--- a/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java Tue Sep 18 09:44:20 2018 -0700
+++ b/src/jdk.zipfs/share/classes/jdk/nio/zipfs/ZipFileSystemProvider.java Tue Sep 18 10:43:01 2018 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2009, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -124,9 +124,6 @@
public FileSystem newFileSystem(Path path, Map<String, ?> env)
throws IOException
{
- if (path.getFileSystem() != FileSystems.getDefault()) {
- throw new UnsupportedOperationException();
- }
ensureFile(path);
try {
ZipFileSystem zipfs;
--- a/test/jdk/jdk/nio/zipfs/ZipFSTester.java Tue Sep 18 09:44:20 2018 -0700
+++ b/test/jdk/jdk/nio/zipfs/ZipFSTester.java Tue Sep 18 10:43:01 2018 -0700
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2010, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
import java.net.URI;
import java.net.URLDecoder;
import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.SeekableByteChannel;
import java.nio.file.DirectoryStream;
@@ -58,8 +59,10 @@
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.zip.CRC32;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
import static java.nio.file.StandardOpenOption.*;
import static java.nio.file.StandardCopyOption.*;
@@ -70,7 +73,7 @@
* @test
* @bug 6990846 7009092 7009085 7015391 7014948 7005986 7017840 7007596
* 7157656 8002390 7012868 7012856 8015728 8038500 8040059 8069211
- * 8131067
+ * 8131067 8034802
* @summary Test Zip filesystem provider
* @modules jdk.zipfs
* @run main ZipFSTester
@@ -80,23 +83,27 @@
public class ZipFSTester {
public static void main(String[] args) throws Exception {
-
// create JAR file for test, actual contents don't matter
Path jarFile = Utils.createJarFile("tester.jar",
"META-INF/MANIFEST.MF",
"dir1/foo",
- "dir2/bar");
+ "dir2/bar",
+ "dir1/dir3/fooo");
try (FileSystem fs = newZipFileSystem(jarFile, Collections.emptyMap())) {
test0(fs);
test1(fs);
test2(fs); // more tests
}
+
+ testStreamChannel();
testTime(jarFile);
test8069211();
test8131067();
}
+ private static Random rdm = new Random();
+
static void test0(FileSystem fs)
throws Exception
{
@@ -121,13 +128,28 @@
static void test1(FileSystem fs0)
throws Exception
{
- Random rdm = new Random();
- // clone a fs and test on it
+ // prepare a src for testing
+ Path src = getTempPath();
+ String tmpName = src.toString();
+ try (OutputStream os = Files.newOutputStream(src)) {
+ byte[] bits = new byte[12345];
+ rdm.nextBytes(bits);
+ os.write(bits);
+ }
+
+ // clone a fs from fs0 and test on it
Path tmpfsPath = getTempPath();
Map<String, Object> env = new HashMap<String, Object>();
env.put("create", "true");
try (FileSystem copy = newZipFileSystem(tmpfsPath, env)) {
z2zcopy(fs0, copy, "/", 0);
+
+ // copy the test jar itself in
+ Files.copy(Paths.get(fs0.toString()), copy.getPath("/foo.jar"));
+ Path zpath = copy.getPath("/foo.jar");
+ try (FileSystem zzfs = FileSystems.newFileSystem(zpath, null)) {
+ Files.copy(src, zzfs.getPath("/srcInjarjar"));
+ }
}
try (FileSystem fs = newZipFileSystem(tmpfsPath, new HashMap<String, Object>())) {
@@ -142,15 +164,6 @@
throw new RuntimeException("newFileSystem(URI...) does not throw exception");
} catch (FileSystemAlreadyExistsException fsaee) {}
- // prepare a src
- Path src = getTempPath();
- String tmpName = src.toString();
- OutputStream os = Files.newOutputStream(src);
- byte[] bits = new byte[12345];
- rdm.nextBytes(bits);
- os.write(bits);
- os.close();
-
try {
provider.newFileSystem(new File(System.getProperty("test.src", ".")).toPath(),
new HashMap<String, Object>());
@@ -162,6 +175,8 @@
throw new RuntimeException("newFileSystem() opens a non-zip file as zipfs");
} catch (UnsupportedOperationException uoe) {}
+ // walk
+ walk(fs.getPath("/"));
// copyin
Path dst = getPathWithParents(fs, tmpName);
@@ -236,10 +251,29 @@
// test channels
channel(fs, dst);
Files.delete(dst);
- Files.delete(src);
+
+ // test foo.jar in jar/zipfs #8034802
+ Path jpath = fs.getPath("/foo.jar");
+ System.out.println("walking: " + jpath);
+ try (FileSystem zzfs = FileSystems.newFileSystem(jpath, null)) {
+ walk(zzfs.getPath("/"));
+ // foojar:/srcInjarjar
+ checkEqual(src, zzfs.getPath("/srcInjarjar"));
+
+ dst = getPathWithParents(zzfs, tmpName);
+ fchCopy(src, dst);
+ checkEqual(src, dst);
+ tmp = Paths.get(tmpName + "_Tmp");
+ fchCopy(dst, tmp); // out
+ checkEqual(src, tmp);
+ Files.delete(tmp);
+
+ channel(zzfs, dst);
+ Files.delete(dst);
+ }
} finally {
- if (Files.exists(tmpfsPath))
- Files.delete(tmpfsPath);
+ Files.deleteIfExists(tmpfsPath);
+ Files.deleteIfExists(src);
}
}
@@ -383,6 +417,158 @@
Files.delete(fs3Path);
}
+ static final int METHOD_STORED = 0;
+ static final int METHOD_DEFLATED = 8;
+
+ static Object[][] getEntries() {
+ Object[][] entries = new Object[10 + rdm.nextInt(20)][3];
+ for (int i = 0; i < entries.length; i++) {
+ entries[i][0] = "entries" + i;
+ entries[i][1] = rdm.nextInt(10) % 2 == 0 ?
+ METHOD_STORED : METHOD_DEFLATED;
+ entries[i][2] = new byte[rdm.nextInt(8192)];
+ rdm.nextBytes((byte[])entries[i][2]);
+ }
+ return entries;
+ }
+
+ // check the content of read from zipfs is equal to the "bytes"
+ private static void checkRead(Path path, byte[] expected) throws IOException {
+ //streams
+ try (InputStream is = Files.newInputStream(path)) {
+ if (!Arrays.equals(is.readAllBytes(), expected)) {
+ System.out.printf(" newInputStream <%s> failed...%n", path.toString());
+ throw new RuntimeException("CHECK FAILED!");
+ }
+ }
+
+ // channels -- via sun.nio.ch.ChannelInputStream
+ try (SeekableByteChannel sbc = Files.newByteChannel(path);
+ InputStream is = Channels.newInputStream(sbc)) {
+
+ // check all bytes match
+ if (!Arrays.equals(is.readAllBytes(), expected)) {
+ System.out.printf(" newByteChannel <%s> failed...%n", path.toString());
+ throw new RuntimeException("CHECK FAILED!");
+ }
+
+ // Check if read position is at the end
+ if (sbc.position() != expected.length) {
+ System.out.printf("pos [%s]: size=%d, position=%d%n",
+ path.toString(), expected.length, sbc.position());
+ throw new RuntimeException("CHECK FAILED!");
+ }
+
+ // Check position(x) + read() at the random/specific pos/len
+ byte[] buf = new byte[1024];
+ ByteBuffer bb = ByteBuffer.wrap(buf);
+ for (int i = 0; i < 10; i++) {
+ int pos = rdm.nextInt((int)sbc.size());
+ int len = rdm.nextInt(Math.min(buf.length, expected.length - pos));
+ // System.out.printf(" --> %d, %d%n", pos, len);
+ bb.position(0).limit(len); // bb.flip().limit(len);
+ if (sbc.position(pos).position() != pos ||
+ sbc.read(bb) != len ||
+ !Arrays.equals(buf, 0, bb.position(), expected, pos, pos + len)) {
+ System.out.printf("read()/position() failed%n");
+ }
+ }
+ } catch (IOException x) {
+ x.printStackTrace();
+ throw new RuntimeException("CHECK FAILED!");
+ }
+ }
+
+ // test entry stream/channel reading
+ static void testStreamChannel() throws Exception {
+ Path zpath = getTempPath();
+ try {
+ var crc = new CRC32();
+ Object[][] entries = getEntries();
+
+ // [1] create zip via ZipOutputStream
+ try (var os = Files.newOutputStream(zpath);
+ var zos = new ZipOutputStream(os)) {
+ for (Object[] entry : entries) {
+ var ze = new ZipEntry((String)entry[0]);
+ int method = (int)entry[1];
+ byte[] bytes = (byte[])entry[2];
+ if (method == METHOD_STORED) {
+ ze.setSize(bytes.length);
+ crc.reset();
+ crc.update(bytes);
+ ze.setCrc(crc.getValue());
+ }
+ ze.setMethod(method);
+ zos.putNextEntry(ze);
+ zos.write(bytes);
+ zos.closeEntry();
+ }
+ }
+ try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) {
+ for (Object[] e : entries) {
+ Path path = zfs.getPath((String)e[0]);
+ int method = (int)e[1];
+ byte[] bytes = (byte[])e[2];
+ // System.out.printf("checking read [%s, %d, %d]%n",
+ // path.toString(), bytes.length, method);
+ checkRead(path, bytes);
+ }
+ }
+ Files.deleteIfExists(zpath);
+
+ // [2] create zip via zfs.newByteChannel
+ try (var zfs = newZipFileSystem(zpath, Map.of("create", "true"))) {
+ for (Object[] e : entries) {
+ // tbd: method is not used
+ try (var sbc = Files.newByteChannel(zfs.getPath((String)e[0]),
+ CREATE_NEW, WRITE)) {
+ sbc.write(ByteBuffer.wrap((byte[])e[2]));
+ }
+ }
+ }
+ try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) {
+ for (Object[] e : entries) {
+ checkRead(zfs.getPath((String)e[0]), (byte[])e[2]);
+ }
+ }
+ Files.deleteIfExists(zpath);
+
+ // [3] create zip via Files.write()/newoutputStream/
+ try (var zfs = newZipFileSystem(zpath, Map.of("create", "true"))) {
+ for (Object[] e : entries) {
+ Files.write(zfs.getPath((String)e[0]), (byte[])e[2]);
+ }
+ }
+ try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) {
+ for (Object[] e : entries) {
+ checkRead(zfs.getPath((String)e[0]), (byte[])e[2]);
+ }
+ }
+ Files.deleteIfExists(zpath);
+
+ // [4] create zip via zfs.newByteChannel, with "method_stored"
+ try (var zfs = newZipFileSystem(zpath,
+ Map.of("create", true, "noCompression", true))) {
+ for (Object[] e : entries) {
+ try (var sbc = Files.newByteChannel(zfs.getPath((String)e[0]),
+ CREATE_NEW, WRITE)) {
+ sbc.write(ByteBuffer.wrap((byte[])e[2]));
+ }
+ }
+ }
+ try (var zfs = newZipFileSystem(zpath, Collections.emptyMap())) {
+ for (Object[] e : entries) {
+ checkRead(zfs.getPath((String)e[0]), (byte[])e[2]);
+ }
+ }
+ Files.deleteIfExists(zpath);
+
+ } finally {
+ Files.deleteIfExists(zpath);
+ }
+ }
+
// test file stamp
static void testTime(Path src) throws Exception {
BasicFileAttributes attrs = Files
@@ -392,34 +578,35 @@
Map<String, Object> env = new HashMap<String, Object>();
env.put("create", "true");
Path fsPath = getTempPath();
- FileSystem fs = newZipFileSystem(fsPath, env);
+ try (FileSystem fs = newZipFileSystem(fsPath, env)) {
+ System.out.println("test copy with timestamps...");
+ // copyin
+ Path dst = getPathWithParents(fs, "me");
+ Files.copy(src, dst, COPY_ATTRIBUTES);
+ checkEqual(src, dst);
+ System.out.println("mtime: " + attrs.lastModifiedTime());
+ System.out.println("ctime: " + attrs.creationTime());
+ System.out.println("atime: " + attrs.lastAccessTime());
+ System.out.println(" ==============>");
+ BasicFileAttributes dstAttrs = Files
+ .getFileAttributeView(dst, BasicFileAttributeView.class)
+ .readAttributes();
+ System.out.println("mtime: " + dstAttrs.lastModifiedTime());
+ System.out.println("ctime: " + dstAttrs.creationTime());
+ System.out.println("atime: " + dstAttrs.lastAccessTime());
- System.out.println("test copy with timestamps...");
- // copyin
- Path dst = getPathWithParents(fs, "me");
- Files.copy(src, dst, COPY_ATTRIBUTES);
- checkEqual(src, dst);
- System.out.println("mtime: " + attrs.lastModifiedTime());
- System.out.println("ctime: " + attrs.creationTime());
- System.out.println("atime: " + attrs.lastAccessTime());
- System.out.println(" ==============>");
- BasicFileAttributes dstAttrs = Files
- .getFileAttributeView(dst, BasicFileAttributeView.class)
- .readAttributes();
- System.out.println("mtime: " + dstAttrs.lastModifiedTime());
- System.out.println("ctime: " + dstAttrs.creationTime());
- System.out.println("atime: " + dstAttrs.lastAccessTime());
-
- // 1-second granularity
- if (attrs.lastModifiedTime().to(TimeUnit.SECONDS) !=
- dstAttrs.lastModifiedTime().to(TimeUnit.SECONDS) ||
- attrs.lastAccessTime().to(TimeUnit.SECONDS) !=
- dstAttrs.lastAccessTime().to(TimeUnit.SECONDS) ||
- attrs.creationTime().to(TimeUnit.SECONDS) !=
- dstAttrs.creationTime().to(TimeUnit.SECONDS)) {
- throw new RuntimeException("Timestamp Copy Failed!");
+ // 1-second granularity
+ if (attrs.lastModifiedTime().to(TimeUnit.SECONDS) !=
+ dstAttrs.lastModifiedTime().to(TimeUnit.SECONDS) ||
+ attrs.lastAccessTime().to(TimeUnit.SECONDS) !=
+ dstAttrs.lastAccessTime().to(TimeUnit.SECONDS) ||
+ attrs.creationTime().to(TimeUnit.SECONDS) !=
+ dstAttrs.creationTime().to(TimeUnit.SECONDS)) {
+ throw new RuntimeException("Timestamp Copy Failed!");
+ }
+ } finally {
+ Files.delete(fsPath);
}
- Files.delete(fsPath);
}
static void test8069211() throws Exception {
@@ -624,8 +811,8 @@
// check the content of two paths are equal
private static void checkEqual(Path src, Path dst) throws IOException
{
- //System.out.printf("checking <%s> vs <%s>...%n",
- // src.toString(), dst.toString());
+ System.out.printf("checking <%s> vs <%s>...%n",
+ src.toString(), dst.toString());
//streams
byte[] bufSrc = new byte[8192];
@@ -702,6 +889,21 @@
chDst.toString(), chDst.size(), chDst.position());
throw new RuntimeException("CHECK FAILED!");
}
+
+ // Check position(x) + read() at the specific pos/len
+ for (int i = 0; i < 10; i++) {
+ int pos = rdm.nextInt((int)chSrc.size());
+ int limit = rdm.nextInt(1024);
+ if (chSrc.position(pos).position() != chDst.position(pos).position()) {
+ System.out.printf("dst/src.position(pos failed%n");
+ }
+ bbSrc.clear().limit(limit);
+ bbDst.clear().limit(limit);
+ if (chSrc.read(bbSrc) != chDst.read(bbDst) ||
+ !bbSrc.flip().equals(bbDst.flip())) {
+ System.out.printf("dst/src.read() failed%n");
+ }
+ }
} catch (IOException x) {
x.printStackTrace();
}