8168628: (fc) SIGBUS when extending file size to map it
Summary: Synchronize file extension and subsequent map0(); on Linux use fallocate64() instead of ftruncate64().
Reviewed-by: rehn, simonis, alanb
--- a/jdk/make/mapfiles/libnio/mapfile-linux Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/make/mapfiles/libnio/mapfile-linux Fri Dec 02 13:22:16 2016 -0800
@@ -60,6 +60,7 @@
Java_sun_nio_ch_FileChannelImpl_position0;
Java_sun_nio_ch_FileChannelImpl_transferTo0;
Java_sun_nio_ch_FileChannelImpl_unmap0;
+ Java_sun_nio_ch_FileDispatcherImpl_allocate0;
Java_sun_nio_ch_FileDispatcherImpl_close0;
Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/jdk/make/mapfiles/libnio/mapfile-macosx Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/make/mapfiles/libnio/mapfile-macosx Fri Dec 02 13:22:16 2016 -0800
@@ -43,6 +43,7 @@
Java_sun_nio_ch_FileChannelImpl_position0;
Java_sun_nio_ch_FileChannelImpl_transferTo0;
Java_sun_nio_ch_FileChannelImpl_unmap0;
+ Java_sun_nio_ch_FileDispatcherImpl_allocate0;
Java_sun_nio_ch_FileDispatcherImpl_close0;
Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/jdk/make/mapfiles/libnio/mapfile-solaris Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/make/mapfiles/libnio/mapfile-solaris Fri Dec 02 13:22:16 2016 -0800
@@ -48,6 +48,7 @@
Java_sun_nio_ch_FileChannelImpl_position0;
Java_sun_nio_ch_FileChannelImpl_transferTo0;
Java_sun_nio_ch_FileChannelImpl_unmap0;
+ Java_sun_nio_ch_FileDispatcherImpl_allocate0;
Java_sun_nio_ch_FileDispatcherImpl_close0;
Java_sun_nio_ch_FileDispatcherImpl_closeIntFD;
Java_sun_nio_ch_FileDispatcherImpl_force0;
--- a/jdk/src/java.base/share/classes/sun/nio/ch/FileChannelImpl.java Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/share/classes/sun/nio/ch/FileChannelImpl.java Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -898,57 +898,62 @@
if (!isOpen())
return null;
- long filesize;
- do {
- filesize = nd.size(fd);
- } while ((filesize == IOStatus.INTERRUPTED) && isOpen());
- if (!isOpen())
- return null;
-
- if (filesize < position + size) { // Extend file size
- if (!writable) {
- throw new IOException("Channel not open for writing " +
- "- cannot extend file to required size");
- }
- int rv;
+ long mapSize;
+ int pagePosition;
+ synchronized (positionLock) {
+ long filesize;
do {
- rv = nd.truncate(fd, position + size);
- } while ((rv == IOStatus.INTERRUPTED) && isOpen());
+ filesize = nd.size(fd);
+ } while ((filesize == IOStatus.INTERRUPTED) && isOpen());
if (!isOpen())
return null;
- }
- if (size == 0) {
- addr = 0;
- // a valid file descriptor is not required
- FileDescriptor dummy = new FileDescriptor();
- if ((!writable) || (imode == MAP_RO))
- return Util.newMappedByteBufferR(0, 0, dummy, null);
- else
- return Util.newMappedByteBuffer(0, 0, dummy, null);
- }
+
+ if (filesize < position + size) { // Extend file size
+ if (!writable) {
+ throw new IOException("Channel not open for writing " +
+ "- cannot extend file to required size");
+ }
+ int rv;
+ do {
+ rv = nd.allocate(fd, position + size);
+ } while ((rv == IOStatus.INTERRUPTED) && isOpen());
+ if (!isOpen())
+ return null;
+ }
+
+ if (size == 0) {
+ addr = 0;
+ // a valid file descriptor is not required
+ FileDescriptor dummy = new FileDescriptor();
+ if ((!writable) || (imode == MAP_RO))
+ return Util.newMappedByteBufferR(0, 0, dummy, null);
+ else
+ return Util.newMappedByteBuffer(0, 0, dummy, null);
+ }
- int pagePosition = (int)(position % allocationGranularity);
- long mapPosition = position - pagePosition;
- long mapSize = size + pagePosition;
- try {
- // If no exception was thrown from map0, the address is valid
- addr = map0(imode, mapPosition, mapSize);
- } catch (OutOfMemoryError x) {
- // An OutOfMemoryError may indicate that we've exhausted memory
- // so force gc and re-attempt map
- System.gc();
+ pagePosition = (int)(position % allocationGranularity);
+ long mapPosition = position - pagePosition;
+ mapSize = size + pagePosition;
try {
- Thread.sleep(100);
- } catch (InterruptedException y) {
- Thread.currentThread().interrupt();
- }
- try {
+ // If map0 did not throw an exception, the address is valid
addr = map0(imode, mapPosition, mapSize);
- } catch (OutOfMemoryError y) {
- // After a second OOME, fail
- throw new IOException("Map failed", y);
+ } catch (OutOfMemoryError x) {
+ // An OutOfMemoryError may indicate that we've exhausted
+ // memory so force gc and re-attempt map
+ System.gc();
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException y) {
+ Thread.currentThread().interrupt();
+ }
+ try {
+ addr = map0(imode, mapPosition, mapSize);
+ } catch (OutOfMemoryError y) {
+ // After a second OOME, fail
+ throw new IOException("Map failed", y);
+ }
}
- }
+ } // synchronized
// On Windows, and potentially other platforms, we need an open
// file descriptor for some mapping operations.
--- a/jdk/src/java.base/share/classes/sun/nio/ch/FileDispatcher.java Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/share/classes/sun/nio/ch/FileDispatcher.java Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2007, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,8 @@
abstract int truncate(FileDescriptor fd, long size) throws IOException;
+ abstract int allocate(FileDescriptor fd, long size) throws IOException;
+
abstract long size(FileDescriptor fd) throws IOException;
abstract int lock(FileDescriptor fd, boolean blocking, long pos, long size,
--- a/jdk/src/java.base/unix/classes/sun/nio/ch/FileDispatcherImpl.java Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/unix/classes/sun/nio/ch/FileDispatcherImpl.java Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -76,6 +76,10 @@
return truncate0(fd, size);
}
+ int allocate(FileDescriptor fd, long size) throws IOException {
+ return allocate0(fd, size);
+ }
+
long size(FileDescriptor fd) throws IOException {
return size0(fd);
}
@@ -138,6 +142,9 @@
static native int truncate0(FileDescriptor fd, long size)
throws IOException;
+ static native int allocate0(FileDescriptor fd, long size)
+ throws IOException;
+
static native long size0(FileDescriptor fd) throws IOException;
static native int lock0(FileDescriptor fd, boolean blocking, long pos,
--- a/jdk/src/java.base/unix/native/libjava/io_util_md.c Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/unix/native/libjava/io_util_md.c Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -215,6 +215,20 @@
handleSetLength(FD fd, jlong length)
{
int result;
+#if defined(__linux__)
+ /*
+ * On Linux, if the file size is being increased, then ftruncate64()
+ * will modify the metadata value of the size without actually allocating
+ * any blocks which can cause a SIGBUS error if the file is subsequently
+ * memory-mapped.
+ */
+ struct stat64 sb;
+
+ if (fstat64(fd, &sb) == 0 && length > sb.st_blocks*512) {
+ RESTARTABLE(fallocate64(fd, 0, 0, length), result);
+ return result;
+ }
+#endif
RESTARTABLE(ftruncate64(fd, length), result);
return result;
}
--- a/jdk/src/java.base/unix/native/libnio/ch/FileDispatcherImpl.c Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/unix/native/libnio/ch/FileDispatcherImpl.c Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -186,6 +186,27 @@
"Truncation failed");
}
+JNIEXPORT jint JNICALL
+Java_sun_nio_ch_FileDispatcherImpl_allocate0(JNIEnv *env, jobject this,
+ jobject fdo, jlong size)
+{
+#if defined(__linux__)
+ /*
+ * On Linux, if the file size is being increased, then ftruncate64()
+ * will modify the metadata value of the size without actually allocating
+ * any blocks which can cause a SIGBUS error if the file is subsequently
+ * memory-mapped.
+ */
+ return handle(env,
+ fallocate64(fdval(env, fdo), 0, 0, size),
+ "Allocation failed");
+#else
+ return handle(env,
+ ftruncate64(fdval(env, fdo), size),
+ "Truncation failed");
+#endif
+}
+
JNIEXPORT jlong JNICALL
Java_sun_nio_ch_FileDispatcherImpl_size0(JNIEnv *env, jobject this, jobject fdo)
{
--- a/jdk/src/java.base/windows/classes/sun/nio/ch/FileDispatcherImpl.java Wed Dec 07 15:32:31 2016 -0500
+++ b/jdk/src/java.base/windows/classes/sun/nio/ch/FileDispatcherImpl.java Fri Dec 02 13:22:16 2016 -0800
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -84,6 +84,11 @@
return truncate0(fd, size);
}
+ int allocate(FileDescriptor fd, long size) throws IOException {
+ // truncate0() works for extending and truncating file size
+ return truncate0(fd, size);
+ }
+
long size(FileDescriptor fd) throws IOException {
return size0(fd);
}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/java/nio/channels/FileChannel/FileExtensionAndMap.java Fri Dec 02 13:22:16 2016 -0800
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/* @test
+ * @ignore This test has huge disk space requirements
+ * @bug 8168628
+ * @summary Test extending files to very large sizes without hitting a SIGBUS
+ * @requires (os.family == "linux")
+ * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap
+ * @run main/othervm/timeout=600 -Xms4g -Xmx4g FileExtensionAndMap true
+ */
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.Semaphore;
+import java.util.stream.IntStream;
+
+public class FileExtensionAndMap {
+
+ private static final ExecutorService CACHED_EXECUTORSERVICE =
+ Executors.newCachedThreadPool();
+
+ private static final String TMPDIR = System.getProperty("test.dir", ".");
+
+ private static boolean useRaf = false;
+
+ public static void main(String args[]) throws Exception {
+ if (args.length > 2) {
+ throw new IllegalArgumentException
+ ("Arguments: [true|false [targetFolder]]");
+ }
+
+ String defaultFolder = TMPDIR + File.separator + "target";
+ if (args.length > 0) {
+ useRaf = Boolean.valueOf(args[0]);
+ if (args.length > 1) {
+ defaultFolder = args[1];
+ }
+ }
+ final String targetFolder = defaultFolder;
+ Path p = Paths.get(targetFolder);
+ boolean targetExists = Files.exists(p);
+ if (!targetExists) {
+ Files.createDirectory(p);
+ }
+
+ System.out.printf("Using RandomAccessFile: %s; target folder: %s%n",
+ useRaf, targetFolder);
+
+ ForkJoinPool fjPool = new ForkJoinPool(3);
+ fjPool.submit(() -> {
+ IntStream.range(0, 20).parallel().forEach((index) -> {
+ String fileName = "testBigFile_" + index + ".dat";
+ Path source = null;
+ Path target = null;
+ try {
+ source = Paths.get(TMPDIR, fileName);
+ testCreateBigFile(source);
+ target = Paths.get(targetFolder, fileName);
+ testFileCopy(source, target);
+ } catch (Throwable th) {
+ System.err.println("Error copying file with fileName: "
+ + fileName + " : " + th.getMessage());
+ th.printStackTrace(System.err);
+ } finally {
+ try {
+ if (source != null) {
+ Files.deleteIfExists(source);
+ }
+ } catch (Throwable ignored) {
+ }
+ try {
+ if (target != null) {
+ Files.deleteIfExists(target);
+ }
+ } catch (Throwable ignored) {
+ }
+ }
+ });
+ }).join();
+
+ if (!targetExists) {
+ Files.delete(p);
+ }
+ }
+
+ private static void testFileCopy(Path source, Path target)
+ throws IOException {
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+ System.out.println("Finished copying file with fileName: "
+ + source.getFileName());
+ }
+
+ private static void testCreateBigFile(Path segmentFile)
+ throws IOException {
+ final Semaphore concurrencySemaphore = new Semaphore(5);
+ long fileSize = 3L * 1024L * 1024L * 1024L;
+ int blockSize = 10 * 1024 * 1024;
+ int loopCount = (int) Math.floorDiv(fileSize, blockSize);
+
+ String fileName = segmentFile.getFileName().toString();
+ if (useRaf) {
+ try (RandomAccessFile raf
+ = new RandomAccessFile(segmentFile.toFile(), "rw")) {
+ raf.setLength(fileSize);
+ try (FileChannel fc = raf.getChannel()) {
+ for (int i = 0; i < loopCount; i++) {
+ final long startPosition = 1L * blockSize * i;
+ concurrencySemaphore.acquireUninterruptibly();
+ CACHED_EXECUTORSERVICE.submit(() -> {
+ writeTemplateData(fileName, fc, startPosition,
+ blockSize, concurrencySemaphore);
+ });
+ }
+ } finally {
+ concurrencySemaphore.acquireUninterruptibly(5);
+ }
+ }
+ } else {
+ Path file = Files.createFile(segmentFile);
+ try (FileChannel fc = FileChannel.open(file,
+ StandardOpenOption.READ, StandardOpenOption.WRITE)) {
+ for (int i = 0; i < loopCount; i++) {
+ final long startPosition = 1L * blockSize * i;
+ concurrencySemaphore.acquireUninterruptibly();
+ CACHED_EXECUTORSERVICE.submit(() -> {
+ writeTemplateData(fileName, fc, startPosition,
+ blockSize, concurrencySemaphore);
+ });
+ }
+ }
+ }
+ }
+
+ private static void writeTemplateData(String fileName,
+ FileChannel fc, long startPosition, int blockSize,
+ Semaphore concurrencySemaphore) {
+ try {
+ byte[] EMPTY_RECORD = new byte[blockSize / 256];
+
+ MappedByteBuffer mappedByteBuffer = fc.map(MapMode.READ_WRITE,
+ startPosition, blockSize);
+ IntStream.range(0, 256).forEach((recordIndex) -> {
+ try {
+ mappedByteBuffer.position((int) (recordIndex *
+ EMPTY_RECORD.length));
+ mappedByteBuffer.put(EMPTY_RECORD, 0, EMPTY_RECORD.length);
+ } catch (Throwable th) {
+ System.err.println
+ ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: "
+ + fileName + ", startPosition: " + startPosition + ", recordIndex: "
+ + recordIndex + " : " + th.getMessage());
+ th.printStackTrace(System.err);
+ }
+ });
+
+ mappedByteBuffer.force();
+ } catch (Throwable th) {
+ if (!(th instanceof ClosedChannelException)) {
+ System.err.println
+ ("Error in FileExtensionAndMap.writeTemplateData empty record for fileName: "
+ + fileName + ", startPosition: " + startPosition + " : "
+ + th.getMessage());
+ th.printStackTrace(System.err);
+ }
+ } finally {
+ concurrencySemaphore.release();
+ }
+ }
+}