8043780: Use open(O_CLOEXEC) instead of fcntl(FD_CLOEXEC)
authormartin
Tue, 01 Jul 2014 13:29:24 -0700
changeset 27457 adbe834be3a0
parent 27408 9a8090dd6ec3
child 27458 eb5f1b4f01e1
8043780: Use open(O_CLOEXEC) instead of fcntl(FD_CLOEXEC) Summary: Use open(O_CLOEXEC) where available; fall back to FD_CLOEXEC when necessary Reviewed-by: rasbold, dholmes
hotspot/src/os/linux/vm/os_linux.cpp
--- a/hotspot/src/os/linux/vm/os_linux.cpp	Tue Oct 28 17:02:08 2014 -0400
+++ b/hotspot/src/os/linux/vm/os_linux.cpp	Tue Jul 01 13:29:24 2014 -0700
@@ -5103,9 +5103,38 @@
     errno = ENAMETOOLONG;
     return -1;
   }
-  int fd;
-
-  fd = ::open64(path, oflag, mode);
+
+  // All file descriptors that are opened in the Java process and not
+  // specifically destined for a subprocess should have the close-on-exec
+  // flag set.  If we don't set it, then careless 3rd party native code
+  // might fork and exec without closing all appropriate file descriptors
+  // (e.g. as we do in closeDescriptors in UNIXProcess.c), and this in
+  // turn might:
+  //
+  // - cause end-of-file to fail to be detected on some file
+  //   descriptors, resulting in mysterious hangs, or
+  //
+  // - might cause an fopen in the subprocess to fail on a system
+  //   suffering from bug 1085341.
+  //
+  // (Yes, the default setting of the close-on-exec flag is a Unix
+  // design flaw)
+  //
+  // See:
+  // 1085341: 32-bit stdio routines should support file descriptors >255
+  // 4843136: (process) pipe file descriptor from Runtime.exec not being closed
+  // 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
+  //
+  // Modern Linux kernels (after 2.6.23 2007) support O_CLOEXEC with open().
+  // O_CLOEXEC is preferable to using FD_CLOEXEC on an open file descriptor
+  // because it saves a system call and removes a small window where the flag
+  // is unset.  On ancient Linux kernels the O_CLOEXEC flag will be ignored
+  // and we fall back to using FD_CLOEXEC (see below).
+#ifdef O_CLOEXEC
+  oflag |= O_CLOEXEC;
+#endif
+
+  int fd = ::open64(path, oflag, mode);
   if (fd == -1) return -1;
 
   //If the open succeeded, the file might still be a directory
@@ -5126,32 +5155,17 @@
     }
   }
 
-  // All file descriptors that are opened in the JVM and not
-  // specifically destined for a subprocess should have the
-  // close-on-exec flag set.  If we don't set it, then careless 3rd
-  // party native code might fork and exec without closing all
-  // appropriate file descriptors (e.g. as we do in closeDescriptors in
-  // UNIXProcess.c), and this in turn might:
-  //
-  // - cause end-of-file to fail to be detected on some file
-  //   descriptors, resulting in mysterious hangs, or
-  //
-  // - might cause an fopen in the subprocess to fail on a system
-  //   suffering from bug 1085341.
-  //
-  // (Yes, the default setting of the close-on-exec flag is a Unix
-  // design flaw)
-  //
-  // See:
-  // 1085341: 32-bit stdio routines should support file descriptors >255
-  // 4843136: (process) pipe file descriptor from Runtime.exec not being closed
-  // 6339493: (process) Runtime.exec does not close all file descriptors on Solaris 9
-  //
 #ifdef FD_CLOEXEC
-  {
+  // Validate that the use of the O_CLOEXEC flag on open above worked.
+  // With recent kernels, we will perform this check exactly once.
+  static sig_atomic_t O_CLOEXEC_is_known_to_work = 0;
+  if (!O_CLOEXEC_is_known_to_work) {
     int flags = ::fcntl(fd, F_GETFD);
     if (flags != -1) {
-      ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
+      if ((flags & FD_CLOEXEC) != 0)
+        O_CLOEXEC_is_known_to_work = 1;
+      else
+        ::fcntl(fd, F_SETFD, flags | FD_CLOEXEC);
     }
   }
 #endif