1 /*

     2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 #include "precompiled.hpp"

    25 #include "gc/z/zArray.inline.hpp"

    26 #include "gc/z/zBackingFile_linux_x86.hpp"

    27 #include "gc/z/zBackingPath_linux_x86.hpp"

    28 #include "gc/z/zErrno.hpp"

    29 #include "gc/z/zLargePages.inline.hpp"

    30 #include "logging/log.hpp"

    31 #include "runtime/init.hpp"

    32 #include "runtime/os.hpp"

    33 #include "utilities/align.hpp"

    34 #include "utilities/debug.hpp"

    35 

    36 #include <fcntl.h>

    37 #include <sys/mman.h>

    38 #include <sys/stat.h>

    39 #include <sys/statfs.h>

    40 #include <sys/types.h>

    41 #include <unistd.h>

    42 

    43 // Filesystem names

    44 #define ZFILESYSTEM_TMPFS                "tmpfs"

    45 #define ZFILESYSTEM_HUGETLBFS            "hugetlbfs"

    46 

    47 // Sysfs file for transparent huge page on tmpfs

    48 #define ZFILENAME_SHMEM_ENABLED          "/sys/kernel/mm/transparent_hugepage/shmem_enabled"

    49 

    50 // Default mount points

    51 #define ZMOUNTPOINT_TMPFS                "/dev/shm"

    52 #define ZMOUNTPOINT_HUGETLBFS            "/hugepages"

    53 

    54 // Java heap filename

    55 #define ZFILENAME_HEAP                   "java_heap"

    56 

    57 // Support for building on older Linux systems

    58 #ifndef __NR_memfd_create

    59 #define __NR_memfd_create                319

    60 #endif

    61 #ifndef MFD_CLOEXEC

    62 #define MFD_CLOEXEC                      0x0001U

    63 #endif

    64 #ifndef MFD_HUGETLB

    65 #define MFD_HUGETLB                      0x0004U

    66 #endif

    67 #ifndef O_CLOEXEC

    68 #define O_CLOEXEC                        02000000

    69 #endif

    70 #ifndef O_TMPFILE

    71 #define O_TMPFILE                        (020000000 | O_DIRECTORY)

    72 #endif

    73 

    74 // Filesystem types, see statfs(2)

    75 #ifndef TMPFS_MAGIC

    76 #define TMPFS_MAGIC                      0x01021994

    77 #endif

    78 #ifndef HUGETLBFS_MAGIC

    79 #define HUGETLBFS_MAGIC                  0x958458f6

    80 #endif

    81 

    82 static int z_memfd_create(const char *name, unsigned int flags) {

    83   return syscall(__NR_memfd_create, name, flags);

    84 }

    85 

    86 ZBackingFile::ZBackingFile() :

    87     _fd(-1),

    88     _filesystem(0),

    89     _initialized(false) {

    90 

    91   // Create backing file

    92   _fd = create_fd(ZFILENAME_HEAP);

    93   if (_fd == -1) {

    94     return;

    95   }

    96 

    97   // Get filesystem type

    98   struct statfs statfs_buf;

    99   if (fstatfs(_fd, &statfs_buf) == -1) {

   100     ZErrno err;

   101     log_error(gc, init)("Failed to determine filesystem type for backing file (%s)", err.to_string());

   102     return;

   103   }

   104   _filesystem = statfs_buf.f_type;

   105 

   106   // Make sure we're on a supported filesystem

   107   if (!is_tmpfs() && !is_hugetlbfs()) {

   108     log_error(gc, init)("Backing file must be located on a %s or a %s filesystem", ZFILESYSTEM_TMPFS, ZFILESYSTEM_HUGETLBFS);

   109     return;

   110   }

   111 

   112   // Make sure the filesystem type matches requested large page type

   113   if (ZLargePages::is_transparent() && !is_tmpfs()) {

   114     log_error(gc, init)("-XX:+UseTransparentHugePages can only be enable when using a %s filesystem", ZFILESYSTEM_TMPFS);

   115     return;

   116   }

   117 

   118   if (ZLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {

   119     log_error(gc, init)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel", ZFILESYSTEM_TMPFS);

   120     return;

   121   }

   122 

   123   if (ZLargePages::is_explicit() && !is_hugetlbfs()) {

   124     log_error(gc, init)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);

   125     return;

   126   }

   127 

   128   if (!ZLargePages::is_explicit() && is_hugetlbfs()) {

   129     log_error(gc, init)("-XX:+UseLargePages must be enabled when using a %s filesystem", ZFILESYSTEM_HUGETLBFS);

   130     return;

   131   }

   132 

   133   // Successfully initialized

   134   _initialized = true;

   135 }

   136 

   137 int ZBackingFile::create_mem_fd(const char* name) const {

   138   // Create file name

   139   char filename[PATH_MAX];

   140   snprintf(filename, sizeof(filename), "%s%s", name, ZLargePages::is_explicit() ? ".hugetlb" : "");

   141 

   142   // Create file

   143   const int extra_flags = ZLargePages::is_explicit() ? MFD_HUGETLB : 0;

   144   const int fd = z_memfd_create(filename, MFD_CLOEXEC | extra_flags);

   145   if (fd == -1) {

   146     ZErrno err;

   147     log_debug(gc, init)("Failed to create memfd file (%s)",

   148                         ((UseLargePages && err == EINVAL) ? "Hugepages not supported" : err.to_string()));

   149     return -1;

   150   }

   151 

   152   log_debug(gc, init)("Heap backed by file /memfd:%s", filename);

   153 

   154   return fd;

   155 }

   156 

   157 int ZBackingFile::create_file_fd(const char* name) const {

   158   const char* const filesystem = ZLargePages::is_explicit() ? ZFILESYSTEM_HUGETLBFS : ZFILESYSTEM_TMPFS;

   159   const char* const mountpoint = ZLargePages::is_explicit() ? ZMOUNTPOINT_HUGETLBFS : ZMOUNTPOINT_TMPFS;

   160 

   161   // Find mountpoint

   162   ZBackingPath path(filesystem, mountpoint);

   163   if (path.get() == NULL) {

   164     log_error(gc, init)("Use -XX:ZPath to specify the path to a %s filesystem", filesystem);

   165     return -1;

   166   }

   167 

   168   // Try to create an anonymous file using the O_TMPFILE flag. Note that this

   169   // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.

   170   const int fd_anon = open(path.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);

   171   if (fd_anon == -1) {

   172     ZErrno err;

   173     log_debug(gc, init)("Failed to create anonymouns file in %s (%s)", path.get(),

   174                         (err == EINVAL ? "Not supported" : err.to_string()));

   175   } else {

   176     // Get inode number for anonymous file

   177     struct stat stat_buf;

   178     if (fstat(fd_anon, &stat_buf) == -1) {

   179       ZErrno err;

   180       log_error(gc, init)("Failed to determine inode number for anonymous file (%s)", err.to_string());

   181       return -1;

   182     }

   183 

   184     log_debug(gc, init)("Heap backed by file %s/#" UINT64_FORMAT, path.get(), (uint64_t)stat_buf.st_ino);

   185 

   186     return fd_anon;

   187   }

   188 

   189   log_debug(gc, init)("Falling back to open/unlink");

   190 

   191   // Create file name

   192   char filename[PATH_MAX];

   193   snprintf(filename, sizeof(filename), "%s/%s.%d", path.get(), name, os::current_process_id());

   194 

   195   // Create file

   196   const int fd = open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);

   197   if (fd == -1) {

   198     ZErrno err;

   199     log_error(gc, init)("Failed to create file %s (%s)", filename, err.to_string());

   200     return -1;

   201   }

   202 

   203   // Unlink file

   204   if (unlink(filename) == -1) {

   205     ZErrno err;

   206     log_error(gc, init)("Failed to unlink file %s (%s)", filename, err.to_string());

   207     return -1;

   208   }

   209 

   210   log_debug(gc, init)("Heap backed by file %s", filename);

   211 

   212   return fd;

   213 }

   214 

   215 int ZBackingFile::create_fd(const char* name) const {

   216   if (ZPath == NULL) {

   217     // If the path is not explicitly specified, then we first try to create a memfd file

   218     // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might

   219     // not be supported at all (requires kernel >= 3.17), or it might not support large

   220     // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a

   221     // file on an accessible tmpfs or hugetlbfs mount point.

   222     const int fd = create_mem_fd(name);

   223     if (fd != -1) {

   224       return fd;

   225     }

   226 

   227     log_debug(gc, init)("Falling back to searching for an accessible moint point");

   228   }

   229 

   230   return create_file_fd(name);

   231 }

   232 

   233 bool ZBackingFile::is_initialized() const {

   234   return _initialized;

   235 }

   236 

   237 int ZBackingFile::fd() const {

   238   return _fd;

   239 }

   240 

   241 bool ZBackingFile::is_tmpfs() const {

   242   return _filesystem == TMPFS_MAGIC;

   243 }

   244 

   245 bool ZBackingFile::is_hugetlbfs() const {

   246   return _filesystem == HUGETLBFS_MAGIC;

   247 }

   248 

   249 bool ZBackingFile::tmpfs_supports_transparent_huge_pages() const {

   250   // If the shmem_enabled file exists and is readable then we

   251   // know the kernel supports transparent huge pages for tmpfs.

   252   return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;

   253 }

   254 

   255 bool ZBackingFile::try_split_and_expand_tmpfs(size_t offset, size_t length, size_t alignment) const {

   256   // Try first smaller part.

   257   const size_t offset0 = offset;

   258   const size_t length0 = align_up(length / 2, alignment);

   259   if (!try_expand_tmpfs(offset0, length0, alignment)) {

   260     return false;

   261   }

   262 

   263   // Try second smaller part.

   264   const size_t offset1 = offset0 + length0;

   265   const size_t length1 = length - length0;

   266   if (!try_expand_tmpfs(offset1, length1, alignment)) {

   267     return false;

   268   }

   269 

   270   return true;

   271 }

   272 

   273 bool ZBackingFile::try_expand_tmpfs(size_t offset, size_t length, size_t alignment) const {

   274   assert(length > 0, "Invalid length");

   275   assert(is_aligned(length, alignment), "Invalid length");

   276 

   277   ZErrno err = posix_fallocate(_fd, offset, length);

   278 

   279   if (err == EINTR && length > alignment) {

   280     // Calling posix_fallocate() with a large length can take a long

   281     // time to complete. When running profilers, such as VTune, this

   282     // syscall will be constantly interrupted by signals. Expanding

   283     // the file in smaller steps avoids this problem.

   284     return try_split_and_expand_tmpfs(offset, length, alignment);

   285   }

   286 

   287   if (err) {

   288     log_error(gc)("Failed to allocate backing file (%s)", err.to_string());

   289     return false;

   290   }

   291 

   292   return true;

   293 }

   294 

   295 bool ZBackingFile::expand_tmpfs(size_t offset, size_t length) const {

   296   assert(is_tmpfs(), "Wrong filesystem");

   297   return try_expand_tmpfs(offset, length, os::vm_page_size());

   298 }

   299 

   300 bool ZBackingFile::expand_hugetlbfs(size_t offset, size_t length) const {

   301   assert(is_hugetlbfs(), "Wrong filesystem");

   302 

   303   // Prior to kernel 4.3, hugetlbfs did not support posix_fallocate().

   304   // Instead of posix_fallocate() we can use a well-known workaround,

   305   // which involves truncating the file to requested size and then try

   306   // to map it to verify that there are enough huge pages available to

   307   // back it.

   308   while (ftruncate(_fd, offset + length) == -1) {

   309     ZErrno err;

   310     if (err != EINTR) {

   311       log_error(gc)("Failed to truncate backing file (%s)", err.to_string());

   312       return false;

   313     }

   314   }

   315 

   316   // If we fail mapping during initialization, i.e. when we are pre-mapping

   317   // the heap, then we wait and retry a few times before giving up. Otherwise

   318   // there is a risk that running JVMs back-to-back will fail, since there

   319   // is a delay between process termination and the huge pages owned by that

   320   // process being returned to the huge page pool and made available for new

   321   // allocations.

   322   void* addr = MAP_FAILED;

   323   const int max_attempts = 3;

   324   for (int attempt = 1; attempt <= max_attempts; attempt++) {

   325     addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);

   326     if (addr != MAP_FAILED || is_init_completed()) {

   327       // Mapping was successful or initialization phase has completed

   328       break;

   329     }

   330 

   331     ZErrno err;

   332     log_debug(gc)("Failed to map backing file (%s), attempt %d of %d",

   333                   err.to_string(), attempt, max_attempts);

   334 

   335     // Wait and retry in one second, in the hope that

   336     // huge pages will be available by then.

   337     sleep(1);

   338   }

   339 

   340   if (addr == MAP_FAILED) {

   341     // Not enough huge pages left

   342     ZErrno err;

   343     log_error(gc)("Failed to map backing file (%s)", err.to_string());

   344     return false;

   345   }

   346 

   347   // Successful mapping, unmap again. From now on the pages we mapped

   348   // will be reserved for this file.

   349   if (munmap(addr, length) == -1) {

   350     ZErrno err;

   351     log_error(gc)("Failed to unmap backing file (%s)", err.to_string());

   352     return false;

   353   }

   354 

   355   return true;

   356 }

   357 

   358 bool ZBackingFile::expand(size_t offset, size_t length) const {

   359   return is_hugetlbfs() ? expand_hugetlbfs(offset, length) : expand_tmpfs(offset, length);

   360 }