1 /*

     2  * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "oops/markOop.hpp"

    27 #include "oops/oop.inline.hpp"

    28 #include "runtime/virtualspace.hpp"

    29 #include "services/memTracker.hpp"

    30 

    31 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC

    32 

    33 // ReservedSpace

    34 

    35 // Dummy constructor

    36 ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),

    37     _alignment(0), _special(false), _executable(false) {

    38 }

    39 

    40 ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) {

    41   bool has_preferred_page_size = preferred_page_size != 0;

    42   // Want to use large pages where possible and pad with small pages.

    43   size_t page_size = has_preferred_page_size ? preferred_page_size : os::page_size_for_region_unaligned(size, 1);

    44   bool large_pages = page_size != (size_t)os::vm_page_size();

    45   size_t alignment;

    46   if (large_pages && has_preferred_page_size) {

    47     alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());

    48     // ReservedSpace initialization requires size to be aligned to the given

    49     // alignment. Align the size up.

    50     size = align_size_up(size, alignment);

    51   } else {

    52     // Don't force the alignment to be large page aligned,

    53     // since that will waste memory.

    54     alignment = os::vm_allocation_granularity();

    55   }

    56   initialize(size, alignment, large_pages, NULL, false);

    57 }

    58 

    59 ReservedSpace::ReservedSpace(size_t size, size_t alignment,

    60                              bool large,

    61                              char* requested_address) {

    62   initialize(size, alignment, large, requested_address, false);

    63 }

    64 

    65 ReservedSpace::ReservedSpace(size_t size, size_t alignment,

    66                              bool large,

    67                              bool executable) {

    68   initialize(size, alignment, large, NULL, executable);

    69 }

    70 

    71 // Helper method.

    72 static bool failed_to_reserve_as_requested(char* base, char* requested_address,

    73                                            const size_t size, bool special)

    74 {

    75   if (base == requested_address || requested_address == NULL)

    76     return false; // did not fail

    77 

    78   if (base != NULL) {

    79     // Different reserve address may be acceptable in other cases

    80     // but for compressed oops heap should be at requested address.

    81     assert(UseCompressedOops, "currently requested address used only for compressed oops");

    82     if (PrintCompressedOopsMode) {

    83       tty->cr();

    84       tty->print_cr("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, base, requested_address);

    85     }

    86     // OS ignored requested address. Try different address.

    87     if (special) {

    88       if (!os::release_memory_special(base, size)) {

    89         fatal("os::release_memory_special failed");

    90       }

    91     } else {

    92       if (!os::release_memory(base, size)) {

    93         fatal("os::release_memory failed");

    94       }

    95     }

    96   }

    97   return true;

    98 }

    99 

   100 void ReservedSpace::initialize(size_t size, size_t alignment, bool large,

   101                                char* requested_address,

   102                                bool executable) {

   103   const size_t granularity = os::vm_allocation_granularity();

   104   assert((size & (granularity - 1)) == 0,

   105          "size not aligned to os::vm_allocation_granularity()");

   106   assert((alignment & (granularity - 1)) == 0,

   107          "alignment not aligned to os::vm_allocation_granularity()");

   108   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

   109          "not a power of 2");

   110 

   111   alignment = MAX2(alignment, (size_t)os::vm_page_size());

   112 

   113   _base = NULL;

   114   _size = 0;

   115   _special = false;

   116   _executable = executable;

   117   _alignment = 0;

   118   _noaccess_prefix = 0;

   119   if (size == 0) {

   120     return;

   121   }

   122 

   123   // If OS doesn't support demand paging for large page memory, we need

   124   // to use reserve_memory_special() to reserve and pin the entire region.

   125   bool special = large && !os::can_commit_large_page_memory();

   126   char* base = NULL;

   127 

   128   if (special) {

   129 

   130     base = os::reserve_memory_special(size, alignment, requested_address, executable);

   131 

   132     if (base != NULL) {

   133       if (failed_to_reserve_as_requested(base, requested_address, size, true)) {

   134         // OS ignored requested address. Try different address.

   135         return;

   136       }

   137       // Check alignment constraints.

   138       assert((uintptr_t) base % alignment == 0,

   139              err_msg("Large pages returned a non-aligned address, base: "

   140                  PTR_FORMAT " alignment: " PTR_FORMAT,

   141                  base, (void*)(uintptr_t)alignment));

   142       _special = true;

   143     } else {

   144       // failed; try to reserve regular memory below

   145       if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

   146                             !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

   147         if (PrintCompressedOopsMode) {

   148           tty->cr();

   149           tty->print_cr("Reserve regular memory without large pages.");

   150         }

   151       }

   152     }

   153   }

   154 

   155   if (base == NULL) {

   156     // Optimistically assume that the OSes returns an aligned base pointer.

   157     // When reserving a large address range, most OSes seem to align to at

   158     // least 64K.

   159 

   160     // If the memory was requested at a particular address, use

   161     // os::attempt_reserve_memory_at() to avoid over mapping something

   162     // important.  If available space is not detected, return NULL.

   163 

   164     if (requested_address != 0) {

   165       base = os::attempt_reserve_memory_at(size, requested_address);

   166       if (failed_to_reserve_as_requested(base, requested_address, size, false)) {

   167         // OS ignored requested address. Try different address.

   168         base = NULL;

   169       }

   170     } else {

   171       base = os::reserve_memory(size, NULL, alignment);

   172     }

   173 

   174     if (base == NULL) return;

   175 

   176     // Check alignment constraints

   177     if ((((size_t)base) & (alignment - 1)) != 0) {

   178       // Base not aligned, retry

   179       if (!os::release_memory(base, size)) fatal("os::release_memory failed");

   180       // Make sure that size is aligned

   181       size = align_size_up(size, alignment);

   182       base = os::reserve_memory_aligned(size, alignment);

   183 

   184       if (requested_address != 0 &&

   185           failed_to_reserve_as_requested(base, requested_address, size, false)) {

   186         // As a result of the alignment constraints, the allocated base differs

   187         // from the requested address. Return back to the caller who can

   188         // take remedial action (like try again without a requested address).

   189         assert(_base == NULL, "should be");

   190         return;

   191       }

   192     }

   193   }

   194   // Done

   195   _base = base;

   196   _size = size;

   197   _alignment = alignment;

   198 }

   199 

   200 

   201 ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,

   202                              bool special, bool executable) {

   203   assert((size % os::vm_allocation_granularity()) == 0,

   204          "size not allocation aligned");

   205   _base = base;

   206   _size = size;

   207   _alignment = alignment;

   208   _noaccess_prefix = 0;

   209   _special = special;

   210   _executable = executable;

   211 }

   212 

   213 

   214 ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,

   215                                         bool split, bool realloc) {

   216   assert(partition_size <= size(), "partition failed");

   217   if (split) {

   218     os::split_reserved_memory(base(), size(), partition_size, realloc);

   219   }

   220   ReservedSpace result(base(), partition_size, alignment, special(),

   221                        executable());

   222   return result;

   223 }

   224 

   225 

   226 ReservedSpace

   227 ReservedSpace::last_part(size_t partition_size, size_t alignment) {

   228   assert(partition_size <= size(), "partition failed");

   229   ReservedSpace result(base() + partition_size, size() - partition_size,

   230                        alignment, special(), executable());

   231   return result;

   232 }

   233 

   234 

   235 size_t ReservedSpace::page_align_size_up(size_t size) {

   236   return align_size_up(size, os::vm_page_size());

   237 }

   238 

   239 

   240 size_t ReservedSpace::page_align_size_down(size_t size) {

   241   return align_size_down(size, os::vm_page_size());

   242 }

   243 

   244 

   245 size_t ReservedSpace::allocation_align_size_up(size_t size) {

   246   return align_size_up(size, os::vm_allocation_granularity());

   247 }

   248 

   249 

   250 size_t ReservedSpace::allocation_align_size_down(size_t size) {

   251   return align_size_down(size, os::vm_allocation_granularity());

   252 }

   253 

   254 

   255 void ReservedSpace::release() {

   256   if (is_reserved()) {

   257     char *real_base = _base - _noaccess_prefix;

   258     const size_t real_size = _size + _noaccess_prefix;

   259     if (special()) {

   260       os::release_memory_special(real_base, real_size);

   261     } else{

   262       os::release_memory(real_base, real_size);

   263     }

   264     _base = NULL;

   265     _size = 0;

   266     _noaccess_prefix = 0;

   267     _alignment = 0;

   268     _special = false;

   269     _executable = false;

   270   }

   271 }

   272 

   273 static size_t noaccess_prefix_size(size_t alignment) {

   274   return lcm(os::vm_page_size(), alignment);

   275 }

   276 

   277 void ReservedHeapSpace::establish_noaccess_prefix() {

   278   assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");

   279   _noaccess_prefix = noaccess_prefix_size(_alignment);

   280 

   281   if (base() && base() + _size > (char *)OopEncodingHeapMax) {

   282     if (true

   283         WIN64_ONLY(&& !UseLargePages)

   284         AIX_ONLY(&& os::vm_page_size() != SIZE_64K)) {

   285       // Protect memory at the base of the allocated region.

   286       // If special, the page was committed (only matters on windows)

   287       if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {

   288         fatal("cannot protect protection page");

   289       }

   290       if (PrintCompressedOopsMode) {

   291         tty->cr();

   292         tty->print_cr("Protected page at the reserved heap base: "

   293                       PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix);

   294       }

   295       assert(Universe::narrow_oop_use_implicit_null_checks() == true, "not initialized?");

   296     } else {

   297       Universe::set_narrow_oop_use_implicit_null_checks(false);

   298     }

   299   }

   300 

   301   _base += _noaccess_prefix;

   302   _size -= _noaccess_prefix;

   303   assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");

   304 }

   305 

   306 // Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.

   307 // Does not check whether the reserved memory actually is at requested_address, as the memory returned

   308 // might still fulfill the wishes of the caller.

   309 // Assures the memory is aligned to 'alignment'.

   310 // NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.

   311 void ReservedHeapSpace::try_reserve_heap(size_t size,

   312                                          size_t alignment,

   313                                          bool large,

   314                                          char* requested_address) {

   315   if (_base != NULL) {

   316     // We tried before, but we didn't like the address delivered.

   317     release();

   318   }

   319 

   320   // If OS doesn't support demand paging for large page memory, we need

   321   // to use reserve_memory_special() to reserve and pin the entire region.

   322   bool special = large && !os::can_commit_large_page_memory();

   323   char* base = NULL;

   324 

   325   if (PrintCompressedOopsMode && Verbose) {

   326     tty->print("Trying to allocate at address " PTR_FORMAT " heap of size " PTR_FORMAT ".\n",

   327                requested_address, (address)size);

   328   }

   329 

   330   if (special) {

   331     base = os::reserve_memory_special(size, alignment, requested_address, false);

   332 

   333     if (base != NULL) {

   334       // Check alignment constraints.

   335       assert((uintptr_t) base % alignment == 0,

   336              err_msg("Large pages returned a non-aligned address, base: "

   337                      PTR_FORMAT " alignment: " PTR_FORMAT,

   338                      base, (void*)(uintptr_t)alignment));

   339       _special = true;

   340     }

   341   }

   342 

   343   if (base == NULL) {

   344     // Failed; try to reserve regular memory below

   345     if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

   346                           !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

   347       if (PrintCompressedOopsMode) {

   348         tty->cr();

   349         tty->print_cr("Reserve regular memory without large pages.");

   350       }

   351     }

   352 

   353     // Optimistically assume that the OSes returns an aligned base pointer.

   354     // When reserving a large address range, most OSes seem to align to at

   355     // least 64K.

   356 

   357     // If the memory was requested at a particular address, use

   358     // os::attempt_reserve_memory_at() to avoid over mapping something

   359     // important.  If available space is not detected, return NULL.

   360 

   361     if (requested_address != 0) {

   362       base = os::attempt_reserve_memory_at(size, requested_address);

   363     } else {

   364       base = os::reserve_memory(size, NULL, alignment);

   365     }

   366   }

   367   if (base == NULL) { return; }

   368 

   369   // Done

   370   _base = base;

   371   _size = size;

   372   _alignment = alignment;

   373 

   374   // Check alignment constraints

   375   if ((((size_t)base) & (alignment - 1)) != 0) {

   376     // Base not aligned, retry.

   377     release();

   378   }

   379 }

   380 

   381 void ReservedHeapSpace::try_reserve_range(char *highest_start,

   382                                           char *lowest_start,

   383                                           size_t attach_point_alignment,

   384                                           char *aligned_heap_base_min_address,

   385                                           char *upper_bound,

   386                                           size_t size,

   387                                           size_t alignment,

   388                                           bool large) {

   389   const size_t attach_range = highest_start - lowest_start;

   390   // Cap num_attempts at possible number.

   391   // At least one is possible even for 0 sized attach range.

   392   const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;

   393   const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);

   394 

   395   const size_t stepsize = (attach_range == 0) ? // Only one try.

   396     (size_t) highest_start : align_size_up(attach_range / num_attempts_to_try, attach_point_alignment);

   397 

   398   // Try attach points from top to bottom.

   399   char* attach_point = highest_start;

   400   while (attach_point >= lowest_start  &&

   401          attach_point <= highest_start &&  // Avoid wrap around.

   402          ((_base == NULL) ||

   403           (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {

   404     try_reserve_heap(size, alignment, large, attach_point);

   405     attach_point -= stepsize;

   406   }

   407 }

   408 

   409 #define SIZE_64K  ((uint64_t) UCONST64(      0x10000))

   410 #define SIZE_256M ((uint64_t) UCONST64(   0x10000000))

   411 #define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))

   412 

   413 // Helper for heap allocation. Returns an array with addresses

   414 // (OS-specific) which are suited for disjoint base mode. Array is

   415 // NULL terminated.

   416 static char** get_attach_addresses_for_disjoint_mode() {

   417   static uint64_t addresses[] = {

   418      2 * SIZE_32G,

   419      3 * SIZE_32G,

   420      4 * SIZE_32G,

   421      8 * SIZE_32G,

   422     10 * SIZE_32G,

   423      1 * SIZE_64K * SIZE_32G,

   424      2 * SIZE_64K * SIZE_32G,

   425      3 * SIZE_64K * SIZE_32G,

   426      4 * SIZE_64K * SIZE_32G,

   427     16 * SIZE_64K * SIZE_32G,

   428     32 * SIZE_64K * SIZE_32G,

   429     34 * SIZE_64K * SIZE_32G,

   430     0

   431   };

   432 

   433   // Sort out addresses smaller than HeapBaseMinAddress. This assumes

   434   // the array is sorted.

   435   uint i = 0;

   436   while (addresses[i] != 0 &&

   437          (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {

   438     i++;

   439   }

   440   uint start = i;

   441 

   442   // Avoid more steps than requested.

   443   i = 0;

   444   while (addresses[start+i] != 0) {

   445     if (i == HeapSearchSteps) {

   446       addresses[start+i] = 0;

   447       break;

   448     }

   449     i++;

   450   }

   451 

   452   return (char**) &addresses[start];

   453 }

   454 

   455 void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, bool large) {

   456   guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,

   457             "can not allocate compressed oop heap for this size");

   458   guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");

   459   assert(HeapBaseMinAddress > 0, "sanity");

   460 

   461   const size_t granularity = os::vm_allocation_granularity();

   462   assert((size & (granularity - 1)) == 0,

   463          "size not aligned to os::vm_allocation_granularity()");

   464   assert((alignment & (granularity - 1)) == 0,

   465          "alignment not aligned to os::vm_allocation_granularity()");

   466   assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

   467          "not a power of 2");

   468 

   469   // The necessary attach point alignment for generated wish addresses.

   470   // This is needed to increase the chance of attaching for mmap and shmat.

   471   const size_t os_attach_point_alignment =

   472     AIX_ONLY(SIZE_256M)  // Known shm boundary alignment.

   473     NOT_AIX(os::vm_allocation_granularity());

   474   const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);

   475 

   476   char *aligned_heap_base_min_address = (char *)align_ptr_up((void *)HeapBaseMinAddress, alignment);

   477   size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?

   478     noaccess_prefix_size(alignment) : 0;

   479 

   480   // Attempt to alloc at user-given address.

   481   if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {

   482     try_reserve_heap(size + noaccess_prefix, alignment, large, aligned_heap_base_min_address);

   483     if (_base != aligned_heap_base_min_address) { // Enforce this exact address.

   484       release();

   485     }

   486   }

   487 

   488   // Keep heap at HeapBaseMinAddress.

   489   if (_base == NULL) {

   490 

   491     // Try to allocate the heap at addresses that allow efficient oop compression.

   492     // Different schemes are tried, in order of decreasing optimization potential.

   493     //

   494     // For this, try_reserve_heap() is called with the desired heap base addresses.

   495     // A call into the os layer to allocate at a given address can return memory

   496     // at a different address than requested.  Still, this might be memory at a useful

   497     // address. try_reserve_heap() always returns this allocated memory, as only here

   498     // the criteria for a good heap are checked.

   499 

   500     // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).

   501     // Give it several tries from top of range to bottom.

   502     if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {

   503 

   504       // Calc address range within we try to attach (range of possible start addresses).

   505       char* const highest_start = (char *)align_ptr_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);

   506       char* const lowest_start  = (char *)align_ptr_up  (        aligned_heap_base_min_address             , attach_point_alignment);

   507       try_reserve_range(highest_start, lowest_start, attach_point_alignment,

   508                         aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, large);

   509     }

   510 

   511     // zerobased: Attempt to allocate in the lower 32G.

   512     // But leave room for the compressed class pointers, which is allocated above

   513     // the heap.

   514     char *zerobased_max = (char *)OopEncodingHeapMax;

   515     const size_t class_space = align_size_up(CompressedClassSpaceSize, alignment);

   516     // For small heaps, save some space for compressed class pointer

   517     // space so it can be decoded with no base.

   518     if (UseCompressedClassPointers && !UseSharedSpaces &&

   519         OopEncodingHeapMax <= KlassEncodingMetaspaceMax &&

   520         (uint64_t)(aligned_heap_base_min_address + size + class_space) <= KlassEncodingMetaspaceMax) {

   521       zerobased_max = (char *)OopEncodingHeapMax - class_space;

   522     }

   523 

   524     // Give it several tries from top of range to bottom.

   525     if (aligned_heap_base_min_address + size <= zerobased_max &&    // Zerobased theoretical possible.

   526         ((_base == NULL) ||                        // No previous try succeeded.

   527          (_base + size > zerobased_max))) {        // Unscaled delivered an arbitrary address.

   528 

   529       // Calc address range within we try to attach (range of possible start addresses).

   530       char *const highest_start = (char *)align_ptr_down(zerobased_max - size, attach_point_alignment);

   531       // Need to be careful about size being guaranteed to be less

   532       // than UnscaledOopHeapMax due to type constraints.

   533       char *lowest_start = aligned_heap_base_min_address;

   534       uint64_t unscaled_end = UnscaledOopHeapMax - size;

   535       if (unscaled_end < UnscaledOopHeapMax) { // unscaled_end wrapped if size is large

   536         lowest_start = MAX2(lowest_start, (char*)unscaled_end);

   537       }

   538       lowest_start  = (char *)align_ptr_up(lowest_start, attach_point_alignment);

   539       try_reserve_range(highest_start, lowest_start, attach_point_alignment,

   540                         aligned_heap_base_min_address, zerobased_max, size, alignment, large);

   541     }

   542 

   543     // Now we go for heaps with base != 0.  We need a noaccess prefix to efficiently

   544     // implement null checks.

   545     noaccess_prefix = noaccess_prefix_size(alignment);

   546 

   547     // Try to attach at addresses that are aligned to OopEncodingHeapMax. Disjointbase mode.

   548     char** addresses = get_attach_addresses_for_disjoint_mode();

   549     int i = 0;

   550     while (addresses[i] &&                                 // End of array not yet reached.

   551            ((_base == NULL) ||                             // No previous try succeeded.

   552             (_base + size >  (char *)OopEncodingHeapMax && // Not zerobased or unscaled address.

   553              !Universe::is_disjoint_heap_base_address((address)_base)))) {  // Not disjoint address.

   554       char* const attach_point = addresses[i];

   555       assert(attach_point >= aligned_heap_base_min_address, "Flag support broken");

   556       try_reserve_heap(size + noaccess_prefix, alignment, large, attach_point);

   557       i++;

   558     }

   559 

   560     // Last, desperate try without any placement.

   561     if (_base == NULL) {

   562       if (PrintCompressedOopsMode && Verbose) {

   563         tty->print("Trying to allocate at address NULL heap of size " PTR_FORMAT ".\n", (address)size + noaccess_prefix);

   564       }

   565       initialize(size + noaccess_prefix, alignment, large, NULL, false);

   566     }

   567   }

   568 }

   569 

   570 ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment, bool large) : ReservedSpace() {

   571 

   572   if (size == 0) {

   573     return;

   574   }

   575 

   576   // Heap size should be aligned to alignment, too.

   577   guarantee(is_size_aligned(size, alignment), "set by caller");

   578 

   579   if (UseCompressedOops) {

   580     initialize_compressed_heap(size, alignment, large);

   581     if (_size > size) {

   582       // We allocated heap with noaccess prefix.

   583       // It can happen we get a zerobased/unscaled heap with noaccess prefix,

   584       // if we had to try at arbitrary address.

   585       establish_noaccess_prefix();

   586     }

   587   } else {

   588     initialize(size, alignment, large, NULL, false);

   589   }

   590 

   591   assert(markOopDesc::encode_pointer_as_mark(_base)->decode_pointer() == _base,

   592          "area must be distinguishable from marks for mark-sweep");

   593   assert(markOopDesc::encode_pointer_as_mark(&_base[size])->decode_pointer() == &_base[size],

   594          "area must be distinguishable from marks for mark-sweep");

   595 

   596   if (base() > 0) {

   597     MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);

   598   }

   599 }

   600 

   601 // Reserve space for code segment.  Same as Java heap only we mark this as

   602 // executable.

   603 ReservedCodeSpace::ReservedCodeSpace(size_t r_size,

   604                                      size_t rs_align,

   605                                      bool large) :

   606   ReservedSpace(r_size, rs_align, large, /*executable*/ true) {

   607   MemTracker::record_virtual_memory_type((address)base(), mtCode);

   608 }

   609 

   610 // VirtualSpace

   611 

   612 VirtualSpace::VirtualSpace() {

   613   _low_boundary           = NULL;

   614   _high_boundary          = NULL;

   615   _low                    = NULL;

   616   _high                   = NULL;

   617   _lower_high             = NULL;

   618   _middle_high            = NULL;

   619   _upper_high             = NULL;

   620   _lower_high_boundary    = NULL;

   621   _middle_high_boundary   = NULL;

   622   _upper_high_boundary    = NULL;

   623   _lower_alignment        = 0;

   624   _middle_alignment       = 0;

   625   _upper_alignment        = 0;

   626   _special                = false;

   627   _executable             = false;

   628 }

   629 

   630 

   631 bool VirtualSpace::initialize(ReservedSpace rs, size_t committed_size) {

   632   const size_t max_commit_granularity = os::page_size_for_region_unaligned(rs.size(), 1);

   633   return initialize_with_granularity(rs, committed_size, max_commit_granularity);

   634 }

   635 

   636 bool VirtualSpace::initialize_with_granularity(ReservedSpace rs, size_t committed_size, size_t max_commit_granularity) {

   637   if(!rs.is_reserved()) return false;  // allocation failed.

   638   assert(_low_boundary == NULL, "VirtualSpace already initialized");

   639   assert(max_commit_granularity > 0, "Granularity must be non-zero.");

   640 

   641   _low_boundary  = rs.base();

   642   _high_boundary = low_boundary() + rs.size();

   643 

   644   _low = low_boundary();

   645   _high = low();

   646 

   647   _special = rs.special();

   648   _executable = rs.executable();

   649 

   650   // When a VirtualSpace begins life at a large size, make all future expansion

   651   // and shrinking occur aligned to a granularity of large pages.  This avoids

   652   // fragmentation of physical addresses that inhibits the use of large pages

   653   // by the OS virtual memory system.  Empirically,  we see that with a 4MB

   654   // page size, the only spaces that get handled this way are codecache and

   655   // the heap itself, both of which provide a substantial performance

   656   // boost in many benchmarks when covered by large pages.

   657   //

   658   // No attempt is made to force large page alignment at the very top and

   659   // bottom of the space if they are not aligned so already.

   660   _lower_alignment  = os::vm_page_size();

   661   _middle_alignment = max_commit_granularity;

   662   _upper_alignment  = os::vm_page_size();

   663 

   664   // End of each region

   665   _lower_high_boundary = (char*) round_to((intptr_t) low_boundary(), middle_alignment());

   666   _middle_high_boundary = (char*) round_down((intptr_t) high_boundary(), middle_alignment());

   667   _upper_high_boundary = high_boundary();

   668 

   669   // High address of each region

   670   _lower_high = low_boundary();

   671   _middle_high = lower_high_boundary();

   672   _upper_high = middle_high_boundary();

   673 

   674   // commit to initial size

   675   if (committed_size > 0) {

   676     if (!expand_by(committed_size)) {

   677       return false;

   678     }

   679   }

   680   return true;

   681 }

   682 

   683 

   684 VirtualSpace::~VirtualSpace() {

   685   release();

   686 }

   687 

   688 

   689 void VirtualSpace::release() {

   690   // This does not release memory it never reserved.

   691   // Caller must release via rs.release();

   692   _low_boundary           = NULL;

   693   _high_boundary          = NULL;

   694   _low                    = NULL;

   695   _high                   = NULL;

   696   _lower_high             = NULL;

   697   _middle_high            = NULL;

   698   _upper_high             = NULL;

   699   _lower_high_boundary    = NULL;

   700   _middle_high_boundary   = NULL;

   701   _upper_high_boundary    = NULL;

   702   _lower_alignment        = 0;

   703   _middle_alignment       = 0;

   704   _upper_alignment        = 0;

   705   _special                = false;

   706   _executable             = false;

   707 }

   708 

   709 

   710 size_t VirtualSpace::committed_size() const {

   711   return pointer_delta(high(), low(), sizeof(char));

   712 }

   713 

   714 

   715 size_t VirtualSpace::reserved_size() const {

   716   return pointer_delta(high_boundary(), low_boundary(), sizeof(char));

   717 }

   718 

   719 

   720 size_t VirtualSpace::uncommitted_size()  const {

   721   return reserved_size() - committed_size();

   722 }

   723 

   724 size_t VirtualSpace::actual_committed_size() const {

   725   // Special VirtualSpaces commit all reserved space up front.

   726   if (special()) {

   727     return reserved_size();

   728   }

   729 

   730   size_t committed_low    = pointer_delta(_lower_high,  _low_boundary,         sizeof(char));

   731   size_t committed_middle = pointer_delta(_middle_high, _lower_high_boundary,  sizeof(char));

   732   size_t committed_high   = pointer_delta(_upper_high,  _middle_high_boundary, sizeof(char));

   733 

   734 #ifdef ASSERT

   735   size_t lower  = pointer_delta(_lower_high_boundary,  _low_boundary,         sizeof(char));

   736   size_t middle = pointer_delta(_middle_high_boundary, _lower_high_boundary,  sizeof(char));

   737   size_t upper  = pointer_delta(_upper_high_boundary,  _middle_high_boundary, sizeof(char));

   738 

   739   if (committed_high > 0) {

   740     assert(committed_low == lower, "Must be");

   741     assert(committed_middle == middle, "Must be");

   742   }

   743 

   744   if (committed_middle > 0) {

   745     assert(committed_low == lower, "Must be");

   746   }

   747   if (committed_middle < middle) {

   748     assert(committed_high == 0, "Must be");

   749   }

   750 

   751   if (committed_low < lower) {

   752     assert(committed_high == 0, "Must be");

   753     assert(committed_middle == 0, "Must be");

   754   }

   755 #endif

   756 

   757   return committed_low + committed_middle + committed_high;

   758 }

   759 

   760 

   761 bool VirtualSpace::contains(const void* p) const {

   762   return low() <= (const char*) p && (const char*) p < high();

   763 }

   764 

   765 /*

   766    First we need to determine if a particular virtual space is using large

   767    pages.  This is done at the initialize function and only virtual spaces

   768    that are larger than LargePageSizeInBytes use large pages.  Once we

   769    have determined this, all expand_by and shrink_by calls must grow and

   770    shrink by large page size chunks.  If a particular request

   771    is within the current large page, the call to commit and uncommit memory

   772    can be ignored.  In the case that the low and high boundaries of this

   773    space is not large page aligned, the pages leading to the first large

   774    page address and the pages after the last large page address must be

   775    allocated with default pages.

   776 */

   777 bool VirtualSpace::expand_by(size_t bytes, bool pre_touch) {

   778   if (uncommitted_size() < bytes) return false;

   779 

   780   if (special()) {

   781     // don't commit memory if the entire space is pinned in memory

   782     _high += bytes;

   783     return true;

   784   }

   785 

   786   char* previous_high = high();

   787   char* unaligned_new_high = high() + bytes;

   788   assert(unaligned_new_high <= high_boundary(),

   789          "cannot expand by more than upper boundary");

   790 

   791   // Calculate where the new high for each of the regions should be.  If

   792   // the low_boundary() and high_boundary() are LargePageSizeInBytes aligned

   793   // then the unaligned lower and upper new highs would be the

   794   // lower_high() and upper_high() respectively.

   795   char* unaligned_lower_new_high =

   796     MIN2(unaligned_new_high, lower_high_boundary());

   797   char* unaligned_middle_new_high =

   798     MIN2(unaligned_new_high, middle_high_boundary());

   799   char* unaligned_upper_new_high =

   800     MIN2(unaligned_new_high, upper_high_boundary());

   801 

   802   // Align the new highs based on the regions alignment.  lower and upper

   803   // alignment will always be default page size.  middle alignment will be

   804   // LargePageSizeInBytes if the actual size of the virtual space is in

   805   // fact larger than LargePageSizeInBytes.

   806   char* aligned_lower_new_high =

   807     (char*) round_to((intptr_t) unaligned_lower_new_high, lower_alignment());

   808   char* aligned_middle_new_high =

   809     (char*) round_to((intptr_t) unaligned_middle_new_high, middle_alignment());

   810   char* aligned_upper_new_high =

   811     (char*) round_to((intptr_t) unaligned_upper_new_high, upper_alignment());

   812 

   813   // Determine which regions need to grow in this expand_by call.

   814   // If you are growing in the lower region, high() must be in that

   815   // region so calculate the size based on high().  For the middle and

   816   // upper regions, determine the starting point of growth based on the

   817   // location of high().  By getting the MAX of the region's low address

   818   // (or the previous region's high address) and high(), we can tell if it

   819   // is an intra or inter region growth.

   820   size_t lower_needs = 0;

   821   if (aligned_lower_new_high > lower_high()) {

   822     lower_needs =

   823       pointer_delta(aligned_lower_new_high, lower_high(), sizeof(char));

   824   }

   825   size_t middle_needs = 0;

   826   if (aligned_middle_new_high > middle_high()) {

   827     middle_needs =

   828       pointer_delta(aligned_middle_new_high, middle_high(), sizeof(char));

   829   }

   830   size_t upper_needs = 0;

   831   if (aligned_upper_new_high > upper_high()) {

   832     upper_needs =

   833       pointer_delta(aligned_upper_new_high, upper_high(), sizeof(char));

   834   }

   835 

   836   // Check contiguity.

   837   assert(low_boundary() <= lower_high() &&

   838          lower_high() <= lower_high_boundary(),

   839          "high address must be contained within the region");

   840   assert(lower_high_boundary() <= middle_high() &&

   841          middle_high() <= middle_high_boundary(),

   842          "high address must be contained within the region");

   843   assert(middle_high_boundary() <= upper_high() &&

   844          upper_high() <= upper_high_boundary(),

   845          "high address must be contained within the region");

   846 

   847   // Commit regions

   848   if (lower_needs > 0) {

   849     assert(low_boundary() <= lower_high() &&

   850            lower_high() + lower_needs <= lower_high_boundary(),

   851            "must not expand beyond region");

   852     if (!os::commit_memory(lower_high(), lower_needs, _executable)) {

   853       debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT

   854                          ", lower_needs=" SIZE_FORMAT ", %d) failed",

   855                          lower_high(), lower_needs, _executable);)

   856       return false;

   857     } else {

   858       _lower_high += lower_needs;

   859     }

   860   }

   861   if (middle_needs > 0) {

   862     assert(lower_high_boundary() <= middle_high() &&

   863            middle_high() + middle_needs <= middle_high_boundary(),

   864            "must not expand beyond region");

   865     if (!os::commit_memory(middle_high(), middle_needs, middle_alignment(),

   866                            _executable)) {

   867       debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT

   868                          ", middle_needs=" SIZE_FORMAT ", " SIZE_FORMAT

   869                          ", %d) failed", middle_high(), middle_needs,

   870                          middle_alignment(), _executable);)

   871       return false;

   872     }

   873     _middle_high += middle_needs;

   874   }

   875   if (upper_needs > 0) {

   876     assert(middle_high_boundary() <= upper_high() &&

   877            upper_high() + upper_needs <= upper_high_boundary(),

   878            "must not expand beyond region");

   879     if (!os::commit_memory(upper_high(), upper_needs, _executable)) {

   880       debug_only(warning("INFO: os::commit_memory(" PTR_FORMAT

   881                          ", upper_needs=" SIZE_FORMAT ", %d) failed",

   882                          upper_high(), upper_needs, _executable);)

   883       return false;

   884     } else {

   885       _upper_high += upper_needs;

   886     }

   887   }

   888 

   889   if (pre_touch || AlwaysPreTouch) {

   890     os::pretouch_memory(previous_high, unaligned_new_high);

   891   }

   892 

   893   _high += bytes;

   894   return true;

   895 }

   896 

   897 // A page is uncommitted if the contents of the entire page is deemed unusable.

   898 // Continue to decrement the high() pointer until it reaches a page boundary

   899 // in which case that particular page can now be uncommitted.

   900 void VirtualSpace::shrink_by(size_t size) {

   901   if (committed_size() < size)

   902     fatal("Cannot shrink virtual space to negative size");

   903 

   904   if (special()) {

   905     // don't uncommit if the entire space is pinned in memory

   906     _high -= size;

   907     return;

   908   }

   909 

   910   char* unaligned_new_high = high() - size;

   911   assert(unaligned_new_high >= low_boundary(), "cannot shrink past lower boundary");

   912 

   913   // Calculate new unaligned address

   914   char* unaligned_upper_new_high =

   915     MAX2(unaligned_new_high, middle_high_boundary());

   916   char* unaligned_middle_new_high =

   917     MAX2(unaligned_new_high, lower_high_boundary());

   918   char* unaligned_lower_new_high =

   919     MAX2(unaligned_new_high, low_boundary());

   920 

   921   // Align address to region's alignment

   922   char* aligned_upper_new_high =

   923     (char*) round_to((intptr_t) unaligned_upper_new_high, upper_alignment());

   924   char* aligned_middle_new_high =

   925     (char*) round_to((intptr_t) unaligned_middle_new_high, middle_alignment());

   926   char* aligned_lower_new_high =

   927     (char*) round_to((intptr_t) unaligned_lower_new_high, lower_alignment());

   928 

   929   // Determine which regions need to shrink

   930   size_t upper_needs = 0;

   931   if (aligned_upper_new_high < upper_high()) {

   932     upper_needs =

   933       pointer_delta(upper_high(), aligned_upper_new_high, sizeof(char));

   934   }

   935   size_t middle_needs = 0;

   936   if (aligned_middle_new_high < middle_high()) {

   937     middle_needs =

   938       pointer_delta(middle_high(), aligned_middle_new_high, sizeof(char));

   939   }

   940   size_t lower_needs = 0;

   941   if (aligned_lower_new_high < lower_high()) {

   942     lower_needs =

   943       pointer_delta(lower_high(), aligned_lower_new_high, sizeof(char));

   944   }

   945 

   946   // Check contiguity.

   947   assert(middle_high_boundary() <= upper_high() &&

   948          upper_high() <= upper_high_boundary(),

   949          "high address must be contained within the region");

   950   assert(lower_high_boundary() <= middle_high() &&

   951          middle_high() <= middle_high_boundary(),

   952          "high address must be contained within the region");

   953   assert(low_boundary() <= lower_high() &&

   954          lower_high() <= lower_high_boundary(),

   955          "high address must be contained within the region");

   956 

   957   // Uncommit

   958   if (upper_needs > 0) {

   959     assert(middle_high_boundary() <= aligned_upper_new_high &&

   960            aligned_upper_new_high + upper_needs <= upper_high_boundary(),

   961            "must not shrink beyond region");

   962     if (!os::uncommit_memory(aligned_upper_new_high, upper_needs)) {

   963       debug_only(warning("os::uncommit_memory failed"));

   964       return;

   965     } else {

   966       _upper_high -= upper_needs;

   967     }

   968   }

   969   if (middle_needs > 0) {

   970     assert(lower_high_boundary() <= aligned_middle_new_high &&

   971            aligned_middle_new_high + middle_needs <= middle_high_boundary(),

   972            "must not shrink beyond region");

   973     if (!os::uncommit_memory(aligned_middle_new_high, middle_needs)) {

   974       debug_only(warning("os::uncommit_memory failed"));

   975       return;

   976     } else {

   977       _middle_high -= middle_needs;

   978     }

   979   }

   980   if (lower_needs > 0) {

   981     assert(low_boundary() <= aligned_lower_new_high &&

   982            aligned_lower_new_high + lower_needs <= lower_high_boundary(),

   983            "must not shrink beyond region");

   984     if (!os::uncommit_memory(aligned_lower_new_high, lower_needs)) {

   985       debug_only(warning("os::uncommit_memory failed"));

   986       return;

   987     } else {

   988       _lower_high -= lower_needs;

   989     }

   990   }

   991 

   992   _high -= size;

   993 }

   994 

   995 #ifndef PRODUCT

   996 void VirtualSpace::check_for_contiguity() {

   997   // Check contiguity.

   998   assert(low_boundary() <= lower_high() &&

   999          lower_high() <= lower_high_boundary(),

  1000          "high address must be contained within the region");

  1001   assert(lower_high_boundary() <= middle_high() &&

  1002          middle_high() <= middle_high_boundary(),

  1003          "high address must be contained within the region");

  1004   assert(middle_high_boundary() <= upper_high() &&

  1005          upper_high() <= upper_high_boundary(),

  1006          "high address must be contained within the region");

  1007   assert(low() >= low_boundary(), "low");

  1008   assert(low_boundary() <= lower_high_boundary(), "lower high boundary");

  1009   assert(upper_high_boundary() <= high_boundary(), "upper high boundary");

  1010   assert(high() <= upper_high(), "upper high");

  1011 }

  1012 

  1013 void VirtualSpace::print_on(outputStream* out) {

  1014   out->print   ("Virtual space:");

  1015   if (special()) out->print(" (pinned in memory)");

  1016   out->cr();

  1017   out->print_cr(" - committed: " SIZE_FORMAT, committed_size());

  1018   out->print_cr(" - reserved:  " SIZE_FORMAT, reserved_size());

  1019   out->print_cr(" - [low, high]:     [" INTPTR_FORMAT ", " INTPTR_FORMAT "]",  low(), high());

  1020   out->print_cr(" - [low_b, high_b]: [" INTPTR_FORMAT ", " INTPTR_FORMAT "]",  low_boundary(), high_boundary());

  1021 }

  1022 

  1023 void VirtualSpace::print() {

  1024   print_on(tty);

  1025 }

  1026 

  1027 /////////////// Unit tests ///////////////

  1028 

  1029 #ifndef PRODUCT

  1030 

  1031 #define test_log(...) \

  1032   do {\

  1033     if (VerboseInternalVMTests) { \

  1034       tty->print_cr(__VA_ARGS__); \

  1035       tty->flush(); \

  1036     }\

  1037   } while (false)

  1038 

  1039 class TestReservedSpace : AllStatic {

  1040  public:

  1041   static void small_page_write(void* addr, size_t size) {

  1042     size_t page_size = os::vm_page_size();

  1043 

  1044     char* end = (char*)addr + size;

  1045     for (char* p = (char*)addr; p < end; p += page_size) {

  1046       *p = 1;

  1047     }

  1048   }

  1049 

  1050   static void release_memory_for_test(ReservedSpace rs) {

  1051     if (rs.special()) {

  1052       guarantee(os::release_memory_special(rs.base(), rs.size()), "Shouldn't fail");

  1053     } else {

  1054       guarantee(os::release_memory(rs.base(), rs.size()), "Shouldn't fail");

  1055     }

  1056   }

  1057 

  1058   static void test_reserved_space1(size_t size, size_t alignment) {

  1059     test_log("test_reserved_space1(%p)", (void*) (uintptr_t) size);

  1060 

  1061     assert(is_size_aligned(size, alignment), "Incorrect input parameters");

  1062 

  1063     ReservedSpace rs(size,          // size

  1064                      alignment,     // alignment

  1065                      UseLargePages, // large

  1066                      (char *)NULL); // requested_address

  1067 

  1068     test_log(" rs.special() == %d", rs.special());

  1069 

  1070     assert(rs.base() != NULL, "Must be");

  1071     assert(rs.size() == size, "Must be");

  1072 

  1073     assert(is_ptr_aligned(rs.base(), alignment), "aligned sizes should always give aligned addresses");

  1074     assert(is_size_aligned(rs.size(), alignment), "aligned sizes should always give aligned addresses");

  1075 

  1076     if (rs.special()) {

  1077       small_page_write(rs.base(), size);

  1078     }

  1079 

  1080     release_memory_for_test(rs);

  1081   }

  1082 

  1083   static void test_reserved_space2(size_t size) {

  1084     test_log("test_reserved_space2(%p)", (void*)(uintptr_t)size);

  1085 

  1086     assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");

  1087 

  1088     ReservedSpace rs(size);

  1089 

  1090     test_log(" rs.special() == %d", rs.special());

  1091 

  1092     assert(rs.base() != NULL, "Must be");

  1093     assert(rs.size() == size, "Must be");

  1094 

  1095     if (rs.special()) {

  1096       small_page_write(rs.base(), size);

  1097     }

  1098 

  1099     release_memory_for_test(rs);

  1100   }

  1101 

  1102   static void test_reserved_space3(size_t size, size_t alignment, bool maybe_large) {

  1103     test_log("test_reserved_space3(%p, %p, %d)",

  1104         (void*)(uintptr_t)size, (void*)(uintptr_t)alignment, maybe_large);

  1105 

  1106     assert(is_size_aligned(size, os::vm_allocation_granularity()), "Must be at least AG aligned");

  1107     assert(is_size_aligned(size, alignment), "Must be at least aligned against alignment");

  1108 

  1109     bool large = maybe_large && UseLargePages && size >= os::large_page_size();

  1110 

  1111     ReservedSpace rs(size, alignment, large, false);

  1112 

  1113     test_log(" rs.special() == %d", rs.special());

  1114 

  1115     assert(rs.base() != NULL, "Must be");

  1116     assert(rs.size() == size, "Must be");

  1117 

  1118     if (rs.special()) {

  1119       small_page_write(rs.base(), size);

  1120     }

  1121 

  1122     release_memory_for_test(rs);

  1123   }

  1124 

  1125 

  1126   static void test_reserved_space1() {

  1127     size_t size = 2 * 1024 * 1024;

  1128     size_t ag   = os::vm_allocation_granularity();

  1129 

  1130     test_reserved_space1(size,      ag);

  1131     test_reserved_space1(size * 2,  ag);

  1132     test_reserved_space1(size * 10, ag);

  1133   }

  1134 

  1135   static void test_reserved_space2() {