/*

 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "oops/markOop.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/virtualspace.hpp"

#include "services/memTracker.hpp"

PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC

// ReservedSpace

// Dummy constructor

ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),

    _alignment(0), _special(false), _executable(false) {

}

ReservedSpace::ReservedSpace(size_t size) {

  // Want to use large pages where possible and pad with small pages.

  size_t page_size = os::page_size_for_region_unaligned(size, 1);

  bool large_pages = page_size != (size_t)os::vm_page_size();

  // Don't force the alignment to be large page aligned,

  // since that will waste memory.

  size_t alignment = os::vm_allocation_granularity();

  initialize(size, alignment, large_pages, NULL, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             char* requested_address) {

  initialize(size, alignment, large, requested_address, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             bool executable) {

  initialize(size, alignment, large, NULL, executable);

}

// Helper method.

static bool failed_to_reserve_as_requested(char* base, char* requested_address,

                                           const size_t size, bool special)

{

  if (base == requested_address || requested_address == NULL)

    return false; // did not fail

  if (base != NULL) {

    // Different reserve address may be acceptable in other cases

    // but for compressed oops heap should be at requested address.

    assert(UseCompressedOops, "currently requested address used only for compressed oops");

    if (PrintCompressedOopsMode) {

      tty->cr();

      tty->print_cr("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, base, requested_address);

    }

    // OS ignored requested address. Try different address.

    if (special) {

      if (!os::release_memory_special(base, size)) {

        fatal("os::release_memory_special failed");

      }

    } else {

      if (!os::release_memory(base, size)) {

        fatal("os::release_memory failed");

      }

    }

  }

  return true;

}

void ReservedSpace::initialize(size_t size, size_t alignment, bool large,

                               char* requested_address,

                               bool executable) {

  const size_t granularity = os::vm_allocation_granularity();

  assert((size & (granularity - 1)) == 0,

         "size not aligned to os::vm_allocation_granularity()");

  assert((alignment & (granularity - 1)) == 0,

         "alignment not aligned to os::vm_allocation_granularity()");

  assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

         "not a power of 2");

  alignment = MAX2(alignment, (size_t)os::vm_page_size());

  _base = NULL;

  _size = 0;

  _special = false;

  _executable = executable;

  _alignment = 0;

  _noaccess_prefix = 0;

  if (size == 0) {

    return;

  }

  // If OS doesn't support demand paging for large page memory, we need

  // to use reserve_memory_special() to reserve and pin the entire region.

  bool special = large && !os::can_commit_large_page_memory();

  char* base = NULL;

  if (special) {

    base = os::reserve_memory_special(size, alignment, requested_address, executable);

    if (base != NULL) {

      if (failed_to_reserve_as_requested(base, requested_address, size, true)) {

        // OS ignored requested address. Try different address.

        return;

      }

      // Check alignment constraints.

      assert((uintptr_t) base % alignment == 0,

             err_msg("Large pages returned a non-aligned address, base: "

                 PTR_FORMAT " alignment: " PTR_FORMAT,

                 base, (void*)(uintptr_t)alignment));

      _special = true;

    } else {

      // failed; try to reserve regular memory below

      if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                            !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

        if (PrintCompressedOopsMode) {

          tty->cr();

          tty->print_cr("Reserve regular memory without large pages.");

        }

      }

    }

  }

  if (base == NULL) {

    // Optimistically assume that the OSes returns an aligned base pointer.

    // When reserving a large address range, most OSes seem to align to at

    // least 64K.

    // If the memory was requested at a particular address, use

    // os::attempt_reserve_memory_at() to avoid over mapping something

    // important.  If available space is not detected, return NULL.

    if (requested_address != 0) {

      base = os::attempt_reserve_memory_at(size, requested_address);

      if (failed_to_reserve_as_requested(base, requested_address, size, false)) {

        // OS ignored requested address. Try different address.

        base = NULL;

      }

    } else {

      base = os::reserve_memory(size, NULL, alignment);

    }

    if (base == NULL) return;

    // Check alignment constraints

    if ((((size_t)base) & (alignment - 1)) != 0) {

      // Base not aligned, retry

      if (!os::release_memory(base, size)) fatal("os::release_memory failed");

      // Make sure that size is aligned

      size = align_size_up(size, alignment);

      base = os::reserve_memory_aligned(size, alignment);

      if (requested_address != 0 &&

          failed_to_reserve_as_requested(base, requested_address, size, false)) {

        // As a result of the alignment constraints, the allocated base differs

        // from the requested address. Return back to the caller who can

        // take remedial action (like try again without a requested address).

        assert(_base == NULL, "should be");

        return;

      }

    }

  }

  // Done

  _base = base;

  _size = size;

  _alignment = alignment;

}

ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,

                             bool special, bool executable) {

  assert((size % os::vm_allocation_granularity()) == 0,

         "size not allocation aligned");

  _base = base;

  _size = size;

  _alignment = alignment;

  _noaccess_prefix = 0;

  _special = special;

  _executable = executable;

}

ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,

                                        bool split, bool realloc) {

  assert(partition_size <= size(), "partition failed");

  if (split) {

    os::split_reserved_memory(base(), size(), partition_size, realloc);

  }

  ReservedSpace result(base(), partition_size, alignment, special(),

                       executable());

  return result;

}

ReservedSpace

ReservedSpace::last_part(size_t partition_size, size_t alignment) {

  assert(partition_size <= size(), "partition failed");

  ReservedSpace result(base() + partition_size, size() - partition_size,

                       alignment, special(), executable());

  return result;

}

size_t ReservedSpace::page_align_size_up(size_t size) {

  return align_size_up(size, os::vm_page_size());

}

size_t ReservedSpace::page_align_size_down(size_t size) {

  return align_size_down(size, os::vm_page_size());

}

size_t ReservedSpace::allocation_align_size_up(size_t size) {

  return align_size_up(size, os::vm_allocation_granularity());

}

size_t ReservedSpace::allocation_align_size_down(size_t size) {

  return align_size_down(size, os::vm_allocation_granularity());

}

void ReservedSpace::release() {

  if (is_reserved()) {

    char *real_base = _base - _noaccess_prefix;

    const size_t real_size = _size + _noaccess_prefix;

    if (special()) {

      os::release_memory_special(real_base, real_size);

    } else{

      os::release_memory(real_base, real_size);

    }

    _base = NULL;

    _size = 0;

    _noaccess_prefix = 0;

    _alignment = 0;

    _special = false;

    _executable = false;

  }

}

static size_t noaccess_prefix_size(size_t alignment) {

  return lcm(os::vm_page_size(), alignment);

}

void ReservedHeapSpace::establish_noaccess_prefix() {

  assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");

  _noaccess_prefix = noaccess_prefix_size(_alignment);

  if (base() && base() + _size > (char *)OopEncodingHeapMax) {

    if (true

        WIN64_ONLY(&& !UseLargePages)

        AIX_ONLY(&& os::vm_page_size() != SIZE_64K)) {

      // Protect memory at the base of the allocated region.

      // If special, the page was committed (only matters on windows)

      if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {

        fatal("cannot protect protection page");

      }

      if (PrintCompressedOopsMode) {

        tty->cr();

        tty->print_cr("Protected page at the reserved heap base: "

                      PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix);

      }

      assert(Universe::narrow_oop_use_implicit_null_checks() == true, "not initialized?");

    } else {

      Universe::set_narrow_oop_use_implicit_null_checks(false);

    }

  }

  _base += _noaccess_prefix;

  _size -= _noaccess_prefix;

  assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");

}

// Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.

// Does not check whether the reserved memory actually is at requested_address, as the memory returned

// might still fulfill the wishes of the caller.

// Assures the memory is aligned to 'alignment'.

// NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.

void ReservedHeapSpace::try_reserve_heap(size_t size,

                                         size_t alignment,

                                         bool large,

                                         char* requested_address) {

  if (_base != NULL) {

    // We tried before, but we didn't like the address delivered.

    release();

  }

  // If OS doesn't support demand paging for large page memory, we need

  // to use reserve_memory_special() to reserve and pin the entire region.

  bool special = large && !os::can_commit_large_page_memory();

  char* base = NULL;

  if (PrintCompressedOopsMode && Verbose) {

    tty->print("Trying to allocate at address " PTR_FORMAT " heap of size " PTR_FORMAT ".\n",

               requested_address, (address)size);

  }

  if (special) {

    base = os::reserve_memory_special(size, alignment, requested_address, false);

    if (base != NULL) {

      // Check alignment constraints.

      assert((uintptr_t) base % alignment == 0,

             err_msg("Large pages returned a non-aligned address, base: "

                     PTR_FORMAT " alignment: " PTR_FORMAT,

                     base, (void*)(uintptr_t)alignment));

      _special = true;

    }

  }

  if (base == NULL) {

    // Failed; try to reserve regular memory below

    if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                          !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

      if (PrintCompressedOopsMode) {

        tty->cr();

        tty->print_cr("Reserve regular memory without large pages.");

      }

    }

    // Optimistically assume that the OSes returns an aligned base pointer.

    // When reserving a large address range, most OSes seem to align to at

    // least 64K.

    // If the memory was requested at a particular address, use

    // os::attempt_reserve_memory_at() to avoid over mapping something

    // important.  If available space is not detected, return NULL.

    if (requested_address != 0) {

      base = os::attempt_reserve_memory_at(size, requested_address);

    } else {

      base = os::reserve_memory(size, NULL, alignment);

    }

  }

  if (base == NULL) { return; }

  // Done

  _base = base;

  _size = size;

  _alignment = alignment;

  // Check alignment constraints

  if ((((size_t)base) & (alignment - 1)) != 0) {

    // Base not aligned, retry.

    release();

  }

}

void ReservedHeapSpace::try_reserve_range(char *highest_start,

                                          char *lowest_start,

                                          size_t attach_point_alignment,

                                          char *aligned_heap_base_min_address,

                                          char *upper_bound,

                                          size_t size,

                                          size_t alignment,

                                          bool large) {

  const size_t attach_range = highest_start - lowest_start;

  // Cap num_attempts at possible number.

  // At least one is possible even for 0 sized attach range.

  const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;

  const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);

  const size_t stepsize = (attach_range == 0) ? // Only one try.

    (size_t) highest_start : align_size_up(attach_range / num_attempts_to_try, attach_point_alignment);

  // Try attach points from top to bottom.

  char* attach_point = highest_start;

  while (attach_point >= lowest_start  &&

         attach_point <= highest_start &&  // Avoid wrap around.

         ((_base == NULL) ||

          (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {

    try_reserve_heap(size, alignment, large, attach_point);

    attach_point -= stepsize;

  }

}

#define SIZE_64K  ((uint64_t) UCONST64(      0x10000))

#define SIZE_256M ((uint64_t) UCONST64(   0x10000000))

#define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))

// Helper for heap allocation. Returns an array with addresses

// (OS-specific) which are suited for disjoint base mode. Array is

// NULL terminated.

static char** get_attach_addresses_for_disjoint_mode() {

  static uint64_t addresses[] = {

     2 * SIZE_32G,

     3 * SIZE_32G,

     4 * SIZE_32G,

     8 * SIZE_32G,

    10 * SIZE_32G,

     1 * SIZE_64K * SIZE_32G,

     2 * SIZE_64K * SIZE_32G,

     3 * SIZE_64K * SIZE_32G,

     4 * SIZE_64K * SIZE_32G,

    16 * SIZE_64K * SIZE_32G,

    32 * SIZE_64K * SIZE_32G,

    34 * SIZE_64K * SIZE_32G,

    0

  };

  // Sort out addresses smaller than HeapBaseMinAddress. This assumes

  // the array is sorted.

  uint i = 0;

  while (addresses[i] != 0 &&

         (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {

    i++;

  }

  uint start = i;

  // Avoid more steps than requested.

  i = 0;

  while (addresses[start+i] != 0) {

    if (i == HeapSearchSteps) {

      addresses[start+i] = 0;

      break;

    }

    i++;

  }

  return (char**) &addresses[start];

}

void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, bool large) {

  guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,

            "can not allocate compressed oop heap for this size");

  guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");

  assert(HeapBaseMinAddress > 0, "sanity");

  const size_t granularity = os::vm_allocation_granularity();

  assert((size & (granularity - 1)) == 0,

         "size not aligned to os::vm_allocation_granularity()");

  assert((alignment & (granularity - 1)) == 0,

         "alignment not aligned to os::vm_allocation_granularity()");

  assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

         "not a power of 2");

  // The necessary attach point alignment for generated wish addresses.

  // This is needed to increase the chance of attaching for mmap and shmat.

  const size_t os_attach_point_alignment =

    AIX_ONLY(SIZE_256M)  // Known shm boundary alignment.

    NOT_AIX(os::vm_allocation_granularity());

  const size_t attach_point_alignment = lcm(alignment, os_attach_point_alignment);

  char *aligned_heap_base_min_address = (char *)align_ptr_up((void *)HeapBaseMinAddress, alignment);

  size_t noaccess_prefix = ((aligned_heap_base_min_address + size) > (char*)OopEncodingHeapMax) ?

    noaccess_prefix_size(alignment) : 0;

  // Attempt to alloc at user-given address.

  if (!FLAG_IS_DEFAULT(HeapBaseMinAddress)) {

    try_reserve_heap(size + noaccess_prefix, alignment, large, aligned_heap_base_min_address);

    if (_base != aligned_heap_base_min_address) { // Enforce this exact address.

      release();

    }

  }

  // Keep heap at HeapBaseMinAddress.

  if (_base == NULL) {

    // Try to allocate the heap at addresses that allow efficient oop compression.

    // Different schemes are tried, in order of decreasing optimization potential.

    //

    // For this, try_reserve_heap() is called with the desired heap base addresses.

    // A call into the os layer to allocate at a given address can return memory

    // at a different address than requested.  Still, this might be memory at a useful

    // address. try_reserve_heap() always returns this allocated memory, as only here

    // the criteria for a good heap are checked.

    // Attempt to allocate so that we can run without base and scale (32-Bit unscaled compressed oops).

    // Give it several tries from top of range to bottom.

    if (aligned_heap_base_min_address + size <= (char *)UnscaledOopHeapMax) {

      // Calc address range within we try to attach (range of possible start addresses).

      char* const highest_start = (char *)align_ptr_down((char *)UnscaledOopHeapMax - size, attach_point_alignment);

      char* const lowest_start  = (char *)align_ptr_up  (        aligned_heap_base_min_address             , attach_point_alignment);

      try_reserve_range(highest_start, lowest_start, attach_point_alignment,

                        aligned_heap_base_min_address, (char *)UnscaledOopHeapMax, size, alignment, large);