/*

 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "logging/log.hpp"

#include "memory/resourceArea.hpp"

#include "memory/virtualspace.hpp"

#include "oops/compressedOops.hpp"

#include "oops/markWord.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/os.inline.hpp"

#include "services/memTracker.hpp"

#include "utilities/align.hpp"

// ReservedSpace

// Dummy constructor

ReservedSpace::ReservedSpace() : _base(NULL), _size(0), _noaccess_prefix(0),

    _alignment(0), _special(false), _fd_for_heap(-1), _executable(false) {

}

ReservedSpace::ReservedSpace(size_t size, size_t preferred_page_size) : _fd_for_heap(-1) {

  bool has_preferred_page_size = preferred_page_size != 0;

  // Want to use large pages where possible and pad with small pages.

  size_t page_size = has_preferred_page_size ? preferred_page_size : os::page_size_for_region_unaligned(size, 1);

  bool large_pages = page_size != (size_t)os::vm_page_size();

  size_t alignment;

  if (large_pages && has_preferred_page_size) {

    alignment = MAX2(page_size, (size_t)os::vm_allocation_granularity());

    // ReservedSpace initialization requires size to be aligned to the given

    // alignment. Align the size up.

    size = align_up(size, alignment);

  } else {

    // Don't force the alignment to be large page aligned,

    // since that will waste memory.

    alignment = os::vm_allocation_granularity();

  }

  initialize(size, alignment, large_pages, NULL, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             char* requested_address) : _fd_for_heap(-1) {

  initialize(size, alignment, large, requested_address, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             bool executable) : _fd_for_heap(-1) {

  initialize(size, alignment, large, NULL, executable);

}

ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,

                             bool special, bool executable) : _fd_for_heap(-1) {

  assert((size % os::vm_allocation_granularity()) == 0,

         "size not allocation aligned");

  _base = base;

  _size = size;

  _alignment = alignment;

  _noaccess_prefix = 0;

  _special = special;

  _executable = executable;

}

// Helper method

static void unmap_or_release_memory(char* base, size_t size, bool is_file_mapped) {

  if (is_file_mapped) {

    if (!os::unmap_memory(base, size)) {

      fatal("os::unmap_memory failed");

    }

  } else if (!os::release_memory(base, size)) {

    fatal("os::release_memory failed");

  }

}

// Helper method.

static bool failed_to_reserve_as_requested(char* base, char* requested_address,

                                           const size_t size, bool special, bool is_file_mapped = false)

{

  if (base == requested_address || requested_address == NULL)

    return false; // did not fail

  if (base != NULL) {

    // Different reserve address may be acceptable in other cases

    // but for compressed oops heap should be at requested address.

    assert(UseCompressedOops, "currently requested address used only for compressed oops");

    log_debug(gc, heap, coops)("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, p2i(base), p2i(requested_address));

    // OS ignored requested address. Try different address.

    if (special) {

      if (!os::release_memory_special(base, size)) {

        fatal("os::release_memory_special failed");

      }

    } else {

      unmap_or_release_memory(base, size, is_file_mapped);

    }

  }

  return true;

}

void ReservedSpace::initialize(size_t size, size_t alignment, bool large,

                               char* requested_address,

                               bool executable) {

  const size_t granularity = os::vm_allocation_granularity();

  assert((size & (granularity - 1)) == 0,

         "size not aligned to os::vm_allocation_granularity()");

  assert((alignment & (granularity - 1)) == 0,

         "alignment not aligned to os::vm_allocation_granularity()");

  assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

         "not a power of 2");

  alignment = MAX2(alignment, (size_t)os::vm_page_size());

  _base = NULL;

  _size = 0;

  _special = false;

  _executable = executable;

  _alignment = 0;

  _noaccess_prefix = 0;

  if (size == 0) {

    return;

  }

  // If OS doesn't support demand paging for large page memory, we need

  // to use reserve_memory_special() to reserve and pin the entire region.

  // If there is a backing file directory for this space then whether

  // large pages are allocated is up to the filesystem of the backing file.

  // So we ignore the UseLargePages flag in this case.

  bool special = large && !os::can_commit_large_page_memory();

  if (special && _fd_for_heap != -1) {

    special = false;

    if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

      !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

      log_debug(gc, heap)("Ignoring UseLargePages since large page support is up to the file system of the backing file for Java heap");

    }

  }

  char* base = NULL;

  if (special) {

    base = os::reserve_memory_special(size, alignment, requested_address, executable);

    if (base != NULL) {

      if (failed_to_reserve_as_requested(base, requested_address, size, true)) {

        // OS ignored requested address. Try different address.

        return;

      }

      // Check alignment constraints.

      assert((uintptr_t) base % alignment == 0,

             "Large pages returned a non-aligned address, base: "

             PTR_FORMAT " alignment: " SIZE_FORMAT_HEX,

             p2i(base), alignment);

      _special = true;

    } else {

      // failed; try to reserve regular memory below

      if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                            !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

        log_debug(gc, heap, coops)("Reserve regular memory without large pages");

      }

    }

  }

  if (base == NULL) {

    // Optimistically assume that the OSes returns an aligned base pointer.

    // When reserving a large address range, most OSes seem to align to at

    // least 64K.

    // If the memory was requested at a particular address, use

    // os::attempt_reserve_memory_at() to avoid over mapping something

    // important.  If available space is not detected, return NULL.

    if (requested_address != 0) {

      base = os::attempt_reserve_memory_at(size, requested_address, _fd_for_heap);

      if (failed_to_reserve_as_requested(base, requested_address, size, false, _fd_for_heap != -1)) {

        // OS ignored requested address. Try different address.

        base = NULL;

      }

    } else {

      base = os::reserve_memory(size, NULL, alignment, _fd_for_heap);

    }

    if (base == NULL) return;

    // Check alignment constraints

    if ((((size_t)base) & (alignment - 1)) != 0) {

      // Base not aligned, retry

      unmap_or_release_memory(base, size, _fd_for_heap != -1 /*is_file_mapped*/);

      // Make sure that size is aligned

      size = align_up(size, alignment);

      base = os::reserve_memory_aligned(size, alignment, _fd_for_heap);

      if (requested_address != 0 &&

          failed_to_reserve_as_requested(base, requested_address, size, false, _fd_for_heap != -1)) {

        // As a result of the alignment constraints, the allocated base differs

        // from the requested address. Return back to the caller who can

        // take remedial action (like try again without a requested address).

        assert(_base == NULL, "should be");

        return;

      }

    }

  }

  // Done

  _base = base;

  _size = size;

  _alignment = alignment;

  // If heap is reserved with a backing file, the entire space has been committed. So set the _special flag to true

  if (_fd_for_heap != -1) {

    _special = true;

  }

}

ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,

                                        bool split, bool realloc) {

  assert(partition_size <= size(), "partition failed");

  if (split) {

    os::split_reserved_memory(base(), size(), partition_size, realloc);

  }

  ReservedSpace result(base(), partition_size, alignment, special(),

                       executable());

  return result;

}

ReservedSpace

ReservedSpace::last_part(size_t partition_size, size_t alignment) {

  assert(partition_size <= size(), "partition failed");

  ReservedSpace result(base() + partition_size, size() - partition_size,

                       alignment, special(), executable());

  return result;

}

size_t ReservedSpace::page_align_size_up(size_t size) {

  return align_up(size, os::vm_page_size());

}

size_t ReservedSpace::page_align_size_down(size_t size) {

  return align_down(size, os::vm_page_size());

}

size_t ReservedSpace::allocation_align_size_up(size_t size) {

  return align_up(size, os::vm_allocation_granularity());

}

void ReservedSpace::release() {

  if (is_reserved()) {

    char *real_base = _base - _noaccess_prefix;

    const size_t real_size = _size + _noaccess_prefix;

    if (special()) {

      if (_fd_for_heap != -1) {

        os::unmap_memory(real_base, real_size);

      } else {

        os::release_memory_special(real_base, real_size);

      }

    } else{

      os::release_memory(real_base, real_size);

    }

    _base = NULL;

    _size = 0;

    _noaccess_prefix = 0;

    _alignment = 0;

    _special = false;

    _executable = false;

  }

}

static size_t noaccess_prefix_size(size_t alignment) {

  return lcm(os::vm_page_size(), alignment);

}

void ReservedHeapSpace::establish_noaccess_prefix() {

  assert(_alignment >= (size_t)os::vm_page_size(), "must be at least page size big");

  _noaccess_prefix = noaccess_prefix_size(_alignment);

  if (base() && base() + _size > (char *)OopEncodingHeapMax) {

    if (true

        WIN64_ONLY(&& !UseLargePages)

        AIX_ONLY(&& os::vm_page_size() != 64*K)) {

      // Protect memory at the base of the allocated region.

      // If special, the page was committed (only matters on windows)

      if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE, _special)) {

        fatal("cannot protect protection page");

      }

      log_debug(gc, heap, coops)("Protected page at the reserved heap base: "

                                 PTR_FORMAT " / " INTX_FORMAT " bytes",

                                 p2i(_base),

                                 _noaccess_prefix);

      assert(CompressedOops::use_implicit_null_checks() == true, "not initialized?");

    } else {

      CompressedOops::set_use_implicit_null_checks(false);

    }

  }

  _base += _noaccess_prefix;

  _size -= _noaccess_prefix;

  assert(((uintptr_t)_base % _alignment == 0), "must be exactly of required alignment");

}

// Tries to allocate memory of size 'size' at address requested_address with alignment 'alignment'.

// Does not check whether the reserved memory actually is at requested_address, as the memory returned

// might still fulfill the wishes of the caller.

// Assures the memory is aligned to 'alignment'.

// NOTE: If ReservedHeapSpace already points to some reserved memory this is freed, first.

void ReservedHeapSpace::try_reserve_heap(size_t size,

                                         size_t alignment,

                                         bool large,

                                         char* requested_address) {

  if (_base != NULL) {

    // We tried before, but we didn't like the address delivered.

    release();

  }

  // If OS doesn't support demand paging for large page memory, we need

  // to use reserve_memory_special() to reserve and pin the entire region.

  // If there is a backing file directory for this space then whether

  // large pages are allocated is up to the filesystem of the backing file.

  // So we ignore the UseLargePages flag in this case.

  bool special = large && !os::can_commit_large_page_memory();

  if (special && _fd_for_heap != -1) {

    special = false;

    if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                          !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

      log_debug(gc, heap)("Cannot allocate large pages for Java Heap when AllocateHeapAt option is set.");

    }

  }

  char* base = NULL;

  log_trace(gc, heap, coops)("Trying to allocate at address " PTR_FORMAT

                             " heap of size " SIZE_FORMAT_HEX,

                             p2i(requested_address),

                             size);

  if (special) {

    base = os::reserve_memory_special(size, alignment, requested_address, false);

    if (base != NULL) {

      // Check alignment constraints.

      assert((uintptr_t) base % alignment == 0,

             "Large pages returned a non-aligned address, base: "

             PTR_FORMAT " alignment: " SIZE_FORMAT_HEX,

             p2i(base), alignment);

      _special = true;

    }

  }

  if (base == NULL) {

    // Failed; try to reserve regular memory below

    if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                          !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

      log_debug(gc, heap, coops)("Reserve regular memory without large pages");

    }

    // Optimistically assume that the OSes returns an aligned base pointer.

    // When reserving a large address range, most OSes seem to align to at

    // least 64K.

    // If the memory was requested at a particular address, use

    // os::attempt_reserve_memory_at() to avoid over mapping something

    // important.  If available space is not detected, return NULL.

    if (requested_address != 0) {

      base = os::attempt_reserve_memory_at(size, requested_address, _fd_for_heap);

    } else {

      base = os::reserve_memory(size, NULL, alignment, _fd_for_heap);

    }

  }

  if (base == NULL) { return; }

  // Done

  _base = base;

  _size = size;

  _alignment = alignment;

  // If heap is reserved with a backing file, the entire space has been committed. So set the _special flag to true

  if (_fd_for_heap != -1) {

    _special = true;

  }

  // Check alignment constraints

  if ((((size_t)base) & (alignment - 1)) != 0) {

    // Base not aligned, retry.

    release();

  }

}

void ReservedHeapSpace::try_reserve_range(char *highest_start,

                                          char *lowest_start,

                                          size_t attach_point_alignment,

                                          char *aligned_heap_base_min_address,

                                          char *upper_bound,

                                          size_t size,

                                          size_t alignment,

                                          bool large) {

  const size_t attach_range = highest_start - lowest_start;

  // Cap num_attempts at possible number.

  // At least one is possible even for 0 sized attach range.

  const uint64_t num_attempts_possible = (attach_range / attach_point_alignment) + 1;

  const uint64_t num_attempts_to_try   = MIN2((uint64_t)HeapSearchSteps, num_attempts_possible);

  const size_t stepsize = (attach_range == 0) ? // Only one try.

    (size_t) highest_start : align_up(attach_range / num_attempts_to_try, attach_point_alignment);

  // Try attach points from top to bottom.

  char* attach_point = highest_start;

  while (attach_point >= lowest_start  &&

         attach_point <= highest_start &&  // Avoid wrap around.

         ((_base == NULL) ||

          (_base < aligned_heap_base_min_address || _base + size > upper_bound))) {

    try_reserve_heap(size, alignment, large, attach_point);

    attach_point -= stepsize;

  }

}

#define SIZE_64K  ((uint64_t) UCONST64(      0x10000))

#define SIZE_256M ((uint64_t) UCONST64(   0x10000000))

#define SIZE_32G  ((uint64_t) UCONST64(  0x800000000))

// Helper for heap allocation. Returns an array with addresses

// (OS-specific) which are suited for disjoint base mode. Array is

// NULL terminated.

static char** get_attach_addresses_for_disjoint_mode() {

  static uint64_t addresses[] = {

     2 * SIZE_32G,

     3 * SIZE_32G,

     4 * SIZE_32G,

     8 * SIZE_32G,

    10 * SIZE_32G,

     1 * SIZE_64K * SIZE_32G,

     2 * SIZE_64K * SIZE_32G,

     3 * SIZE_64K * SIZE_32G,

     4 * SIZE_64K * SIZE_32G,

    16 * SIZE_64K * SIZE_32G,

    32 * SIZE_64K * SIZE_32G,

    34 * SIZE_64K * SIZE_32G,

    0

  };

  // Sort out addresses smaller than HeapBaseMinAddress. This assumes

  // the array is sorted.

  uint i = 0;

  while (addresses[i] != 0 &&

         (addresses[i] < OopEncodingHeapMax || addresses[i] < HeapBaseMinAddress)) {

    i++;

  }

  uint start = i;

  // Avoid more steps than requested.

  i = 0;

  while (addresses[start+i] != 0) {

    if (i == HeapSearchSteps) {

      addresses[start+i] = 0;

      break;

    }

    i++;

  }

  return (char**) &addresses[start];

}

void ReservedHeapSpace::initialize_compressed_heap(const size_t size, size_t alignment, bool large) {

  guarantee(size + noaccess_prefix_size(alignment) <= OopEncodingHeapMax,

            "can not allocate compressed oop heap for this size");

  guarantee(alignment == MAX2(alignment, (size_t)os::vm_page_size()), "alignment too small");

  const size_t granularity = os::vm_allocation_granularity();

  assert((size & (granularity - 1)) == 0,

         "size not aligned to os::vm_allocation_granularity()");

  assert((alignment & (granularity - 1)) == 0,