/*

 * Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "oops/markOop.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/virtualspace.hpp"

#include "services/memTracker.hpp"

#ifdef TARGET_OS_FAMILY_linux

# include "os_linux.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_solaris

# include "os_solaris.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_windows

# include "os_windows.inline.hpp"

#endif

#ifdef TARGET_OS_FAMILY_bsd

# include "os_bsd.inline.hpp"

#endif

// ReservedSpace

ReservedSpace::ReservedSpace(size_t size) {

  initialize(size, 0, false, NULL, 0, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             char* requested_address,

                             const size_t noaccess_prefix) {

  initialize(size+noaccess_prefix, alignment, large, requested_address,

             noaccess_prefix, false);

}

ReservedSpace::ReservedSpace(size_t size, size_t alignment,

                             bool large,

                             bool executable) {

  initialize(size, alignment, large, NULL, 0, executable);

}

char *

ReservedSpace::align_reserved_region(char* addr, const size_t len,

                                     const size_t prefix_size,

                                     const size_t prefix_align,

                                     const size_t suffix_size,

                                     const size_t suffix_align)

{

  assert(addr != NULL, "sanity");

  const size_t required_size = prefix_size + suffix_size;

  assert(len >= required_size, "len too small");

  const size_t s = size_t(addr);

  const size_t beg_ofs = (s + prefix_size) & (suffix_align - 1);

  const size_t beg_delta = beg_ofs == 0 ? 0 : suffix_align - beg_ofs;

  if (len < beg_delta + required_size) {

     return NULL; // Cannot do proper alignment.

  }

  const size_t end_delta = len - (beg_delta + required_size);

  if (beg_delta != 0) {

    os::release_memory(addr, beg_delta);

  }

  if (end_delta != 0) {

    char* release_addr = (char*) (s + beg_delta + required_size);

    os::release_memory(release_addr, end_delta);

  }

  return (char*) (s + beg_delta);

}

char* ReservedSpace::reserve_and_align(const size_t reserve_size,

                                       const size_t prefix_size,

                                       const size_t prefix_align,

                                       const size_t suffix_size,

                                       const size_t suffix_align)

{

  assert(reserve_size > prefix_size + suffix_size, "should not be here");

  char* raw_addr = os::reserve_memory(reserve_size, NULL, prefix_align);

  if (raw_addr == NULL) return NULL;

  char* result = align_reserved_region(raw_addr, reserve_size, prefix_size,

                                       prefix_align, suffix_size,

                                       suffix_align);

  if (result == NULL && !os::release_memory(raw_addr, reserve_size)) {

    fatal("os::release_memory failed");

  }

#ifdef ASSERT

  if (result != NULL) {

    const size_t raw = size_t(raw_addr);

    const size_t res = size_t(result);

    assert(res >= raw, "alignment decreased start addr");

    assert(res + prefix_size + suffix_size <= raw + reserve_size,

           "alignment increased end addr");

    assert((res & (prefix_align - 1)) == 0, "bad alignment of prefix");

    assert(((res + prefix_size) & (suffix_align - 1)) == 0,

           "bad alignment of suffix");

  }

#endif

  return result;

}

// Helper method.

static bool failed_to_reserve_as_requested(char* base, char* requested_address,

                                           const size_t size, bool special)

{

  if (base == requested_address || requested_address == NULL)

    return false; // did not fail

  if (base != NULL) {

    // Different reserve address may be acceptable in other cases

    // but for compressed oops heap should be at requested address.

    assert(UseCompressedOops, "currently requested address used only for compressed oops");

    if (PrintCompressedOopsMode) {

      tty->cr();

      tty->print_cr("Reserved memory not at requested address: " PTR_FORMAT " vs " PTR_FORMAT, base, requested_address);

    }

    // OS ignored requested address. Try different address.

    if (special) {

      if (!os::release_memory_special(base, size)) {

        fatal("os::release_memory_special failed");

      }

    } else {

      if (!os::release_memory(base, size)) {

        fatal("os::release_memory failed");

      }

    }

  }

  return true;

}

ReservedSpace::ReservedSpace(const size_t prefix_size,

                             const size_t prefix_align,

                             const size_t suffix_size,

                             const size_t suffix_align,

                             char* requested_address,

                             const size_t noaccess_prefix)

{

  assert(prefix_size != 0, "sanity");

  assert(prefix_align != 0, "sanity");

  assert(suffix_size != 0, "sanity");

  assert(suffix_align != 0, "sanity");

  assert((prefix_size & (prefix_align - 1)) == 0,

    "prefix_size not divisible by prefix_align");

  assert((suffix_size & (suffix_align - 1)) == 0,

    "suffix_size not divisible by suffix_align");

  assert((suffix_align & (prefix_align - 1)) == 0,

    "suffix_align not divisible by prefix_align");

  // Assert that if noaccess_prefix is used, it is the same as prefix_align.

  assert(noaccess_prefix == 0 ||

         noaccess_prefix == prefix_align, "noaccess prefix wrong");

  // Add in noaccess_prefix to prefix_size;

  const size_t adjusted_prefix_size = prefix_size + noaccess_prefix;

  const size_t size = adjusted_prefix_size + suffix_size;

  // On systems where the entire region has to be reserved and committed up

  // front, the compound alignment normally done by this method is unnecessary.

  const bool try_reserve_special = UseLargePages &&

    prefix_align == os::large_page_size();

  if (!os::can_commit_large_page_memory() && try_reserve_special) {

    initialize(size, prefix_align, true, requested_address, noaccess_prefix,

               false);

    return;

  }

  _base = NULL;

  _size = 0;

  _alignment = 0;

  _special = false;

  _noaccess_prefix = 0;

  _executable = false;

  // Optimistically try to reserve the exact size needed.

  char* addr;

  if (requested_address != 0) {

    requested_address -= noaccess_prefix; // adjust address

    assert(requested_address != NULL, "huge noaccess prefix?");

    addr = os::attempt_reserve_memory_at(size, requested_address);

    if (failed_to_reserve_as_requested(addr, requested_address, size, false)) {

      // OS ignored requested address. Try different address.

      addr = NULL;

    }

  } else {

    addr = os::reserve_memory(size, NULL, prefix_align);

  }

  if (addr == NULL) return;

  // Check whether the result has the needed alignment (unlikely unless

  // prefix_align < suffix_align).

  const size_t ofs = (size_t(addr) + adjusted_prefix_size) & (suffix_align - 1);

  if (ofs != 0) {

    // Wrong alignment.  Release, allocate more space and do manual alignment.

    //

    // On most operating systems, another allocation with a somewhat larger size

    // will return an address "close to" that of the previous allocation.  The

    // result is often the same address (if the kernel hands out virtual

    // addresses from low to high), or an address that is offset by the increase

    // in size.  Exploit that to minimize the amount of extra space requested.

    if (!os::release_memory(addr, size)) {

      fatal("os::release_memory failed");

    }

    const size_t extra = MAX2(ofs, suffix_align - ofs);

    addr = reserve_and_align(size + extra, adjusted_prefix_size, prefix_align,

                             suffix_size, suffix_align);

    if (addr == NULL) {

      // Try an even larger region.  If this fails, address space is exhausted.

      addr = reserve_and_align(size + suffix_align, adjusted_prefix_size,

                               prefix_align, suffix_size, suffix_align);

    }

    if (requested_address != 0 &&

        failed_to_reserve_as_requested(addr, requested_address, size, false)) {

      // As a result of the alignment constraints, the allocated addr differs

      // from the requested address. Return back to the caller who can

      // take remedial action (like try again without a requested address).

      assert(_base == NULL, "should be");

      return;

    }

  }

  _base = addr;

  _size = size;

  _alignment = prefix_align;

  _noaccess_prefix = noaccess_prefix;

}

void ReservedSpace::initialize(size_t size, size_t alignment, bool large,

                               char* requested_address,

                               const size_t noaccess_prefix,

                               bool executable) {

  const size_t granularity = os::vm_allocation_granularity();

  assert((size & (granularity - 1)) == 0,

         "size not aligned to os::vm_allocation_granularity()");

  assert((alignment & (granularity - 1)) == 0,

         "alignment not aligned to os::vm_allocation_granularity()");

  assert(alignment == 0 || is_power_of_2((intptr_t)alignment),

         "not a power of 2");

  alignment = MAX2(alignment, (size_t)os::vm_page_size());

  // Assert that if noaccess_prefix is used, it is the same as alignment.

  assert(noaccess_prefix == 0 ||

         noaccess_prefix == alignment, "noaccess prefix wrong");

  _base = NULL;

  _size = 0;

  _special = false;

  _executable = executable;

  _alignment = 0;

  _noaccess_prefix = 0;

  if (size == 0) {

    return;

  }

  // If OS doesn't support demand paging for large page memory, we need

  // to use reserve_memory_special() to reserve and pin the entire region.

  bool special = large && !os::can_commit_large_page_memory();

  char* base = NULL;

  if (requested_address != 0) {

    requested_address -= noaccess_prefix; // adjust requested address

    assert(requested_address != NULL, "huge noaccess prefix?");

  }

  if (special) {

    base = os::reserve_memory_special(size, requested_address, executable);

    if (base != NULL) {

      if (failed_to_reserve_as_requested(base, requested_address, size, true)) {

        // OS ignored requested address. Try different address.

        return;

      }

      // Check alignment constraints

      assert((uintptr_t) base % alignment == 0,

             "Large pages returned a non-aligned address");

      _special = true;

    } else {

      // failed; try to reserve regular memory below

      if (UseLargePages && (!FLAG_IS_DEFAULT(UseLargePages) ||

                            !FLAG_IS_DEFAULT(LargePageSizeInBytes))) {

        if (PrintCompressedOopsMode) {

          tty->cr();

          tty->print_cr("Reserve regular memory without large pages.");

        }

      }

    }

  }

  if (base == NULL) {

    // Optimistically assume that the OSes returns an aligned base pointer.

    // When reserving a large address range, most OSes seem to align to at

    // least 64K.

    // If the memory was requested at a particular address, use

    // os::attempt_reserve_memory_at() to avoid over mapping something

    // important.  If available space is not detected, return NULL.

    if (requested_address != 0) {

      base = os::attempt_reserve_memory_at(size, requested_address);

      if (failed_to_reserve_as_requested(base, requested_address, size, false)) {

        // OS ignored requested address. Try different address.

        base = NULL;

      }

    } else {

      base = os::reserve_memory(size, NULL, alignment);

    }

    if (base == NULL) return;

    // Check alignment constraints

    if ((((size_t)base + noaccess_prefix) & (alignment - 1)) != 0) {

      // Base not aligned, retry

      if (!os::release_memory(base, size)) fatal("os::release_memory failed");

      // Reserve size large enough to do manual alignment and

      // increase size to a multiple of the desired alignment

      size = align_size_up(size, alignment);

      size_t extra_size = size + alignment;

      do {

        char* extra_base = os::reserve_memory(extra_size, NULL, alignment);

        if (extra_base == NULL) return;

        // Do manual alignement

        base = (char*) align_size_up((uintptr_t) extra_base, alignment);

        assert(base >= extra_base, "just checking");

        // Re-reserve the region at the aligned base address.

        os::release_memory(extra_base, extra_size);

        base = os::reserve_memory(size, base);

      } while (base == NULL);

      if (requested_address != 0 &&

          failed_to_reserve_as_requested(base, requested_address, size, false)) {

        // As a result of the alignment constraints, the allocated base differs

        // from the requested address. Return back to the caller who can

        // take remedial action (like try again without a requested address).

        assert(_base == NULL, "should be");

        return;

      }

    }

  }

  // Done

  _base = base;

  _size = size;

  _alignment = alignment;

  _noaccess_prefix = noaccess_prefix;

  // Assert that if noaccess_prefix is used, it is the same as alignment.

  assert(noaccess_prefix == 0 ||

         noaccess_prefix == _alignment, "noaccess prefix wrong");

  assert(markOopDesc::encode_pointer_as_mark(_base)->decode_pointer() == _base,

         "area must be distinguisable from marks for mark-sweep");

  assert(markOopDesc::encode_pointer_as_mark(&_base[size])->decode_pointer() == &_base[size],

         "area must be distinguisable from marks for mark-sweep");

}

ReservedSpace::ReservedSpace(char* base, size_t size, size_t alignment,

                             bool special, bool executable) {

  assert((size % os::vm_allocation_granularity()) == 0,

         "size not allocation aligned");

  _base = base;

  _size = size;

  _alignment = alignment;

  _noaccess_prefix = 0;

  _special = special;

  _executable = executable;

}

ReservedSpace ReservedSpace::first_part(size_t partition_size, size_t alignment,

                                        bool split, bool realloc) {

  assert(partition_size <= size(), "partition failed");

  if (split) {

    os::split_reserved_memory(base(), size(), partition_size, realloc);

  }

  ReservedSpace result(base(), partition_size, alignment, special(),

                       executable());

  return result;

}

ReservedSpace

ReservedSpace::last_part(size_t partition_size, size_t alignment) {

  assert(partition_size <= size(), "partition failed");

  ReservedSpace result(base() + partition_size, size() - partition_size,

                       alignment, special(), executable());

  return result;

}

size_t ReservedSpace::page_align_size_up(size_t size) {

  return align_size_up(size, os::vm_page_size());

}

size_t ReservedSpace::page_align_size_down(size_t size) {

  return align_size_down(size, os::vm_page_size());

}

size_t ReservedSpace::allocation_align_size_up(size_t size) {

  return align_size_up(size, os::vm_allocation_granularity());

}

size_t ReservedSpace::allocation_align_size_down(size_t size) {

  return align_size_down(size, os::vm_allocation_granularity());

}

void ReservedSpace::release() {

  if (is_reserved()) {

    char *real_base = _base - _noaccess_prefix;

    const size_t real_size = _size + _noaccess_prefix;

    if (special()) {

      os::release_memory_special(real_base, real_size);

    } else{

      os::release_memory(real_base, real_size);

    }

    _base = NULL;

    _size = 0;

    _noaccess_prefix = 0;

    _special = false;

    _executable = false;

  }

}

void ReservedSpace::protect_noaccess_prefix(const size_t size) {

  assert( (_noaccess_prefix != 0) == (UseCompressedOops && _base != NULL &&

                                      (Universe::narrow_oop_base() != NULL) &&

                                      Universe::narrow_oop_use_implicit_null_checks()),

         "noaccess_prefix should be used only with non zero based compressed oops");

  // If there is no noaccess prefix, return.

  if (_noaccess_prefix == 0) return;

  assert(_noaccess_prefix >= (size_t)os::vm_page_size(),

         "must be at least page size big");

  // Protect memory at the base of the allocated region.

  // If special, the page was committed (only matters on windows)

  if (!os::protect_memory(_base, _noaccess_prefix, os::MEM_PROT_NONE,

                          _special)) {

    fatal("cannot protect protection page");

  }

  if (PrintCompressedOopsMode) {

    tty->cr();

    tty->print_cr("Protected page at the reserved heap base: " PTR_FORMAT " / " INTX_FORMAT " bytes", _base, _noaccess_prefix);

  }

  _base += _noaccess_prefix;

  _size -= _noaccess_prefix;

  assert((size == _size) && ((uintptr_t)_base % _alignment == 0),

         "must be exactly of required size and alignment");

}

ReservedHeapSpace::ReservedHeapSpace(size_t size, size_t alignment,

                                     bool large, char* requested_address) :

  ReservedSpace(size, alignment, large,

                requested_address,

                (UseCompressedOops && (Universe::narrow_oop_base() != NULL) &&

                 Universe::narrow_oop_use_implicit_null_checks()) ?

                  lcm(os::vm_page_size(), alignment) : 0) {

  if (base() > 0) {

    MemTracker::record_virtual_memory_type((address)base(), mtJavaHeap);

  }

  // Only reserved space for the java heap should have a noaccess_prefix

  // if using compressed oops.

  protect_noaccess_prefix(size);

}

ReservedHeapSpace::ReservedHeapSpace(const size_t prefix_size,

                                     const size_t prefix_align,

                                     const size_t suffix_size,

                                     const size_t suffix_align,