/*

 * Copyright (c) 2011, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "aot/aotLoader.hpp"

#include "classfile/classLoaderDataGraph.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "memory/filemap.hpp"

#include "memory/metaspace.hpp"

#include "memory/metaspace/chunkManager.hpp"

#include "memory/metaspace/metachunk.hpp"

#include "memory/metaspace/metaspaceCommon.hpp"

#include "memory/metaspace/printCLDMetaspaceInfoClosure.hpp"

#include "memory/metaspace/spaceManager.hpp"

#include "memory/metaspace/virtualSpaceList.hpp"

#include "memory/metaspaceShared.hpp"

#include "memory/metaspaceTracer.hpp"

#include "memory/universe.hpp"

#include "oops/compressedOops.hpp"

#include "runtime/init.hpp"

#include "runtime/orderAccess.hpp"

#include "services/memTracker.hpp"

#include "utilities/copy.hpp"

#include "utilities/debug.hpp"

#include "utilities/formatBuffer.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/vmError.hpp"

using namespace metaspace;

MetaWord* last_allocated = 0;

size_t Metaspace::_compressed_class_space_size;

const MetaspaceTracer* Metaspace::_tracer = NULL;

DEBUG_ONLY(bool Metaspace::_frozen = false;)

static const char* space_type_name(Metaspace::MetaspaceType t) {

  const char* s = NULL;

  switch (t) {

    case Metaspace::StandardMetaspaceType: s = "Standard"; break;

    case Metaspace::BootMetaspaceType: s = "Boot"; break;

    case Metaspace::UnsafeAnonymousMetaspaceType: s = "UnsafeAnonymous"; break;

    case Metaspace::ReflectionMetaspaceType: s = "Reflection"; break;

    default: ShouldNotReachHere();

  }

  return s;

}

volatile size_t MetaspaceGC::_capacity_until_GC = 0;

uint MetaspaceGC::_shrink_factor = 0;

// BlockFreelist methods

// VirtualSpaceNode methods

// MetaspaceGC methods

// VM_CollectForMetadataAllocation is the vm operation used to GC.

// Within the VM operation after the GC the attempt to allocate the metadata

// should succeed.  If the GC did not free enough space for the metaspace

// allocation, the HWM is increased so that another virtualspace will be

// allocated for the metadata.  With perm gen the increase in the perm

// gen had bounds, MinMetaspaceExpansion and MaxMetaspaceExpansion.  The

// metaspace policy uses those as the small and large steps for the HWM.

//

// After the GC the compute_new_size() for MetaspaceGC is called to

// resize the capacity of the metaspaces.  The current implementation

// is based on the flags MinMetaspaceFreeRatio and MaxMetaspaceFreeRatio used

// to resize the Java heap by some GC's.  New flags can be implemented

// if really needed.  MinMetaspaceFreeRatio is used to calculate how much

// free space is desirable in the metaspace capacity to decide how much

// to increase the HWM.  MaxMetaspaceFreeRatio is used to decide how much

// free space is desirable in the metaspace capacity before decreasing

// the HWM.

// Calculate the amount to increase the high water mark (HWM).

// Increase by a minimum amount (MinMetaspaceExpansion) so that

// another expansion is not requested too soon.  If that is not

// enough to satisfy the allocation, increase by MaxMetaspaceExpansion.

// If that is still not enough, expand by the size of the allocation

// plus some.

size_t MetaspaceGC::delta_capacity_until_GC(size_t bytes) {

  size_t min_delta = MinMetaspaceExpansion;

  size_t max_delta = MaxMetaspaceExpansion;

  size_t delta = align_up(bytes, Metaspace::commit_alignment());

  if (delta <= min_delta) {

    delta = min_delta;

  } else if (delta <= max_delta) {

    // Don't want to hit the high water mark on the next

    // allocation so make the delta greater than just enough

    // for this allocation.

    delta = max_delta;

  } else {

    // This allocation is large but the next ones are probably not

    // so increase by the minimum.

    delta = delta + min_delta;

  }

  assert_is_aligned(delta, Metaspace::commit_alignment());

  return delta;

}

size_t MetaspaceGC::capacity_until_GC() {

  size_t value = Atomic::load_acquire(&_capacity_until_GC);

  assert(value >= MetaspaceSize, "Not initialized properly?");

  return value;

}

// Try to increase the _capacity_until_GC limit counter by v bytes.

// Returns true if it succeeded. It may fail if either another thread

// concurrently increased the limit or the new limit would be larger

// than MaxMetaspaceSize.

// On success, optionally returns new and old metaspace capacity in

// new_cap_until_GC and old_cap_until_GC respectively.

// On error, optionally sets can_retry to indicate whether if there is

// actually enough space remaining to satisfy the request.

bool MetaspaceGC::inc_capacity_until_GC(size_t v, size_t* new_cap_until_GC, size_t* old_cap_until_GC, bool* can_retry) {

  assert_is_aligned(v, Metaspace::commit_alignment());

  size_t old_capacity_until_GC = _capacity_until_GC;

  size_t new_value = old_capacity_until_GC + v;

  if (new_value < old_capacity_until_GC) {

    // The addition wrapped around, set new_value to aligned max value.

    new_value = align_down(max_uintx, Metaspace::commit_alignment());

  }

  if (new_value > MaxMetaspaceSize) {

    if (can_retry != NULL) {

      *can_retry = false;

    }

    return false;

  }

  if (can_retry != NULL) {

    *can_retry = true;

  }

  size_t prev_value = Atomic::cmpxchg(new_value, &_capacity_until_GC, old_capacity_until_GC);

  if (old_capacity_until_GC != prev_value) {

    return false;

  }

  if (new_cap_until_GC != NULL) {

    *new_cap_until_GC = new_value;

  }

  if (old_cap_until_GC != NULL) {

    *old_cap_until_GC = old_capacity_until_GC;

  }

  return true;

}

size_t MetaspaceGC::dec_capacity_until_GC(size_t v) {

  assert_is_aligned(v, Metaspace::commit_alignment());

}

void MetaspaceGC::initialize() {

  // Set the high-water mark to MaxMetapaceSize during VM initializaton since

  // we can't do a GC during initialization.

  _capacity_until_GC = MaxMetaspaceSize;

}

void MetaspaceGC::post_initialize() {

  // Reset the high-water mark once the VM initialization is done.

  _capacity_until_GC = MAX2(MetaspaceUtils::committed_bytes(), MetaspaceSize);

}

bool MetaspaceGC::can_expand(size_t word_size, bool is_class) {

  // Check if the compressed class space is full.

  if (is_class && Metaspace::using_class_space()) {

    size_t class_committed = MetaspaceUtils::committed_bytes(Metaspace::ClassType);

    if (class_committed + word_size * BytesPerWord > CompressedClassSpaceSize) {

      log_trace(gc, metaspace, freelist)("Cannot expand %s metaspace by " SIZE_FORMAT " words (CompressedClassSpaceSize = " SIZE_FORMAT " words)",

                (is_class ? "class" : "non-class"), word_size, CompressedClassSpaceSize / sizeof(MetaWord));

      return false;

    }

  }

  // Check if the user has imposed a limit on the metaspace memory.

  size_t committed_bytes = MetaspaceUtils::committed_bytes();

  if (committed_bytes + word_size * BytesPerWord > MaxMetaspaceSize) {

    log_trace(gc, metaspace, freelist)("Cannot expand %s metaspace by " SIZE_FORMAT " words (MaxMetaspaceSize = " SIZE_FORMAT " words)",

              (is_class ? "class" : "non-class"), word_size, MaxMetaspaceSize / sizeof(MetaWord));

    return false;

  }

  return true;

}

size_t MetaspaceGC::allowed_expansion() {

  size_t committed_bytes = MetaspaceUtils::committed_bytes();

  size_t capacity_until_gc = capacity_until_GC();

  assert(capacity_until_gc >= committed_bytes,

         "capacity_until_gc: " SIZE_FORMAT " < committed_bytes: " SIZE_FORMAT,

         capacity_until_gc, committed_bytes);

  size_t left_until_max  = MaxMetaspaceSize - committed_bytes;

  size_t left_until_GC = capacity_until_gc - committed_bytes;

  size_t left_to_commit = MIN2(left_until_GC, left_until_max);

  log_trace(gc, metaspace, freelist)("allowed expansion words: " SIZE_FORMAT

            " (left_until_max: " SIZE_FORMAT ", left_until_GC: " SIZE_FORMAT ".",

            left_to_commit / BytesPerWord, left_until_max / BytesPerWord, left_until_GC / BytesPerWord);

  return left_to_commit / BytesPerWord;

}

void MetaspaceGC::compute_new_size() {

  assert(_shrink_factor <= 100, "invalid shrink factor");

  uint current_shrink_factor = _shrink_factor;

  _shrink_factor = 0;

  // Using committed_bytes() for used_after_gc is an overestimation, since the

  // chunk free lists are included in committed_bytes() and the memory in an

  // un-fragmented chunk free list is available for future allocations.

  // However, if the chunk free lists becomes fragmented, then the memory may

  // not be available for future allocations and the memory is therefore "in use".

  // Including the chunk free lists in the definition of "in use" is therefore

  // necessary. Not including the chunk free lists can cause capacity_until_GC to

  // shrink below committed_bytes() and this has caused serious bugs in the past.

  const size_t used_after_gc = MetaspaceUtils::committed_bytes();

  const size_t capacity_until_GC = MetaspaceGC::capacity_until_GC();

  const double minimum_free_percentage = MinMetaspaceFreeRatio / 100.0;

  const double maximum_used_percentage = 1.0 - minimum_free_percentage;

  const double min_tmp = used_after_gc / maximum_used_percentage;

  size_t minimum_desired_capacity =

    (size_t)MIN2(min_tmp, double(MaxMetaspaceSize));

  // Don't shrink less than the initial generation size

  minimum_desired_capacity = MAX2(minimum_desired_capacity,

                                  MetaspaceSize);

  log_trace(gc, metaspace)("MetaspaceGC::compute_new_size: ");

  log_trace(gc, metaspace)("    minimum_free_percentage: %6.2f  maximum_used_percentage: %6.2f",

                           minimum_free_percentage, maximum_used_percentage);

  log_trace(gc, metaspace)("     used_after_gc       : %6.1fKB", used_after_gc / (double) K);

  size_t shrink_bytes = 0;

  if (capacity_until_GC < minimum_desired_capacity) {

    // If we have less capacity below the metaspace HWM, then

    // increment the HWM.

    size_t expand_bytes = minimum_desired_capacity - capacity_until_GC;

    expand_bytes = align_up(expand_bytes, Metaspace::commit_alignment());

    // Don't expand unless it's significant

    if (expand_bytes >= MinMetaspaceExpansion) {

      size_t new_capacity_until_GC = 0;

      bool succeeded = MetaspaceGC::inc_capacity_until_GC(expand_bytes, &new_capacity_until_GC);

      assert(succeeded, "Should always succesfully increment HWM when at safepoint");

      Metaspace::tracer()->report_gc_threshold(capacity_until_GC,

                                               new_capacity_until_GC,

                                               MetaspaceGCThresholdUpdater::ComputeNewSize);

      log_trace(gc, metaspace)("    expanding:  minimum_desired_capacity: %6.1fKB  expand_bytes: %6.1fKB  MinMetaspaceExpansion: %6.1fKB  new metaspace HWM:  %6.1fKB",

                               minimum_desired_capacity / (double) K,

                               expand_bytes / (double) K,

                               MinMetaspaceExpansion / (double) K,

                               new_capacity_until_GC / (double) K);

    }

    return;

  }

  // No expansion, now see if we want to shrink

  // We would never want to shrink more than this

  assert(capacity_until_GC >= minimum_desired_capacity,

         SIZE_FORMAT " >= " SIZE_FORMAT,

         capacity_until_GC, minimum_desired_capacity);

  size_t max_shrink_bytes = capacity_until_GC - minimum_desired_capacity;

  // Should shrinking be considered?

  if (MaxMetaspaceFreeRatio < 100) {

    const double maximum_free_percentage = MaxMetaspaceFreeRatio / 100.0;

    const double minimum_used_percentage = 1.0 - maximum_free_percentage;

    const double max_tmp = used_after_gc / minimum_used_percentage;

    size_t maximum_desired_capacity = (size_t)MIN2(max_tmp, double(MaxMetaspaceSize));

    maximum_desired_capacity = MAX2(maximum_desired_capacity,

                                    MetaspaceSize);

    log_trace(gc, metaspace)("    maximum_free_percentage: %6.2f  minimum_used_percentage: %6.2f",

                             maximum_free_percentage, minimum_used_percentage);

    log_trace(gc, metaspace)("    minimum_desired_capacity: %6.1fKB  maximum_desired_capacity: %6.1fKB",

                             minimum_desired_capacity / (double) K, maximum_desired_capacity / (double) K);

    assert(minimum_desired_capacity <= maximum_desired_capacity,

           "sanity check");

    if (capacity_until_GC > maximum_desired_capacity) {

      // Capacity too large, compute shrinking size

      shrink_bytes = capacity_until_GC - maximum_desired_capacity;

      // We don't want shrink all the way back to initSize if people call

      // System.gc(), because some programs do that between "phases" and then

      // we'd just have to grow the heap up again for the next phase.  So we

      // damp the shrinking: 0% on the first call, 10% on the second call, 40%

      // on the third call, and 100% by the fourth call.  But if we recompute

      // size without shrinking, it goes back to 0%.

      shrink_bytes = shrink_bytes / 100 * current_shrink_factor;

      shrink_bytes = align_down(shrink_bytes, Metaspace::commit_alignment());

      assert(shrink_bytes <= max_shrink_bytes,

             "invalid shrink size " SIZE_FORMAT " not <= " SIZE_FORMAT,

             shrink_bytes, max_shrink_bytes);

      if (current_shrink_factor == 0) {

        _shrink_factor = 10;

      } else {

        _shrink_factor = MIN2(current_shrink_factor * 4, (uint) 100);

      }

      log_trace(gc, metaspace)("    shrinking:  initThreshold: %.1fK  maximum_desired_capacity: %.1fK",

                               MetaspaceSize / (double) K, maximum_desired_capacity / (double) K);

      log_trace(gc, metaspace)("    shrink_bytes: %.1fK  current_shrink_factor: %d  new shrink factor: %d  MinMetaspaceExpansion: %.1fK",

                               shrink_bytes / (double) K, current_shrink_factor, _shrink_factor, MinMetaspaceExpansion / (double) K);

    }

  }

  // Don't shrink unless it's significant

  if (shrink_bytes >= MinMetaspaceExpansion &&

      ((capacity_until_GC - shrink_bytes) >= MetaspaceSize)) {

    size_t new_capacity_until_GC = MetaspaceGC::dec_capacity_until_GC(shrink_bytes);

    Metaspace::tracer()->report_gc_threshold(capacity_until_GC,

                                             new_capacity_until_GC,

                                             MetaspaceGCThresholdUpdater::ComputeNewSize);

  }

}

// MetaspaceUtils

size_t MetaspaceUtils::_capacity_words [Metaspace:: MetadataTypeCount] = {0, 0};

size_t MetaspaceUtils::_overhead_words [Metaspace:: MetadataTypeCount] = {0, 0};

volatile size_t MetaspaceUtils::_used_words [Metaspace:: MetadataTypeCount] = {0, 0};

// Collect used metaspace statistics. This involves walking the CLDG. The resulting

// output will be the accumulated values for all live metaspaces.

// Note: method does not do any locking.

void MetaspaceUtils::collect_statistics(ClassLoaderMetaspaceStatistics* out) {

  out->reset();

  ClassLoaderDataGraphMetaspaceIterator iter;

   while (iter.repeat()) {

     ClassLoaderMetaspace* msp = iter.get_next();

     if (msp != NULL) {

       msp->add_to_statistics(out);

     }

   }

}

size_t MetaspaceUtils::free_in_vs_bytes(Metaspace::MetadataType mdtype) {

  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);

  return list == NULL ? 0 : list->free_bytes();

}

size_t MetaspaceUtils::free_in_vs_bytes() {

  return free_in_vs_bytes(Metaspace::ClassType) + free_in_vs_bytes(Metaspace::NonClassType);

}

static void inc_stat_nonatomically(size_t* pstat, size_t words) {

  assert_lock_strong(MetaspaceExpand_lock);

  (*pstat) += words;

}

static void dec_stat_nonatomically(size_t* pstat, size_t words) {

  assert_lock_strong(MetaspaceExpand_lock);

  const size_t size_now = *pstat;

  assert(size_now >= words, "About to decrement counter below zero "

         "(current value: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ".",

         size_now, words);

  *pstat = size_now - words;

}

static void inc_stat_atomically(volatile size_t* pstat, size_t words) {

  Atomic::add(pstat, words);

}

static void dec_stat_atomically(volatile size_t* pstat, size_t words) {

  const size_t size_now = *pstat;

  assert(size_now >= words, "About to decrement counter below zero "

         "(current value: " SIZE_FORMAT ", decrement value: " SIZE_FORMAT ".",

         size_now, words);

}

void MetaspaceUtils::dec_capacity(Metaspace::MetadataType mdtype, size_t words) {

  dec_stat_nonatomically(&_capacity_words[mdtype], words);

}

void MetaspaceUtils::inc_capacity(Metaspace::MetadataType mdtype, size_t words) {

  inc_stat_nonatomically(&_capacity_words[mdtype], words);

}

void MetaspaceUtils::dec_used(Metaspace::MetadataType mdtype, size_t words) {

  dec_stat_atomically(&_used_words[mdtype], words);

}

void MetaspaceUtils::inc_used(Metaspace::MetadataType mdtype, size_t words) {

  inc_stat_atomically(&_used_words[mdtype], words);

}

void MetaspaceUtils::dec_overhead(Metaspace::MetadataType mdtype, size_t words) {

  dec_stat_nonatomically(&_overhead_words[mdtype], words);

}

void MetaspaceUtils::inc_overhead(Metaspace::MetadataType mdtype, size_t words) {

  inc_stat_nonatomically(&_overhead_words[mdtype], words);

}

size_t MetaspaceUtils::reserved_bytes(Metaspace::MetadataType mdtype) {

  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);

  return list == NULL ? 0 : list->reserved_bytes();

}

size_t MetaspaceUtils::committed_bytes(Metaspace::MetadataType mdtype) {

  VirtualSpaceList* list = Metaspace::get_space_list(mdtype);

  return list == NULL ? 0 : list->committed_bytes();

}

size_t MetaspaceUtils::min_chunk_size_words() { return Metaspace::first_chunk_word_size(); }

size_t MetaspaceUtils::free_chunks_total_words(Metaspace::MetadataType mdtype) {

  ChunkManager* chunk_manager = Metaspace::get_chunk_manager(mdtype);

  if (chunk_manager == NULL) {

    return 0;

  }

  return chunk_manager->free_chunks_total_words();

}

size_t MetaspaceUtils::free_chunks_total_bytes(Metaspace::MetadataType mdtype) {

  return free_chunks_total_words(mdtype) * BytesPerWord;

}

size_t MetaspaceUtils::free_chunks_total_words() {

  return free_chunks_total_words(Metaspace::ClassType) +

         free_chunks_total_words(Metaspace::NonClassType);

}

size_t MetaspaceUtils::free_chunks_total_bytes() {

  return free_chunks_total_words() * BytesPerWord;

}

bool MetaspaceUtils::has_chunk_free_list(Metaspace::MetadataType mdtype) {

  return Metaspace::get_chunk_manager(mdtype) != NULL;

}

MetaspaceChunkFreeListSummary MetaspaceUtils::chunk_free_list_summary(Metaspace::MetadataType mdtype) {

  if (!has_chunk_free_list(mdtype)) {

    return MetaspaceChunkFreeListSummary();

  }

  const ChunkManager* cm = Metaspace::get_chunk_manager(mdtype);

  return cm->chunk_free_list_summary();

}

void MetaspaceUtils::print_metaspace_change(const metaspace::MetaspaceSizesSnapshot& pre_meta_values) {

  const metaspace::MetaspaceSizesSnapshot meta_values;

  if (Metaspace::using_class_space()) {

    log_info(gc, metaspace)(HEAP_CHANGE_FORMAT" "

                            HEAP_CHANGE_FORMAT" "

                            HEAP_CHANGE_FORMAT,

                            HEAP_CHANGE_FORMAT_ARGS("Metaspace",

                                                    pre_meta_values.used(),

                                                    pre_meta_values.committed(),

                                                    meta_values.used(),

                                                    meta_values.committed()),

                            HEAP_CHANGE_FORMAT_ARGS("NonClass",

                                                    pre_meta_values.non_class_used(),

                                                    pre_meta_values.non_class_committed(),

                                                    meta_values.non_class_used(),

                                                    meta_values.non_class_committed()),

                            HEAP_CHANGE_FORMAT_ARGS("Class",

                                                    pre_meta_values.class_used(),

                                                    pre_meta_values.class_committed(),