/*

 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "aot/aotLoader.hpp"

#include "code/codeBlob.hpp"

#include "code/codeCache.hpp"

#include "code/codeHeapState.hpp"

#include "code/compiledIC.hpp"

#include "code/dependencies.hpp"

#include "code/dependencyContext.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "compiler/compilationPolicy.hpp"

#include "compiler/compileBroker.hpp"

#include "jfr/jfrEvents.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/iterator.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "oops/method.inline.hpp"

#include "oops/objArrayOop.hpp"

#include "oops/oop.inline.hpp"

#include "oops/verifyOopClosure.hpp"

#include "runtime/arguments.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/icache.hpp"

#include "runtime/java.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/safepointVerifiers.hpp"

#include "runtime/sweeper.hpp"

#include "runtime/vmThread.hpp"

#include "services/memoryService.hpp"

#include "utilities/align.hpp"

#include "utilities/vmError.hpp"

#include "utilities/xmlstream.hpp"

#ifdef COMPILER1

#include "c1/c1_Compilation.hpp"

#include "c1/c1_Compiler.hpp"

#endif

#ifdef COMPILER2

#include "opto/c2compiler.hpp"

#include "opto/compile.hpp"

#include "opto/node.hpp"

#endif

// Helper class for printing in CodeCache

class CodeBlob_sizes {

 private:

  int count;

  int total_size;

  int header_size;

  int code_size;

  int stub_size;

  int relocation_size;

  int scopes_oop_size;

  int scopes_metadata_size;

  int scopes_data_size;

  int scopes_pcs_size;

 public:

  CodeBlob_sizes() {

    count            = 0;

    total_size       = 0;

    header_size      = 0;

    code_size        = 0;

    stub_size        = 0;

    relocation_size  = 0;

    scopes_oop_size  = 0;

    scopes_metadata_size  = 0;

    scopes_data_size = 0;

    scopes_pcs_size  = 0;

  }

  int total()                                    { return total_size; }

  bool is_empty()                                { return count == 0; }

  void print(const char* title) {

    tty->print_cr(" #%d %s = %dK (hdr %d%%,  loc %d%%, code %d%%, stub %d%%, [oops %d%%, metadata %d%%, data %d%%, pcs %d%%])",

                  count,

                  title,

                  (int)(total() / K),

                  header_size             * 100 / total_size,

                  relocation_size         * 100 / total_size,

                  code_size               * 100 / total_size,

                  stub_size               * 100 / total_size,

                  scopes_oop_size         * 100 / total_size,

                  scopes_metadata_size    * 100 / total_size,

                  scopes_data_size        * 100 / total_size,

                  scopes_pcs_size         * 100 / total_size);

  }

  void add(CodeBlob* cb) {

    count++;

    total_size       += cb->size();

    header_size      += cb->header_size();

    relocation_size  += cb->relocation_size();

    if (cb->is_nmethod()) {

      nmethod* nm = cb->as_nmethod_or_null();

      code_size        += nm->insts_size();

      stub_size        += nm->stub_size();

      scopes_oop_size  += nm->oops_size();

      scopes_metadata_size  += nm->metadata_size();

      scopes_data_size += nm->scopes_data_size();

      scopes_pcs_size  += nm->scopes_pcs_size();

    } else {

      code_size        += cb->code_size();

    }

  }

};

// Iterate over all CodeHeaps

#define FOR_ALL_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _heaps->begin(); heap != _heaps->end(); ++heap)

#define FOR_ALL_NMETHOD_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _nmethod_heaps->begin(); heap != _nmethod_heaps->end(); ++heap)

#define FOR_ALL_ALLOCABLE_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _allocable_heaps->begin(); heap != _allocable_heaps->end(); ++heap)

// Iterate over all CodeBlobs (cb) on the given CodeHeap

#define FOR_ALL_BLOBS(cb, heap) for (CodeBlob* cb = first_blob(heap); cb != NULL; cb = next_blob(heap, cb))

address CodeCache::_low_bound = 0;

address CodeCache::_high_bound = 0;

int CodeCache::_number_of_nmethods_with_dependencies = 0;

ExceptionCache* volatile CodeCache::_exception_cache_purge_list = NULL;

// Initialize arrays of CodeHeap subsets

GrowableArray<CodeHeap*>* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);

GrowableArray<CodeHeap*>* CodeCache::_compiled_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);

GrowableArray<CodeHeap*>* CodeCache::_nmethod_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);

GrowableArray<CodeHeap*>* CodeCache::_allocable_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);

void CodeCache::check_heap_sizes(size_t non_nmethod_size, size_t profiled_size, size_t non_profiled_size, size_t cache_size, bool all_set) {

  size_t total_size = non_nmethod_size + profiled_size + non_profiled_size;

  // Prepare error message

  const char* error = "Invalid code heap sizes";

  err_msg message("NonNMethodCodeHeapSize (" SIZE_FORMAT "K) + ProfiledCodeHeapSize (" SIZE_FORMAT "K)"

                  " + NonProfiledCodeHeapSize (" SIZE_FORMAT "K) = " SIZE_FORMAT "K",

          non_nmethod_size/K, profiled_size/K, non_profiled_size/K, total_size/K);

  if (total_size > cache_size) {

    // Some code heap sizes were explicitly set: total_size must be <= cache_size

    message.append(" is greater than ReservedCodeCacheSize (" SIZE_FORMAT "K).", cache_size/K);

    vm_exit_during_initialization(error, message);

  } else if (all_set && total_size != cache_size) {

    // All code heap sizes were explicitly set: total_size must equal cache_size

    message.append(" is not equal to ReservedCodeCacheSize (" SIZE_FORMAT "K).", cache_size/K);

    vm_exit_during_initialization(error, message);

  }

}

void CodeCache::initialize_heaps() {

  bool non_nmethod_set      = FLAG_IS_CMDLINE(NonNMethodCodeHeapSize);

  bool profiled_set         = FLAG_IS_CMDLINE(ProfiledCodeHeapSize);

  bool non_profiled_set     = FLAG_IS_CMDLINE(NonProfiledCodeHeapSize);

  size_t min_size           = os::vm_page_size();

  size_t cache_size         = ReservedCodeCacheSize;

  size_t non_nmethod_size   = NonNMethodCodeHeapSize;

  size_t profiled_size      = ProfiledCodeHeapSize;

  size_t non_profiled_size  = NonProfiledCodeHeapSize;

  // Check if total size set via command line flags exceeds the reserved size

  check_heap_sizes((non_nmethod_set  ? non_nmethod_size  : min_size),

                   (profiled_set     ? profiled_size     : min_size),

                   (non_profiled_set ? non_profiled_size : min_size),

                   cache_size,

                   non_nmethod_set && profiled_set && non_profiled_set);

  // Determine size of compiler buffers

  size_t code_buffers_size = 0;

#ifdef COMPILER1

  // C1 temporary code buffers (see Compiler::init_buffer_blob())

  const int c1_count = CompilationPolicy::policy()->compiler_count(CompLevel_simple);

  code_buffers_size += c1_count * Compiler::code_buffer_size();

#endif

#ifdef COMPILER2

  // C2 scratch buffers (see Compile::init_scratch_buffer_blob())

  const int c2_count = CompilationPolicy::policy()->compiler_count(CompLevel_full_optimization);

  // Initial size of constant table (this may be increased if a compiled method needs more space)

  code_buffers_size += c2_count * C2Compiler::initial_code_buffer_size();

#endif

  // Increase default non_nmethod_size to account for compiler buffers

  if (!non_nmethod_set) {

    non_nmethod_size += code_buffers_size;

  }

  // Calculate default CodeHeap sizes if not set by user

  if (!non_nmethod_set && !profiled_set && !non_profiled_set) {

    // Check if we have enough space for the non-nmethod code heap

    if (cache_size > non_nmethod_size) {

      // Use the default value for non_nmethod_size and one half of the

      // remaining size for non-profiled and one half for profiled methods

      size_t remaining_size = cache_size - non_nmethod_size;

      profiled_size = remaining_size / 2;

      non_profiled_size = remaining_size - profiled_size;

    } else {

      // Use all space for the non-nmethod heap and set other heaps to minimal size

      non_nmethod_size = cache_size - 2 * min_size;

      profiled_size = min_size;

      non_profiled_size = min_size;

    }

  } else if (!non_nmethod_set || !profiled_set || !non_profiled_set) {

    // The user explicitly set some code heap sizes. Increase or decrease the (default)

    // sizes of the other code heaps accordingly. First adapt non-profiled and profiled

    // code heap sizes and then only change non-nmethod code heap size if still necessary.

    intx diff_size = cache_size - (non_nmethod_size + profiled_size + non_profiled_size);

    if (non_profiled_set) {

      if (!profiled_set) {

        // Adapt size of profiled code heap

        if (diff_size < 0 && ((intx)profiled_size + diff_size) <= 0) {

          // Not enough space available, set to minimum size

          diff_size += profiled_size - min_size;

          profiled_size = min_size;

        } else {

          profiled_size += diff_size;

          diff_size = 0;

        }

      }

    } else if (profiled_set) {

      // Adapt size of non-profiled code heap

      if (diff_size < 0 && ((intx)non_profiled_size + diff_size) <= 0) {

        // Not enough space available, set to minimum size

        diff_size += non_profiled_size - min_size;

        non_profiled_size = min_size;

      } else {

        non_profiled_size += diff_size;

        diff_size = 0;

      }

    } else if (non_nmethod_set) {

      // Distribute remaining size between profiled and non-profiled code heaps

      diff_size = cache_size - non_nmethod_size;

      profiled_size = diff_size / 2;

      non_profiled_size = diff_size - profiled_size;

      diff_size = 0;

    }

    if (diff_size != 0) {

      // Use non-nmethod code heap for remaining space requirements

      assert(!non_nmethod_set && ((intx)non_nmethod_size + diff_size) > 0, "sanity");

      non_nmethod_size += diff_size;

    }

  }

  // We do not need the profiled CodeHeap, use all space for the non-profiled CodeHeap

  if (!heap_available(CodeBlobType::MethodProfiled)) {

    non_profiled_size += profiled_size;

    profiled_size = 0;

  }

  // We do not need the non-profiled CodeHeap, use all space for the non-nmethod CodeHeap

  if (!heap_available(CodeBlobType::MethodNonProfiled)) {

    non_nmethod_size += non_profiled_size;

    non_profiled_size = 0;

  }

  // Make sure we have enough space for VM internal code

  uint min_code_cache_size = CodeCacheMinimumUseSpace DEBUG_ONLY(* 3);

  if (non_nmethod_size < min_code_cache_size) {

    vm_exit_during_initialization(err_msg(

        "Not enough space in non-nmethod code heap to run VM: " SIZE_FORMAT "K < " SIZE_FORMAT "K",

        non_nmethod_size/K, min_code_cache_size/K));

  }

  // Verify sizes and update flag values

  assert(non_profiled_size + profiled_size + non_nmethod_size == cache_size, "Invalid code heap sizes");

  FLAG_SET_ERGO(NonNMethodCodeHeapSize, non_nmethod_size);

  FLAG_SET_ERGO(ProfiledCodeHeapSize, profiled_size);

  FLAG_SET_ERGO(NonProfiledCodeHeapSize, non_profiled_size);

  // If large page support is enabled, align code heaps according to large

  // page size to make sure that code cache is covered by large pages.

  const size_t alignment = MAX2(page_size(false, 8), (size_t) os::vm_allocation_granularity());

  non_nmethod_size = align_up(non_nmethod_size, alignment);

  profiled_size    = align_down(profiled_size, alignment);

  // Reserve one continuous chunk of memory for CodeHeaps and split it into

  // parts for the individual heaps. The memory layout looks like this:

  // ---------- high -----------

  //    Non-profiled nmethods

  //      Profiled nmethods

  //         Non-nmethods

  // ---------- low ------------

  ReservedCodeSpace rs = reserve_heap_memory(cache_size);

  ReservedSpace non_method_space    = rs.first_part(non_nmethod_size);

  ReservedSpace rest                = rs.last_part(non_nmethod_size);

  ReservedSpace profiled_space      = rest.first_part(profiled_size);

  ReservedSpace non_profiled_space  = rest.last_part(profiled_size);

  // Non-nmethods (stubs, adapters, ...)

  add_heap(non_method_space, "CodeHeap 'non-nmethods'", CodeBlobType::NonNMethod);

  // Tier 2 and tier 3 (profiled) methods

  add_heap(profiled_space, "CodeHeap 'profiled nmethods'", CodeBlobType::MethodProfiled);

  // Tier 1 and tier 4 (non-profiled) methods and native methods

  add_heap(non_profiled_space, "CodeHeap 'non-profiled nmethods'", CodeBlobType::MethodNonProfiled);

}

size_t CodeCache::page_size(bool aligned, size_t min_pages) {

  if (os::can_execute_large_page_memory()) {

    if (InitialCodeCacheSize < ReservedCodeCacheSize) {

      // Make sure that the page size allows for an incremental commit of the reserved space

      min_pages = MAX2(min_pages, (size_t)8);

    }

    return aligned ? os::page_size_for_region_aligned(ReservedCodeCacheSize, min_pages) :

                     os::page_size_for_region_unaligned(ReservedCodeCacheSize, min_pages);

  } else {

    return os::vm_page_size();

  }

}

ReservedCodeSpace CodeCache::reserve_heap_memory(size_t size) {

  // Align and reserve space for code cache

  const size_t rs_ps = page_size();

  const size_t rs_align = MAX2(rs_ps, (size_t) os::vm_allocation_granularity());

  const size_t rs_size = align_up(size, rs_align);

  ReservedCodeSpace rs(rs_size, rs_align, rs_ps > (size_t) os::vm_page_size());

  if (!rs.is_reserved()) {

    vm_exit_during_initialization(err_msg("Could not reserve enough space for code cache (" SIZE_FORMAT "K)",

                                          rs_size/K));

  }

  // Initialize bounds

  _low_bound = (address)rs.base();

  _high_bound = _low_bound + rs.size();

  return rs;

}

// Heaps available for allocation

bool CodeCache::heap_available(int code_blob_type) {

  if (!SegmentedCodeCache) {

    // No segmentation: use a single code heap

    return (code_blob_type == CodeBlobType::All);

  } else if (Arguments::is_interpreter_only()) {

    // Interpreter only: we don't need any method code heaps

    return (code_blob_type == CodeBlobType::NonNMethod);

  } else if (TieredCompilation && (TieredStopAtLevel > CompLevel_simple)) {

    // Tiered compilation: use all code heaps

    return (code_blob_type < CodeBlobType::All);

  } else {

    // No TieredCompilation: we only need the non-nmethod and non-profiled code heap

    return (code_blob_type == CodeBlobType::NonNMethod) ||

           (code_blob_type == CodeBlobType::MethodNonProfiled);

  }

}

const char* CodeCache::get_code_heap_flag_name(int code_blob_type) {

  switch(code_blob_type) {

  case CodeBlobType::NonNMethod:

    return "NonNMethodCodeHeapSize";

    break;

  case CodeBlobType::MethodNonProfiled:

    return "NonProfiledCodeHeapSize";

    break;

  case CodeBlobType::MethodProfiled:

    return "ProfiledCodeHeapSize";

    break;

  }

  ShouldNotReachHere();

  return NULL;

}

int CodeCache::code_heap_compare(CodeHeap* const &lhs, CodeHeap* const &rhs) {

  if (lhs->code_blob_type() == rhs->code_blob_type()) {

    return (lhs > rhs) ? 1 : ((lhs < rhs) ? -1 : 0);

  } else {

    return lhs->code_blob_type() - rhs->code_blob_type();

  }

}

void CodeCache::add_heap(CodeHeap* heap) {

  assert(!Universe::is_fully_initialized(), "late heap addition?");

  _heaps->insert_sorted<code_heap_compare>(heap);

  int type = heap->code_blob_type();

  if (code_blob_type_accepts_compiled(type)) {

    _compiled_heaps->insert_sorted<code_heap_compare>(heap);

  }

  if (code_blob_type_accepts_nmethod(type)) {

    _nmethod_heaps->insert_sorted<code_heap_compare>(heap);

  }

  if (code_blob_type_accepts_allocable(type)) {

    _allocable_heaps->insert_sorted<code_heap_compare>(heap);

  }

}

void CodeCache::add_heap(ReservedSpace rs, const char* name, int code_blob_type) {

  // Check if heap is needed

  if (!heap_available(code_blob_type)) {

    return;

  }

  // Create CodeHeap

  CodeHeap* heap = new CodeHeap(name, code_blob_type);

  add_heap(heap);

  // Reserve Space

  size_t size_initial = MIN2((size_t)InitialCodeCacheSize, rs.size());

  size_initial = align_up(size_initial, os::vm_page_size());

  if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) {

    vm_exit_during_initialization(err_msg("Could not reserve enough space in %s (" SIZE_FORMAT "K)",

                                          heap->name(), size_initial/K));

  }

  // Register the CodeHeap

  MemoryService::add_code_heap_memory_pool(heap, name);

}

CodeHeap* CodeCache::get_code_heap_containing(void* start) {

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->contains(start)) {

      return *heap;

    }

  }

  return NULL;

}

CodeHeap* CodeCache::get_code_heap(const CodeBlob* cb) {

  assert(cb != NULL, "CodeBlob is null");

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->contains_blob(cb)) {

      return *heap;

    }

  }

  ShouldNotReachHere();

  return NULL;

}

CodeHeap* CodeCache::get_code_heap(int code_blob_type) {

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->accepts(code_blob_type)) {

      return *heap;

    }

  }

  return NULL;

}

CodeBlob* CodeCache::first_blob(CodeHeap* heap) {

  assert_locked_or_safepoint(CodeCache_lock);

  assert(heap != NULL, "heap is null");

  return (CodeBlob*)heap->first();

}

CodeBlob* CodeCache::first_blob(int code_blob_type) {

  if (heap_available(code_blob_type)) {

    return first_blob(get_code_heap(code_blob_type));

  } else {

    return NULL;

  }

}

CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) {

  assert_locked_or_safepoint(CodeCache_lock);

  assert(heap != NULL, "heap is null");

  return (CodeBlob*)heap->next(cb);

}

/**

 * Do not seize the CodeCache lock here--if the caller has not

 * already done so, we are going to lose bigtime, since the code

 * cache will contain a garbage CodeBlob until the caller can

 * run the constructor for the CodeBlob subclass he is busy

 * instantiating.

 */

CodeBlob* CodeCache::allocate(int size, int code_blob_type, int orig_code_blob_type) {

  // Possibly wakes up the sweeper thread.

  NMethodSweeper::notify(code_blob_type);

  assert_locked_or_safepoint(CodeCache_lock);

  assert(size > 0, "Code cache allocation request must be > 0 but is %d", size);

  if (size <= 0) {