/*

 * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "code/codeBlob.hpp"

#include "code/codeCache.hpp"

#include "code/compiledIC.hpp"

#include "code/dependencies.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "compiler/compileBroker.hpp"

#include "gc/shared/gcLocker.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/iterator.hpp"

#include "memory/resourceArea.hpp"

#include "oops/method.hpp"

#include "oops/objArrayOop.hpp"

#include "oops/oop.inline.hpp"

#include "oops/verifyOopClosure.hpp"

#include "runtime/arguments.hpp"

#include "runtime/compilationPolicy.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/icache.hpp"

#include "runtime/java.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/sweeper.hpp"

#include "services/memoryService.hpp"

#include "trace/tracing.hpp"

#include "utilities/xmlstream.hpp"

#ifdef COMPILER1

#include "c1/c1_Compilation.hpp"

#include "c1/c1_Compiler.hpp"

#endif

#ifdef COMPILER2

#include "opto/c2compiler.hpp"

#include "opto/compile.hpp"

#include "opto/node.hpp"

#endif

// Helper class for printing in CodeCache

class CodeBlob_sizes {

 private:

  int count;

  int total_size;

  int header_size;

  int code_size;

  int stub_size;

  int relocation_size;

  int scopes_oop_size;

  int scopes_metadata_size;

  int scopes_data_size;

  int scopes_pcs_size;

 public:

  CodeBlob_sizes() {

    count            = 0;

    total_size       = 0;

    header_size      = 0;

    code_size        = 0;

    stub_size        = 0;

    relocation_size  = 0;

    scopes_oop_size  = 0;

    scopes_metadata_size  = 0;

    scopes_data_size = 0;

    scopes_pcs_size  = 0;

  }

  int total()                                    { return total_size; }

  bool is_empty()                                { return count == 0; }

  void print(const char* title) {

    tty->print_cr(" #%d %s = %dK (hdr %d%%,  loc %d%%, code %d%%, stub %d%%, [oops %d%%, metadata %d%%, data %d%%, pcs %d%%])",

                  count,

                  title,

                  (int)(total() / K),

                  header_size             * 100 / total_size,

                  relocation_size         * 100 / total_size,

                  code_size               * 100 / total_size,

                  stub_size               * 100 / total_size,

                  scopes_oop_size         * 100 / total_size,

                  scopes_metadata_size    * 100 / total_size,

                  scopes_data_size        * 100 / total_size,

                  scopes_pcs_size         * 100 / total_size);

  }

  void add(CodeBlob* cb) {

    count++;

    total_size       += cb->size();

    header_size      += cb->header_size();

    relocation_size  += cb->relocation_size();

    if (cb->is_nmethod()) {

      nmethod* nm = cb->as_nmethod_or_null();

      code_size        += nm->insts_size();

      stub_size        += nm->stub_size();

      scopes_oop_size  += nm->oops_size();

      scopes_metadata_size  += nm->metadata_size();

      scopes_data_size += nm->scopes_data_size();

      scopes_pcs_size  += nm->scopes_pcs_size();

    } else {

      code_size        += cb->code_size();

    }

  }

};

// Iterate over all CodeHeaps

#define FOR_ALL_HEAPS(heap) for (GrowableArrayIterator<CodeHeap*> heap = _heaps->begin(); heap != _heaps->end(); ++heap)

// Iterate over all CodeBlobs (cb) on the given CodeHeap

#define FOR_ALL_BLOBS(cb, heap) for (CodeBlob* cb = first_blob(heap); cb != NULL; cb = next_blob(heap, cb))

address CodeCache::_low_bound = 0;

address CodeCache::_high_bound = 0;

int CodeCache::_number_of_blobs = 0;

int CodeCache::_number_of_adapters = 0;

int CodeCache::_number_of_nmethods = 0;

int CodeCache::_number_of_nmethods_with_dependencies = 0;

bool CodeCache::_needs_cache_clean = false;

nmethod* CodeCache::_scavenge_root_nmethods = NULL;

int CodeCache::_codemem_full_count = 0;

// Initialize array of CodeHeaps

GrowableArray<CodeHeap*>* CodeCache::_heaps = new(ResourceObj::C_HEAP, mtCode) GrowableArray<CodeHeap*> (CodeBlobType::All, true);

void CodeCache::initialize_heaps() {

  // Determine size of compiler buffers

  size_t code_buffers_size = 0;

#ifdef COMPILER1

  // C1 temporary code buffers (see Compiler::init_buffer_blob())

  const int c1_count = CompilationPolicy::policy()->compiler_count(CompLevel_simple);

  code_buffers_size += c1_count * Compiler::code_buffer_size();

#endif

#ifdef COMPILER2

  // C2 scratch buffers (see Compile::init_scratch_buffer_blob())

  const int c2_count = CompilationPolicy::policy()->compiler_count(CompLevel_full_optimization);

  // Initial size of constant table (this may be increased if a compiled method needs more space)

  code_buffers_size += c2_count * C2Compiler::initial_code_buffer_size();

#endif

  // Calculate default CodeHeap sizes if not set by user

  if (!FLAG_IS_CMDLINE(NonNMethodCodeHeapSize) && !FLAG_IS_CMDLINE(ProfiledCodeHeapSize)

      && !FLAG_IS_CMDLINE(NonProfiledCodeHeapSize)) {

    // Increase default NonNMethodCodeHeapSize to account for compiler buffers

    FLAG_SET_ERGO(uintx, NonNMethodCodeHeapSize, NonNMethodCodeHeapSize + code_buffers_size);

    // Check if we have enough space for the non-nmethod code heap

    if (ReservedCodeCacheSize > NonNMethodCodeHeapSize) {

      // Use the default value for NonNMethodCodeHeapSize and one half of the

      // remaining size for non-profiled methods and one half for profiled methods

      size_t remaining_size = ReservedCodeCacheSize - NonNMethodCodeHeapSize;

      size_t profiled_size = remaining_size / 2;

      size_t non_profiled_size = remaining_size - profiled_size;

      FLAG_SET_ERGO(uintx, ProfiledCodeHeapSize, profiled_size);

      FLAG_SET_ERGO(uintx, NonProfiledCodeHeapSize, non_profiled_size);

    } else {

      // Use all space for the non-nmethod heap and set other heaps to minimal size

      FLAG_SET_ERGO(uintx, NonNMethodCodeHeapSize, ReservedCodeCacheSize - os::vm_page_size() * 2);

      FLAG_SET_ERGO(uintx, ProfiledCodeHeapSize, os::vm_page_size());

      FLAG_SET_ERGO(uintx, NonProfiledCodeHeapSize, os::vm_page_size());

    }

  }

  // We do not need the profiled CodeHeap, use all space for the non-profiled CodeHeap

  if(!heap_available(CodeBlobType::MethodProfiled)) {

    FLAG_SET_ERGO(uintx, NonProfiledCodeHeapSize, NonProfiledCodeHeapSize + ProfiledCodeHeapSize);

    FLAG_SET_ERGO(uintx, ProfiledCodeHeapSize, 0);

  }

  // We do not need the non-profiled CodeHeap, use all space for the non-nmethod CodeHeap

  if(!heap_available(CodeBlobType::MethodNonProfiled)) {

    FLAG_SET_ERGO(uintx, NonNMethodCodeHeapSize, NonNMethodCodeHeapSize + NonProfiledCodeHeapSize);

    FLAG_SET_ERGO(uintx, NonProfiledCodeHeapSize, 0);

  }

  // Make sure we have enough space for VM internal code

  uint min_code_cache_size = CodeCacheMinimumUseSpace DEBUG_ONLY(* 3);

  if (NonNMethodCodeHeapSize < (min_code_cache_size + code_buffers_size)) {

    vm_exit_during_initialization("Not enough space in non-nmethod code heap to run VM.");

  }

  guarantee(NonProfiledCodeHeapSize + ProfiledCodeHeapSize + NonNMethodCodeHeapSize <= ReservedCodeCacheSize, "Size check");

  // Align CodeHeaps

  size_t alignment = heap_alignment();

  size_t non_method_size = align_size_up(NonNMethodCodeHeapSize, alignment);

  size_t profiled_size   = align_size_down(ProfiledCodeHeapSize, alignment);

  // Reserve one continuous chunk of memory for CodeHeaps and split it into

  // parts for the individual heaps. The memory layout looks like this:

  // ---------- high -----------

  //    Non-profiled nmethods

  //      Profiled nmethods

  //         Non-nmethods

  // ---------- low ------------

  ReservedCodeSpace rs = reserve_heap_memory(ReservedCodeCacheSize);

  ReservedSpace non_method_space    = rs.first_part(non_method_size);

  ReservedSpace rest                = rs.last_part(non_method_size);

  ReservedSpace profiled_space      = rest.first_part(profiled_size);

  ReservedSpace non_profiled_space  = rest.last_part(profiled_size);

  // Non-nmethods (stubs, adapters, ...)

  add_heap(non_method_space, "CodeHeap 'non-nmethods'", CodeBlobType::NonNMethod);

  // Tier 2 and tier 3 (profiled) methods

  add_heap(profiled_space, "CodeHeap 'profiled nmethods'", CodeBlobType::MethodProfiled);

  // Tier 1 and tier 4 (non-profiled) methods and native methods

  add_heap(non_profiled_space, "CodeHeap 'non-profiled nmethods'", CodeBlobType::MethodNonProfiled);

}

size_t CodeCache::heap_alignment() {

  // If large page support is enabled, align code heaps according to large

  // page size to make sure that code cache is covered by large pages.

  const size_t page_size = os::can_execute_large_page_memory() ?

             os::page_size_for_region_unaligned(ReservedCodeCacheSize, 8) :

             os::vm_page_size();

  return MAX2(page_size, (size_t) os::vm_allocation_granularity());

}

ReservedCodeSpace CodeCache::reserve_heap_memory(size_t size) {

  // Determine alignment

  const size_t page_size = os::can_execute_large_page_memory() ?

          MIN2(os::page_size_for_region_aligned(InitialCodeCacheSize, 8),

               os::page_size_for_region_aligned(size, 8)) :

          os::vm_page_size();

  const size_t granularity = os::vm_allocation_granularity();

  const size_t r_align = MAX2(page_size, granularity);

  const size_t r_size = align_size_up(size, r_align);

  const size_t rs_align = page_size == (size_t) os::vm_page_size() ? 0 :

    MAX2(page_size, granularity);

  ReservedCodeSpace rs(r_size, rs_align, rs_align > 0);

  // Initialize bounds

  _low_bound = (address)rs.base();

  _high_bound = _low_bound + rs.size();

  return rs;

}

bool CodeCache::heap_available(int code_blob_type) {

  if (!SegmentedCodeCache) {

    // No segmentation: use a single code heap

    return (code_blob_type == CodeBlobType::All);

  } else if (Arguments::mode() == Arguments::_int) {

    // Interpreter only: we don't need any method code heaps

    return (code_blob_type == CodeBlobType::NonNMethod);

  } else if (TieredCompilation && (TieredStopAtLevel > CompLevel_simple)) {

    // Tiered compilation: use all code heaps

    return (code_blob_type < CodeBlobType::All);

  } else {

    // No TieredCompilation: we only need the non-nmethod and non-profiled code heap

    return (code_blob_type == CodeBlobType::NonNMethod) ||

           (code_blob_type == CodeBlobType::MethodNonProfiled);

  }

}

const char* CodeCache::get_code_heap_flag_name(int code_blob_type) {

  switch(code_blob_type) {

  case CodeBlobType::NonNMethod:

    return "NonNMethodCodeHeapSize";

    break;

  case CodeBlobType::MethodNonProfiled:

    return "NonProfiledCodeHeapSize";

    break;

  case CodeBlobType::MethodProfiled:

    return "ProfiledCodeHeapSize";

    break;

  }

  ShouldNotReachHere();

  return NULL;

}

void CodeCache::add_heap(ReservedSpace rs, const char* name, int code_blob_type) {

  // Check if heap is needed

  if (!heap_available(code_blob_type)) {

    return;

  }

  // Create CodeHeap

  CodeHeap* heap = new CodeHeap(name, code_blob_type);

  _heaps->append(heap);

  // Reserve Space

  size_t size_initial = MIN2(InitialCodeCacheSize, rs.size());

  size_initial = round_to(size_initial, os::vm_page_size());

  if (!heap->reserve(rs, size_initial, CodeCacheSegmentSize)) {

    vm_exit_during_initialization("Could not reserve enough space for code cache");

  }

  // Register the CodeHeap

  MemoryService::add_code_heap_memory_pool(heap, name);

}

CodeHeap* CodeCache::get_code_heap(const CodeBlob* cb) {

  assert(cb != NULL, "CodeBlob is null");

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->contains(cb)) {

      return *heap;

    }

  }

  ShouldNotReachHere();

  return NULL;

}

CodeHeap* CodeCache::get_code_heap(int code_blob_type) {

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->accepts(code_blob_type)) {

      return *heap;

    }

  }

  return NULL;

}

CodeBlob* CodeCache::first_blob(CodeHeap* heap) {

  assert_locked_or_safepoint(CodeCache_lock);

  assert(heap != NULL, "heap is null");

  return (CodeBlob*)heap->first();

}

CodeBlob* CodeCache::first_blob(int code_blob_type) {

  if (heap_available(code_blob_type)) {

    return first_blob(get_code_heap(code_blob_type));

  } else {

    return NULL;

  }

}

CodeBlob* CodeCache::next_blob(CodeHeap* heap, CodeBlob* cb) {

  assert_locked_or_safepoint(CodeCache_lock);

  assert(heap != NULL, "heap is null");

  return (CodeBlob*)heap->next(cb);

}

CodeBlob* CodeCache::next_blob(CodeBlob* cb) {

  return next_blob(get_code_heap(cb), cb);

}

/**

 * Do not seize the CodeCache lock here--if the caller has not

 * already done so, we are going to lose bigtime, since the code

 * cache will contain a garbage CodeBlob until the caller can

 * run the constructor for the CodeBlob subclass he is busy

 * instantiating.

 */

CodeBlob* CodeCache::allocate(int size, int code_blob_type, bool strict) {

  // Possibly wakes up the sweeper thread.

  NMethodSweeper::notify(code_blob_type);

  assert_locked_or_safepoint(CodeCache_lock);

  if (size <= 0) {

    return NULL;

  }

  CodeBlob* cb = NULL;

  // Get CodeHeap for the given CodeBlobType

  CodeHeap* heap = get_code_heap(code_blob_type);

  assert(heap != NULL, "heap is null");

  while (true) {

    cb = (CodeBlob*)heap->allocate(size);

    if (cb != NULL) break;

    if (!heap->expand_by(CodeCacheExpansionSize)) {

      // Expansion failed

      if (SegmentedCodeCache && !strict) {

        // Fallback solution: Try to store code in another code heap.

        // Note that in the sweeper, we check the reverse_free_ratio of the code heap

        // and force stack scanning if less than 10% of the code heap are free.

        int type = code_blob_type;

        switch (type) {

        case CodeBlobType::NonNMethod:

          type = CodeBlobType::MethodNonProfiled;

          strict = false;   // Allow recursive search for other heaps

          break;

        case CodeBlobType::MethodProfiled:

          type = CodeBlobType::MethodNonProfiled;

          strict = true;

          break;

        case CodeBlobType::MethodNonProfiled:

          type = CodeBlobType::MethodProfiled;

          strict = true;

          break;

        }

        if (heap_available(type)) {

          return allocate(size, type, strict);

        }

      }

      MutexUnlockerEx mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);

      CompileBroker::handle_full_code_cache(code_blob_type);

      return NULL;

    }

    if (PrintCodeCacheExtension) {

      ResourceMark rm;

      if (_heaps->length() >= 1) {

        tty->print("%s", heap->name());

      } else {

        tty->print("CodeCache");

      }

      tty->print_cr(" extended to [" INTPTR_FORMAT ", " INTPTR_FORMAT "] (" SSIZE_FORMAT " bytes)",

                    (intptr_t)heap->low_boundary(), (intptr_t)heap->high(),

                    (address)heap->high() - (address)heap->low_boundary());

    }

  }

  print_trace("allocation", cb, size);

  _number_of_blobs++;

  return cb;

}

void CodeCache::free(CodeBlob* cb) {

  assert_locked_or_safepoint(CodeCache_lock);

  print_trace("free", cb);

  if (cb->is_nmethod()) {

    _number_of_nmethods--;

    if (((nmethod *)cb)->has_dependencies()) {

      _number_of_nmethods_with_dependencies--;

    }

  }

  if (cb->is_adapter_blob()) {

    _number_of_adapters--;

  }

  _number_of_blobs--;

  // Get heap for given CodeBlob and deallocate

  get_code_heap(cb)->deallocate(cb);

  assert(_number_of_blobs >= 0, "sanity check");

}

void CodeCache::commit(CodeBlob* cb) {

  // this is called by nmethod::nmethod, which must already own CodeCache_lock

  assert_locked_or_safepoint(CodeCache_lock);

  if (cb->is_nmethod()) {

    _number_of_nmethods++;

    if (((nmethod *)cb)->has_dependencies()) {

      _number_of_nmethods_with_dependencies++;

    }

  }

  if (cb->is_adapter_blob()) {

    _number_of_adapters++;

  }

  // flush the hardware I-cache

  ICache::invalidate_range(cb->content_begin(), cb->content_size());

}

bool CodeCache::contains(void *p) {

  // It should be ok to call contains without holding a lock

  FOR_ALL_HEAPS(heap) {

    if ((*heap)->contains(p)) {

      return true;

    }

  }

  return false;

}

// This method is safe to call without holding the CodeCache_lock, as long as a dead CodeBlob is not

// looked up (i.e., one that has been marked for deletion). It only depends on the _segmap to contain

// valid indices, which it will always do, as long as the CodeBlob is not in the process of being recycled.

CodeBlob* CodeCache::find_blob(void* start) {

  CodeBlob* result = find_blob_unsafe(start);

  // We could potentially look up non_entrant methods

  guarantee(result == NULL || !result->is_zombie() || result->is_locked_by_vm() || is_error_reported(), "unsafe access to zombie method");

  return result;

}

// Lookup that does not fail if you lookup a zombie method (if you call this, be sure to know

// what you are doing)

CodeBlob* CodeCache::find_blob_unsafe(void* start) {

  // NMT can walk the stack before code cache is created

  if (_heaps == NULL || _heaps->is_empty()) return NULL;

  FOR_ALL_HEAPS(heap) {

    CodeBlob* result = (CodeBlob*) (*heap)->find_start(start);

    if (result != NULL && result->blob_contains((address)start)) {