/*

 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "classfile/classLoaderDataGraph.inline.hpp"

#include "classfile/dictionary.hpp"

#include "classfile/stringTable.hpp"

#include "classfile/symbolTable.hpp"

#include "classfile/systemDictionary.hpp"

#include "code/codeCache.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "code/scopeDesc.hpp"

#include "compiler/compilationPolicy.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shared/gcLocker.hpp"

#include "gc/shared/oopStorage.hpp"

#include "gc/shared/strongRootsScope.hpp"

#include "gc/shared/workgroup.hpp"

#include "interpreter/interpreter.hpp"

#include "jfr/jfrEvents.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "oops/oop.inline.hpp"

#include "oops/symbol.hpp"

#include "runtime/atomic.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/orderAccess.hpp"

#include "runtime/osThread.hpp"

#include "runtime/safepoint.hpp"

#include "runtime/safepointMechanism.inline.hpp"

#include "runtime/signature.hpp"

#include "runtime/stubCodeGenerator.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/sweeper.hpp"

#include "runtime/synchronizer.hpp"

#include "runtime/thread.inline.hpp"

#include "runtime/threadSMR.hpp"

#include "runtime/timerTrace.hpp"

#include "services/runtimeService.hpp"

#include "utilities/events.hpp"

#include "utilities/macros.hpp"

static void post_safepoint_begin_event(EventSafepointBegin& event,

                                       uint64_t safepoint_id,

                                       int thread_count,

                                       int critical_thread_count) {

  if (event.should_commit()) {

    event.set_safepointId(safepoint_id);

    event.set_totalThreadCount(thread_count);

    event.set_jniCriticalThreadCount(critical_thread_count);

    event.commit();

  }

}

static void post_safepoint_cleanup_event(EventSafepointCleanup& event, uint64_t safepoint_id) {

  if (event.should_commit()) {

    event.set_safepointId(safepoint_id);

    event.commit();

  }

}

static void post_safepoint_synchronize_event(EventSafepointStateSynchronization& event,

                                             uint64_t safepoint_id,

                                             int initial_number_of_threads,

                                             int threads_waiting_to_block,

                                             uint64_t iterations) {

  if (event.should_commit()) {

    event.set_safepointId(safepoint_id);

    event.set_initialThreadCount(initial_number_of_threads);

    event.set_runningThreadCount(threads_waiting_to_block);

    event.set_iterations(iterations);

    event.commit();

  }

}

static void post_safepoint_cleanup_task_event(EventSafepointCleanupTask& event,

                                              uint64_t safepoint_id,

                                              const char* name) {

  if (event.should_commit()) {

    event.set_safepointId(safepoint_id);

    event.set_name(name);

    event.commit();

  }

}

static void post_safepoint_end_event(EventSafepointEnd& event, uint64_t safepoint_id) {

  if (event.should_commit()) {

    event.set_safepointId(safepoint_id);

    event.commit();

  }

}

// SafepointCheck

SafepointStateTracker::SafepointStateTracker(uint64_t safepoint_id, bool at_safepoint)

  : _safepoint_id(safepoint_id), _at_safepoint(at_safepoint) {}

bool SafepointStateTracker::safepoint_state_changed() {

  return _safepoint_id != SafepointSynchronize::safepoint_id() ||

    _at_safepoint != SafepointSynchronize::is_at_safepoint();

}

// --------------------------------------------------------------------------------------------------

// Implementation of Safepoint begin/end

SafepointSynchronize::SynchronizeState volatile SafepointSynchronize::_state = SafepointSynchronize::_not_synchronized;

int SafepointSynchronize::_waiting_to_block = 0;

volatile uint64_t SafepointSynchronize::_safepoint_counter = 0;

uint64_t SafepointSynchronize::_safepoint_id = 0;

const uint64_t SafepointSynchronize::InactiveSafepointCounter = 0;

int SafepointSynchronize::_current_jni_active_count = 0;

WaitBarrier* SafepointSynchronize::_wait_barrier;

static volatile bool PageArmed = false;        // safepoint polling page is RO|RW vs PROT_NONE

static bool timeout_error_printed = false;

// Statistic related

static jlong _safepoint_begin_time = 0;

static volatile int _nof_threads_hit_polling_page = 0;

void SafepointSynchronize::init(Thread* vmthread) {

  // WaitBarrier should never be destroyed since we will have

  // threads waiting on it while exiting.

  _wait_barrier = new WaitBarrier(vmthread);

  SafepointTracing::init();

}

void SafepointSynchronize::increment_jni_active_count() {

  assert(Thread::current()->is_VM_thread(), "Only VM thread may increment");

  ++_current_jni_active_count;

}

void SafepointSynchronize::decrement_waiting_to_block() {

  assert(_waiting_to_block > 0, "sanity check");

  assert(Thread::current()->is_VM_thread(), "Only VM thread may decrement");

  --_waiting_to_block;

}

bool SafepointSynchronize::thread_not_running(ThreadSafepointState *cur_state) {

  if (!cur_state->is_running()) {

    return true;

  }

  cur_state->examine_state_of_thread(SafepointSynchronize::safepoint_counter());

  if (!cur_state->is_running()) {

    return true;

  }

  LogTarget(Trace, safepoint) lt;

  if (lt.is_enabled()) {

    ResourceMark rm;

    LogStream ls(lt);

    cur_state->print_on(&ls);

  }

  return false;

}

#ifdef ASSERT

static void assert_list_is_valid(const ThreadSafepointState* tss_head, int still_running) {

  int a = 0;

  const ThreadSafepointState *tmp_tss = tss_head;

  while (tmp_tss != NULL) {

    ++a;

    assert(tmp_tss->is_running(), "Illegal initial state");

    tmp_tss = tmp_tss->get_next();

  }

  assert(a == still_running, "Must be the same");

}

#endif // ASSERT

static void back_off(int64_t start_time) {

  // We start with fine-grained nanosleeping until a millisecond has

  // passed, at which point we resort to plain naked_short_sleep.

  if (os::javaTimeNanos() - start_time < NANOSECS_PER_MILLISEC) {

    os::naked_short_nanosleep(10 * (NANOUNITS / MICROUNITS));

  } else {

    os::naked_short_sleep(1);

  }

}

int SafepointSynchronize::synchronize_threads(jlong safepoint_limit_time, int nof_threads, int* initial_running)

{

  JavaThreadIteratorWithHandle jtiwh;

#ifdef ASSERT

  for (; JavaThread *cur = jtiwh.next(); ) {

    assert(cur->safepoint_state()->is_running(), "Illegal initial state");

  }

  jtiwh.rewind();

#endif // ASSERT

  // Iterate through all threads until it has been determined how to stop them all at a safepoint.

  int still_running = nof_threads;

  ThreadSafepointState *tss_head = NULL;

  ThreadSafepointState **p_prev = &tss_head;

  for (; JavaThread *cur = jtiwh.next(); ) {

    ThreadSafepointState *cur_tss = cur->safepoint_state();

    assert(cur_tss->get_next() == NULL, "Must be NULL");

    if (thread_not_running(cur_tss)) {

      --still_running;

    } else {

      *p_prev = cur_tss;

      p_prev = cur_tss->next_ptr();

    }

  }

  *p_prev = NULL;

  DEBUG_ONLY(assert_list_is_valid(tss_head, still_running);)

  *initial_running = still_running;

  // If there is no thread still running, we are already done.

  if (still_running <= 0) {

    assert(tss_head == NULL, "Must be empty");

    return 1;

  }

  int iterations = 1; // The first iteration is above.

  int64_t start_time = os::javaTimeNanos();

  do {

    // Check if this has taken too long:

    if (SafepointTimeout && safepoint_limit_time < os::javaTimeNanos()) {

      print_safepoint_timeout();

    }

    if (int(iterations) == -1) { // overflow - something is wrong.

      // We can only overflow here when we are using global

      // polling pages. We keep this guarantee in its original

      // form so that searches of the bug database for this

      // failure mode find the right bugs.

      guarantee (!PageArmed, "invariant");

    }

    p_prev = &tss_head;

    ThreadSafepointState *cur_tss = tss_head;

    while (cur_tss != NULL) {

      assert(cur_tss->is_running(), "Illegal initial state");

      if (thread_not_running(cur_tss)) {

        --still_running;

        *p_prev = NULL;

        ThreadSafepointState *tmp = cur_tss;

        cur_tss = cur_tss->get_next();

        tmp->set_next(NULL);

      } else {

        *p_prev = cur_tss;

        p_prev = cur_tss->next_ptr();

        cur_tss = cur_tss->get_next();

      }

    }

    DEBUG_ONLY(assert_list_is_valid(tss_head, still_running);)

    if (still_running > 0) {

      back_off(start_time);

    }

    iterations++;

  } while (still_running > 0);

  assert(tss_head == NULL, "Must be empty");

  return iterations;

}

void SafepointSynchronize::arm_safepoint() {

  // Begin the process of bringing the system to a safepoint.

  // Java threads can be in several different states and are

  // stopped by different mechanisms:

  //

  //  1. Running interpreted

  //     When executing branching/returning byte codes interpreter

  //     checks if the poll is armed, if so blocks in SS::block().

  //     When using global polling the interpreter dispatch table

  //     is changed to force it to check for a safepoint condition

  //     between bytecodes.

  //  2. Running in native code

  //     When returning from the native code, a Java thread must check

  //     the safepoint _state to see if we must block.  If the

  //     VM thread sees a Java thread in native, it does

  //     not wait for this thread to block.  The order of the memory

  //     writes and reads of both the safepoint state and the Java

  //     threads state is critical.  In order to guarantee that the

  //     memory writes are serialized with respect to each other,

  //     the VM thread issues a memory barrier instruction.

  //  3. Running compiled Code

  //     Compiled code reads the local polling page that

  //     is set to fault if we are trying to get to a safepoint.

  //  4. Blocked

  //     A thread which is blocked will not be allowed to return from the

  //     block condition until the safepoint operation is complete.

  //  5. In VM or Transitioning between states

  //     If a Java thread is currently running in the VM or transitioning

  //     between states, the safepointing code will poll the thread state

  //     until the thread blocks itself when it attempts transitions to a

  //     new state or locking a safepoint checked monitor.

  // We must never miss a thread with correct safepoint id, so we must make sure we arm

  // the wait barrier for the next safepoint id/counter.

  // Arming must be done after resetting _current_jni_active_count, _waiting_to_block.

  _wait_barrier->arm(static_cast<int>(_safepoint_counter + 1));

  assert((_safepoint_counter & 0x1) == 0, "must be even");

  // The store to _safepoint_counter must happen after any stores in arming.

  Atomic::release_store(&_safepoint_counter, _safepoint_counter + 1);

  // We are synchronizing

  OrderAccess::storestore(); // Ordered with _safepoint_counter

  _state = _synchronizing;

  if (SafepointMechanism::uses_thread_local_poll()) {

    // Arming the per thread poll while having _state != _not_synchronized means safepointing

    log_trace(safepoint)("Setting thread local yield flag for threads");

    OrderAccess::storestore(); // storestore, global state -> local state

    for (JavaThreadIteratorWithHandle jtiwh; JavaThread *cur = jtiwh.next(); ) {

      // Make sure the threads start polling, it is time to yield.

      SafepointMechanism::arm_local_poll(cur);

    }

  }

  OrderAccess::fence(); // storestore|storeload, global state -> local state

  if (SafepointMechanism::uses_global_page_poll()) {

    // Make interpreter safepoint aware

    Interpreter::notice_safepoints();

    // Make polling safepoint aware

    guarantee (!PageArmed, "invariant") ;

    PageArmed = true;

    os::make_polling_page_unreadable();

  }

}

// Roll all threads forward to a safepoint and suspend them all

void SafepointSynchronize::begin() {

  assert(Thread::current()->is_VM_thread(), "Only VM thread may execute a safepoint");

  EventSafepointBegin begin_event;

  SafepointTracing::begin(VMThread::vm_op_type());

  Universe::heap()->safepoint_synchronize_begin();

  // By getting the Threads_lock, we assure that no threads are about to start or

  // exit. It is released again in SafepointSynchronize::end().

  Threads_lock->lock();

  assert( _state == _not_synchronized, "trying to safepoint synchronize with wrong state");

  int nof_threads = Threads::number_of_threads();

  _nof_threads_hit_polling_page = 0;

  log_debug(safepoint)("Safepoint synchronization initiated using %s wait barrier. (%d threads)", _wait_barrier->description(), nof_threads);

  // Reset the count of active JNI critical threads

  _current_jni_active_count = 0;

  // Set number of threads to wait for

  _waiting_to_block = nof_threads;

  jlong safepoint_limit_time = 0;

  if (SafepointTimeout) {

    // Set the limit time, so that it can be compared to see if this has taken

    // too long to complete.

    safepoint_limit_time = SafepointTracing::start_of_safepoint() + (jlong)SafepointTimeoutDelay * (NANOUNITS / MILLIUNITS);

    timeout_error_printed = false;

  }

  EventSafepointStateSynchronization sync_event;

  int initial_running = 0;

  // Arms the safepoint, _current_jni_active_count and _waiting_to_block must be set before.

  arm_safepoint();

  // Will spin until all threads are safe.

  int iterations = synchronize_threads(safepoint_limit_time, nof_threads, &initial_running);

  assert(_waiting_to_block == 0, "No thread should be running");

#ifndef PRODUCT

  if (safepoint_limit_time != 0) {

    jlong current_time = os::javaTimeNanos();

    if (safepoint_limit_time < current_time) {

      log_warning(safepoint)("# SafepointSynchronize: Finished after "

                    INT64_FORMAT_W(6) " ms",

                    (int64_t)(current_time - SafepointTracing::start_of_safepoint()) / (NANOUNITS / MILLIUNITS));

    }

  }

#endif

  assert(Threads_lock->owned_by_self(), "must hold Threads_lock");

  // Record state

  _state = _synchronized;

  OrderAccess::fence();

  // Set the new id

  ++_safepoint_id;

#ifdef ASSERT

  // Make sure all the threads were visited.

  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *cur = jtiwh.next(); ) {

    assert(cur->was_visited_for_critical_count(_safepoint_counter), "missed a thread");

  }

#endif // ASSERT

  // Update the count of active JNI critical regions

  GCLocker::set_jni_lock_count(_current_jni_active_count);

  post_safepoint_synchronize_event(sync_event,

                                   _safepoint_id,

                                   initial_running,

                                   _waiting_to_block, iterations);

  SafepointTracing::synchronized(nof_threads, initial_running, _nof_threads_hit_polling_page);

  // We do the safepoint cleanup first since a GC related safepoint

  // needs cleanup to be completed before running the GC op.

  EventSafepointCleanup cleanup_event;

  do_cleanup_tasks();

  post_safepoint_cleanup_event(cleanup_event, _safepoint_id);

  post_safepoint_begin_event(begin_event, _safepoint_id, nof_threads, _current_jni_active_count);

  SafepointTracing::cleanup();

}

void SafepointSynchronize::disarm_safepoint() {

  uint64_t active_safepoint_counter = _safepoint_counter;

  {

    JavaThreadIteratorWithHandle jtiwh;

#ifdef ASSERT

    // A pending_exception cannot be installed during a safepoint.  The threads

    // may install an async exception after they come back from a safepoint into

    // pending_exception after they unblock.  But that should happen later.

    for (; JavaThread *cur = jtiwh.next(); ) {

      assert (!(cur->has_pending_exception() &&

                cur->safepoint_state()->is_at_poll_safepoint()),

              "safepoint installed a pending exception");

    }

#endif // ASSERT

    if (SafepointMechanism::uses_global_page_poll()) {

      guarantee (PageArmed, "invariant");

      // Make polling safepoint aware

      os::make_polling_page_readable();

      PageArmed = false;

      // Remove safepoint check from interpreter

      Interpreter::ignore_safepoints();

    }

    OrderAccess::fence(); // keep read and write of _state from floating up

    assert(_state == _synchronized, "must be synchronized before ending safepoint synchronization");

    // Change state first to _not_synchronized.

    // No threads should see _synchronized when running.

    _state = _not_synchronized;

    // Set the next dormant (even) safepoint id.

    assert((_safepoint_counter & 0x1) == 1, "must be odd");

    Atomic::release_store(&_safepoint_counter, _safepoint_counter + 1);

    OrderAccess::fence(); // Keep the local state from floating up.

    jtiwh.rewind();