/*

 * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "classfile/classLoaderDataGraph.inline.hpp"

#include "classfile/stringTable.hpp"

#include "classfile/symbolTable.hpp"

#include "classfile/systemDictionary.hpp"

#include "code/codeCache.hpp"

#include "code/icBuffer.hpp"

#include "code/nmethod.hpp"

#include "code/pcDesc.hpp"

#include "code/scopeDesc.hpp"

#include "gc/shared/collectedHeap.hpp"

#include "gc/shared/gcLocker.hpp"

#include "gc/shared/strongRootsScope.hpp"

#include "gc/shared/workgroup.hpp"

#include "interpreter/interpreter.hpp"

#include "jfr/jfrEvents.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "memory/resourceArea.hpp"

#include "memory/universe.hpp"

#include "oops/oop.inline.hpp"

#include "oops/symbol.hpp"

#include "runtime/atomic.hpp"

#include "runtime/compilationPolicy.hpp"

#include "runtime/deoptimization.hpp"

#include "runtime/frame.inline.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/orderAccess.hpp"

#include "runtime/osThread.hpp"

#include "runtime/safepoint.hpp"

#include "runtime/safepointMechanism.inline.hpp"

#include "runtime/signature.hpp"

#include "runtime/stubCodeGenerator.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/sweeper.hpp"

#include "runtime/synchronizer.hpp"

#include "runtime/thread.inline.hpp"

#include "runtime/threadSMR.hpp"

#include "runtime/timerTrace.hpp"

#include "services/runtimeService.hpp"

#include "utilities/events.hpp"

#include "utilities/macros.hpp"

#ifdef COMPILER1

#include "c1/c1_globals.hpp"

#endif

template <typename E>

static void set_current_safepoint_id(E* event, int adjustment = 0) {

  assert(event != NULL, "invariant");

  event->set_safepointId(SafepointSynchronize::safepoint_counter() + adjustment);

}

static void post_safepoint_begin_event(EventSafepointBegin* event,

                                       int thread_count,

                                       int critical_thread_count) {

  assert(event != NULL, "invariant");

  assert(event->should_commit(), "invariant");

  set_current_safepoint_id(event);

  event->set_totalThreadCount(thread_count);

  event->set_jniCriticalThreadCount(critical_thread_count);

  event->commit();

}

static void post_safepoint_cleanup_event(EventSafepointCleanup* event) {

  assert(event != NULL, "invariant");

  assert(event->should_commit(), "invariant");

  set_current_safepoint_id(event);

  event->commit();

}

static void post_safepoint_synchronize_event(EventSafepointStateSynchronization* event,

                                             int initial_number_of_threads,

                                             int threads_waiting_to_block,

                                             unsigned int iterations) {

  assert(event != NULL, "invariant");

  if (event->should_commit()) {

    // Group this event together with the ones committed after the counter is increased

    set_current_safepoint_id(event, 1);

    event->set_initialThreadCount(initial_number_of_threads);

    event->set_runningThreadCount(threads_waiting_to_block);

    event->set_iterations(iterations);

    event->commit();

  }

}

static void post_safepoint_wait_blocked_event(EventSafepointWaitBlocked* event,

                                              int initial_threads_waiting_to_block) {

  assert(event != NULL, "invariant");

  assert(event->should_commit(), "invariant");

  set_current_safepoint_id(event);

  event->set_runningThreadCount(initial_threads_waiting_to_block);

  event->commit();

}

static void post_safepoint_cleanup_task_event(EventSafepointCleanupTask* event,

                                              const char* name) {

  assert(event != NULL, "invariant");

  if (event->should_commit()) {

    set_current_safepoint_id(event);

    event->set_name(name);

    event->commit();

  }

}

static void post_safepoint_end_event(EventSafepointEnd* event) {

  assert(event != NULL, "invariant");

  if (event->should_commit()) {

    // Group this event together with the ones committed before the counter increased

    set_current_safepoint_id(event, -1);

    event->commit();

  }

}

// --------------------------------------------------------------------------------------------------

// Implementation of Safepoint begin/end

SafepointSynchronize::SynchronizeState volatile SafepointSynchronize::_state = SafepointSynchronize::_not_synchronized;

volatile int  SafepointSynchronize::_waiting_to_block = 0;

volatile uint64_t SafepointSynchronize::_safepoint_counter = 0;

int SafepointSynchronize::_current_jni_active_count = 0;

long  SafepointSynchronize::_end_of_last_safepoint = 0;

int SafepointSynchronize::_defer_thr_suspend_loop_count = 4000;

static const int safepoint_spin_before_yield = 2000;

static volatile int PageArmed = 0 ;        // safepoint polling page is RO|RW vs PROT_NONE

static volatile int TryingToBlock = 0 ;    // proximate value -- for advisory use only

static bool timeout_error_printed = false;

// Statistic related statics

julong SafepointSynchronize::_coalesced_vmop_count = 0;

static jlong _safepoint_begin_time = 0;

static float _ts_of_current_safepoint = 0.0f;

static volatile int _nof_threads_hit_polling_page = 0;

// Roll all threads forward to a safepoint and suspend them all

void SafepointSynchronize::begin() {

  EventSafepointBegin begin_event;

  Thread* myThread = Thread::current();

  assert(myThread->is_VM_thread(), "Only VM thread may execute a safepoint");

  if (log_is_enabled(Debug, safepoint, stats)) {

    _safepoint_begin_time = os::javaTimeNanos();

    _ts_of_current_safepoint = tty->time_stamp().seconds();

    _nof_threads_hit_polling_page = 0;

  }

  Universe::heap()->safepoint_synchronize_begin();

  // By getting the Threads_lock, we assure that no threads are about to start or

  // exit. It is released again in SafepointSynchronize::end().

  Threads_lock->lock();

  assert( _state == _not_synchronized, "trying to safepoint synchronize with wrong state");

  int nof_threads = Threads::number_of_threads();

  log_debug(safepoint)("Safepoint synchronization initiated. (%d threads)", nof_threads);

  RuntimeService::record_safepoint_begin();

  MutexLocker mu(Safepoint_lock);

  // Reset the count of active JNI critical threads

  _current_jni_active_count = 0;

  // Set number of threads to wait for, before we initiate the callbacks

  _waiting_to_block = nof_threads;

  TryingToBlock     = 0 ;

  int still_running = nof_threads;

  // Save the starting time, so that it can be compared to see if this has taken

  // too long to complete.

  jlong safepoint_limit_time = 0;

  timeout_error_printed = false;

  // Begin the process of bringing the system to a safepoint.

  // Java threads can be in several different states and are

  // stopped by different mechanisms:

  //

  //  1. Running interpreted

  //     The interpreter dispatch table is changed to force it to

  //     check for a safepoint condition between bytecodes.

  //  2. Running in native code

  //     When returning from the native code, a Java thread must check

  //     the safepoint _state to see if we must block.  If the

  //     VM thread sees a Java thread in native, it does

  //     not wait for this thread to block.  The order of the memory

  //     writes and reads of both the safepoint state and the Java

  //     threads state is critical.  In order to guarantee that the

  //     memory writes are serialized with respect to each other,

  //     the VM thread issues a memory barrier instruction.

  //  3. Running compiled Code

  //     Compiled code reads a global (Safepoint Polling) page that

  //     is set to fault if we are trying to get to a safepoint.

  //  4. Blocked

  //     A thread which is blocked will not be allowed to return from the

  //     block condition until the safepoint operation is complete.

  //  5. In VM or Transitioning between states

  //     If a Java thread is currently running in the VM or transitioning

  //     between states, the safepointing code will wait for the thread to

  //     block itself when it attempts transitions to a new state.

  //

  {

    EventSafepointStateSynchronization sync_event;

    int initial_running = 0;

    _state            = _synchronizing;

    if (SafepointMechanism::uses_thread_local_poll()) {

      // Arming the per thread poll while having _state != _not_synchronized means safepointing

      log_trace(safepoint)("Setting thread local yield flag for threads");

      OrderAccess::storestore(); // storestore, global state -> local state

      for (JavaThreadIteratorWithHandle jtiwh; JavaThread *cur = jtiwh.next(); ) {

        // Make sure the threads start polling, it is time to yield.

        SafepointMechanism::arm_local_poll(cur);

      }

    }

    OrderAccess::fence(); // storestore|storeload, global state -> local state

    if (SafepointMechanism::uses_global_page_poll()) {

      // Make interpreter safepoint aware

      Interpreter::notice_safepoints();

      // Make polling safepoint aware

      guarantee (PageArmed == 0, "invariant") ;

      PageArmed = 1 ;

      os::make_polling_page_unreadable();

    }

    // Consider using active_processor_count() ... but that call is expensive.

    int ncpus = os::processor_count() ;

    unsigned int iterations = 0;

    {

      JavaThreadIteratorWithHandle jtiwh;

#ifdef ASSERT

      for (; JavaThread *cur = jtiwh.next(); ) {

        assert(cur->safepoint_state()->is_running(), "Illegal initial state");

        // Clear the visited flag to ensure that the critical counts are collected properly.

        cur->set_visited_for_critical_count(false);

      }

#endif // ASSERT

      if (SafepointTimeout)

        safepoint_limit_time = os::javaTimeNanos() + (jlong)SafepointTimeoutDelay * MICROUNITS;

      // Iterate through all threads until it have been determined how to stop them all at a safepoint

      int steps = 0 ;

      while(still_running > 0) {

        jtiwh.rewind();

        for (; JavaThread *cur = jtiwh.next(); ) {

          assert(!cur->is_ConcurrentGC_thread(), "A concurrent GC thread is unexpectly being suspended");

          ThreadSafepointState *cur_state = cur->safepoint_state();

          if (cur_state->is_running()) {

            cur_state->examine_state_of_thread();

            if (!cur_state->is_running()) {

              still_running--;

              // consider adjusting steps downward:

              //   steps = 0

              //   steps -= NNN

              //   steps >>= 1

              //   steps = MIN(steps, 2000-100)

              //   if (iterations != 0) steps -= NNN

            }

            LogTarget(Trace, safepoint) lt;

            if (lt.is_enabled()) {

              ResourceMark rm;

              LogStream ls(lt);

              cur_state->print_on(&ls);

            }

          }

        }

        if (iterations == 0) {

          initial_running = still_running;

          if (log_is_enabled(Debug, safepoint, stats)) {

            begin_statistics(nof_threads, still_running);

          }

        }

        if (still_running > 0) {

          // Check for if it takes to long

          if (SafepointTimeout && safepoint_limit_time < os::javaTimeNanos()) {

            print_safepoint_timeout(_spinning_timeout);

          }

          // Spin to avoid context switching.

          // There's a tension between allowing the mutators to run (and rendezvous)

          // vs spinning.  As the VM thread spins, wasting cycles, it consumes CPU that

          // a mutator might otherwise use profitably to reach a safepoint.  Excessive

          // spinning by the VM thread on a saturated system can increase rendezvous latency.

          // Blocking or yielding incur their own penalties in the form of context switching

          // and the resultant loss of $ residency.

          //

          // Further complicating matters is that yield() does not work as naively expected

          // on many platforms -- yield() does not guarantee that any other ready threads

          // will run.   As such we revert to naked_short_sleep() after some number of iterations.

          // nakes_short_sleep() is implemented as a short unconditional sleep.

          // Typical operating systems round a "short" sleep period up to 10 msecs, so sleeping

          // can actually increase the time it takes the VM thread to detect that a system-wide

          // stop-the-world safepoint has been reached.  In a pathological scenario such as that

          // described in CR6415670 the VMthread may sleep just before the mutator(s) become safe.

          // In that case the mutators will be stalled waiting for the safepoint to complete and the

          // the VMthread will be sleeping, waiting for the mutators to rendezvous.  The VMthread

          // will eventually wake up and detect that all mutators are safe, at which point

          // we'll again make progress.

          //

          // Beware too that that the VMThread typically runs at elevated priority.

          // Its default priority is higher than the default mutator priority.

          // Obviously, this complicates spinning.

          //

          // Note too that on Windows XP SwitchThreadTo() has quite different behavior than Sleep(0).

          // Sleep(0) will _not yield to lower priority threads, while SwitchThreadTo() will.

          //

          // See the comments in synchronizer.cpp for additional remarks on spinning.

          //

          // In the future we might:

          // -- Modify the safepoint scheme to avoid potentially unbounded spinning.

          //    This is tricky as the path used by a thread exiting the JVM (say on

          //    on JNI call-out) simply stores into its state field.  The burden

          //    is placed on the VM thread, which must poll (spin).

          // -- Find something useful to do while spinning.  If the safepoint is GC-related

          //    we might aggressively scan the stacks of threads that are already safe.

          // -- YieldTo() any still-running mutators that are ready but OFFPROC.

          // -- Check system saturation.  If the system is not fully saturated then

          //    simply spin and avoid sleep/yield.

          // -- As still-running mutators rendezvous they could unpark the sleeping

          //    VMthread.  This works well for still-running mutators that become

          //    safe.  The VMthread must still poll for mutators that call-out.

          // -- Drive the policy on time-since-begin instead of iterations.

          // -- Consider making the spin duration a function of the # of CPUs:

          //    Spin = (((ncpus-1) * M) + K) + F(still_running)

          //    Alternately, instead of counting iterations of the outer loop

          //    we could count the # of threads visited in the inner loop, above.

          // -- On windows consider using the return value from SwitchThreadTo()

          //    to drive subsequent spin/SwitchThreadTo()/Sleep(N) decisions.

          if (int(iterations) == -1) { // overflow - something is wrong.

            // We can only overflow here when we are using global

            // polling pages. We keep this guarantee in its original

            // form so that searches of the bug database for this

            // failure mode find the right bugs.

            guarantee (PageArmed == 0, "invariant");

          }

          // Instead of (ncpus > 1) consider either (still_running < (ncpus + EPSILON)) or

          // ((still_running + _waiting_to_block - TryingToBlock)) < ncpus)

          ++steps ;

          if (ncpus > 1 && steps < safepoint_spin_before_yield) {

            SpinPause() ;     // MP-Polite spin

          } else

            if (steps < _defer_thr_suspend_loop_count) {

              os::naked_yield() ;

            } else {

              os::naked_short_sleep(1);

            }

          iterations ++ ;

        }

        assert(iterations < (uint)max_jint, "We have been iterating in the safepoint loop too long");

      }

    } // ThreadsListHandle destroyed here.

    assert(still_running == 0, "sanity check");

    if (log_is_enabled(Debug, safepoint, stats)) {

      update_statistics_on_spin_end();

    }

    if (sync_event.should_commit()) {

      post_safepoint_synchronize_event(&sync_event, initial_running, _waiting_to_block, iterations);

    }

  }

  // wait until all threads are stopped

  {

    EventSafepointWaitBlocked wait_blocked_event;

    int initial_waiting_to_block = _waiting_to_block;

    while (_waiting_to_block > 0) {

      log_debug(safepoint)("Waiting for %d thread(s) to block", _waiting_to_block);

      if (!SafepointTimeout || timeout_error_printed) {

        Safepoint_lock->wait(true);  // true, means with no safepoint checks

      } else {

        // Compute remaining time

        jlong remaining_time = safepoint_limit_time - os::javaTimeNanos();

        // If there is no remaining time, then there is an error

        if (remaining_time < 0 || Safepoint_lock->wait(true, remaining_time / MICROUNITS)) {

          print_safepoint_timeout(_blocking_timeout);

        }

      }

    }

    assert(_waiting_to_block == 0, "sanity check");

#ifndef PRODUCT

    if (SafepointTimeout) {

      jlong current_time = os::javaTimeNanos();

      if (safepoint_limit_time < current_time) {

        log_warning(safepoint)("# SafepointSynchronize: Finished after "

                      INT64_FORMAT_W(6) " ms",

                      (int64_t)((current_time - safepoint_limit_time) / MICROUNITS +

                                (jlong)SafepointTimeoutDelay));

      }

    }

#endif

    assert((_safepoint_counter & 0x1) == 0, "must be even");

    assert(Threads_lock->owned_by_self(), "must hold Threads_lock");

    _safepoint_counter ++;

    // Record state

    _state = _synchronized;

    OrderAccess::fence();

    if (wait_blocked_event.should_commit()) {

      post_safepoint_wait_blocked_event(&wait_blocked_event, initial_waiting_to_block);

    }

  }

#ifdef ASSERT

  // Make sure all the threads were visited.

  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *cur = jtiwh.next(); ) {

    assert(cur->was_visited_for_critical_count(), "missed a thread");

  }

#endif // ASSERT

  // Update the count of active JNI critical regions

  GCLocker::set_jni_lock_count(_current_jni_active_count);

  log_info(safepoint)("Entering safepoint region: %s", VMThread::vm_safepoint_description());

  RuntimeService::record_safepoint_synchronized();

  if (log_is_enabled(Debug, safepoint, stats)) {

    update_statistics_on_sync_end(os::javaTimeNanos());

  }

  // Call stuff that needs to be run when a safepoint is just about to be completed

  {

    EventSafepointCleanup cleanup_event;

    do_cleanup_tasks();

    if (cleanup_event.should_commit()) {

      post_safepoint_cleanup_event(&cleanup_event);

    }

  }

  if (log_is_enabled(Debug, safepoint, stats)) {

    // Record how much time spend on the above cleanup tasks

    update_statistics_on_cleanup_end(os::javaTimeNanos());

  }

  if (begin_event.should_commit()) {

    post_safepoint_begin_event(&begin_event, nof_threads, _current_jni_active_count);

  }

}

// Wake up all threads, so they are ready to resume execution after the safepoint

// operation has been carried out

void SafepointSynchronize::end() {

  assert(Threads_lock->owned_by_self(), "must hold Threads_lock");

  assert((_safepoint_counter & 0x1) == 1, "must be odd");

  EventSafepointEnd event;

  _safepoint_counter ++;

  // memory fence isn't required here since an odd _safepoint_counter

  // value can do no harm and a fence is issued below anyway.

  DEBUG_ONLY(Thread* myThread = Thread::current();)

  assert(myThread->is_VM_thread(), "Only VM thread can execute a safepoint");