/*

 * Copyright 1997-2007 Sun Microsystems, Inc.  All Rights Reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,

 * CA 95054 USA or visit www.sun.com if you need additional information or

 * have any questions.

 *

 */

# include "incls/_precompiled.incl"

# include "incls/_safepoint.cpp.incl"

// --------------------------------------------------------------------------------------------------

// Implementation of Safepoint begin/end

SafepointSynchronize::SynchronizeState volatile SafepointSynchronize::_state = SafepointSynchronize::_not_synchronized;

volatile int  SafepointSynchronize::_waiting_to_block = 0;

jlong SafepointSynchronize::_last_safepoint = 0;

volatile int SafepointSynchronize::_safepoint_counter = 0;

static volatile int PageArmed = 0 ;        // safepoint polling page is RO|RW vs PROT_NONE

static volatile int TryingToBlock = 0 ;    // proximate value -- for advisory use only

static bool timeout_error_printed = false;

// Roll all threads forward to a safepoint and suspend them all

void SafepointSynchronize::begin() {

  Thread* myThread = Thread::current();

  assert(myThread->is_VM_thread(), "Only VM thread may execute a safepoint");

  _last_safepoint = os::javaTimeNanos();

#ifndef SERIALGC

  if (UseConcMarkSweepGC) {

    // In the future we should investigate whether CMS can use the

    // more-general mechanism below.  DLD (01/05).

    ConcurrentMarkSweepThread::synchronize(false);

  } else {

    ConcurrentGCThread::safepoint_synchronize();

  }

#endif // SERIALGC

  // By getting the Threads_lock, we assure that no threads are about to start or

  // exit. It is released again in SafepointSynchronize::end().

  Threads_lock->lock();

  assert( _state == _not_synchronized, "trying to safepoint synchronize with wrong state");

  int nof_threads = Threads::number_of_threads();

  if (TraceSafepoint) {

    tty->print_cr("Safepoint synchronization initiated. (%d)", nof_threads);

  }

  RuntimeService::record_safepoint_begin();

  {

  MutexLocker mu(Safepoint_lock);

  // Set number of threads to wait for, before we initiate the callbacks

  _waiting_to_block = nof_threads;

  TryingToBlock     = 0 ;

  int still_running = nof_threads;

  // Save the starting time, so that it can be compared to see if this has taken

  // too long to complete.

  jlong safepoint_limit_time;

  timeout_error_printed = false;

  // Begin the process of bringing the system to a safepoint.

  // Java threads can be in several different states and are

  // stopped by different mechanisms:

  //

  //  1. Running interpreted

  //     The interpeter dispatch table is changed to force it to

  //     check for a safepoint condition between bytecodes.

  //  2. Running in native code

  //     When returning from the native code, a Java thread must check

  //     the safepoint _state to see if we must block.  If the

  //     VM thread sees a Java thread in native, it does

  //     not wait for this thread to block.  The order of the memory

  //     writes and reads of both the safepoint state and the Java

  //     threads state is critical.  In order to guarantee that the

  //     memory writes are serialized with respect to each other,

  //     the VM thread issues a memory barrier instruction

  //     (on MP systems).  In order to avoid the overhead of issuing

  //     a memory barrier for each Java thread making native calls, each Java

  //     thread performs a write to a single memory page after changing

  //     the thread state.  The VM thread performs a sequence of

  //     mprotect OS calls which forces all previous writes from all

  //     Java threads to be serialized.  This is done in the

  //     os::serialize_thread_states() call.  This has proven to be

  //     much more efficient than executing a membar instruction

  //     on every call to native code.

  //  3. Running compiled Code

  //     Compiled code reads a global (Safepoint Polling) page that

  //     is set to fault if we are trying to get to a safepoint.

  //  4. Blocked

  //     A thread which is blocked will not be allowed to return from the

  //     block condition until the safepoint operation is complete.

  //  5. In VM or Transitioning between states

  //     If a Java thread is currently running in the VM or transitioning

  //     between states, the safepointing code will wait for the thread to

  //     block itself when it attempts transitions to a new state.

  //

  _state            = _synchronizing;

  OrderAccess::fence();

  // Flush all thread states to memory

  if (!UseMembar) {

    os::serialize_thread_states();

  }

  // Make interpreter safepoint aware

  Interpreter::notice_safepoints();

  if (UseCompilerSafepoints && DeferPollingPageLoopCount < 0) {

    // Make polling safepoint aware

    guarantee (PageArmed == 0, "invariant") ;

    PageArmed = 1 ;

    os::make_polling_page_unreadable();

  }

  // Consider using active_processor_count() ... but that call is expensive.

  int ncpus = os::processor_count() ;

#ifdef ASSERT

  for (JavaThread *cur = Threads::first(); cur != NULL; cur = cur->next()) {

    assert(cur->safepoint_state()->is_running(), "Illegal initial state");

  }

#endif // ASSERT

  if (SafepointTimeout)

    safepoint_limit_time = os::javaTimeNanos() + (jlong)SafepointTimeoutDelay * MICROUNITS;

  // Iterate through all threads until it have been determined how to stop them all at a safepoint

  unsigned int iterations = 0;

  int steps = 0 ;

  while(still_running > 0) {

    for (JavaThread *cur = Threads::first(); cur != NULL; cur = cur->next()) {

      assert(!cur->is_ConcurrentGC_thread(), "A concurrent GC thread is unexpectly being suspended");

      ThreadSafepointState *cur_state = cur->safepoint_state();

      if (cur_state->is_running()) {

        cur_state->examine_state_of_thread();

        if (!cur_state->is_running()) {

           still_running--;

           // consider adjusting steps downward:

           //   steps = 0

           //   steps -= NNN

           //   steps >>= 1

           //   steps = MIN(steps, 2000-100)

           //   if (iterations != 0) steps -= NNN

        }

        if (TraceSafepoint && Verbose) cur_state->print();

      }

    }

    if ( (PrintSafepointStatistics || (PrintSafepointStatisticsTimeout > 0))

         && iterations == 0) {

      begin_statistics(nof_threads, still_running);

    }

    if (still_running > 0) {

      // Check for if it takes to long

      if (SafepointTimeout && safepoint_limit_time < os::javaTimeNanos()) {

        print_safepoint_timeout(_spinning_timeout);

      }

      // Spin to avoid context switching.

      // There's a tension between allowing the mutators to run (and rendezvous)

      // vs spinning.  As the VM thread spins, wasting cycles, it consumes CPU that

      // a mutator might otherwise use profitably to reach a safepoint.  Excessive

      // spinning by the VM thread on a saturated system can increase rendezvous latency.

      // Blocking or yielding incur their own penalties in the form of context switching

      // and the resultant loss of $ residency.

      //

      // Further complicating matters is that yield() does not work as naively expected

      // on many platforms -- yield() does not guarantee that any other ready threads

      // will run.   As such we revert yield_all() after some number of iterations.

      // Yield_all() is implemented as a short unconditional sleep on some platforms.

      // Typical operating systems round a "short" sleep period up to 10 msecs, so sleeping

      // can actually increase the time it takes the VM thread to detect that a system-wide

      // stop-the-world safepoint has been reached.  In a pathological scenario such as that

      // described in CR6415670 the VMthread may sleep just before the mutator(s) become safe.

      // In that case the mutators will be stalled waiting for the safepoint to complete and the

      // the VMthread will be sleeping, waiting for the mutators to rendezvous.  The VMthread

      // will eventually wake up and detect that all mutators are safe, at which point

      // we'll again make progress.

      //

      // Beware too that that the VMThread typically runs at elevated priority.

      // Its default priority is higher than the default mutator priority.

      // Obviously, this complicates spinning.

      //

      // Note too that on Windows XP SwitchThreadTo() has quite different behavior than Sleep(0).

      // Sleep(0) will _not yield to lower priority threads, while SwitchThreadTo() will.

      //

      // See the comments in synchronizer.cpp for additional remarks on spinning.

      //

      // In the future we might:

      // 1. Modify the safepoint scheme to avoid potentally unbounded spinning.

      //    This is tricky as the path used by a thread exiting the JVM (say on

      //    on JNI call-out) simply stores into its state field.  The burden

      //    is placed on the VM thread, which must poll (spin).

      // 2. Find something useful to do while spinning.  If the safepoint is GC-related

      //    we might aggressively scan the stacks of threads that are already safe.

      // 3. Use Solaris schedctl to examine the state of the still-running mutators.

      //    If all the mutators are ONPROC there's no reason to sleep or yield.

      // 4. YieldTo() any still-running mutators that are ready but OFFPROC.

      // 5. Check system saturation.  If the system is not fully saturated then

      //    simply spin and avoid sleep/yield.

      // 6. As still-running mutators rendezvous they could unpark the sleeping

      //    VMthread.  This works well for still-running mutators that become

      //    safe.  The VMthread must still poll for mutators that call-out.

      // 7. Drive the policy on time-since-begin instead of iterations.

      // 8. Consider making the spin duration a function of the # of CPUs:

      //    Spin = (((ncpus-1) * M) + K) + F(still_running)

      //    Alternately, instead of counting iterations of the outer loop

      //    we could count the # of threads visited in the inner loop, above.

      // 9. On windows consider using the return value from SwitchThreadTo()

      //    to drive subsequent spin/SwitchThreadTo()/Sleep(N) decisions.

      if (UseCompilerSafepoints && int(iterations) == DeferPollingPageLoopCount) {

         guarantee (PageArmed == 0, "invariant") ;

         PageArmed = 1 ;

         os::make_polling_page_unreadable();

      }

      // Instead of (ncpus > 1) consider either (still_running < (ncpus + EPSILON)) or

      // ((still_running + _waiting_to_block - TryingToBlock)) < ncpus)

      ++steps ;

      if (ncpus > 1 && steps < SafepointSpinBeforeYield) {

        SpinPause() ;     // MP-Polite spin

      } else

      if (steps < DeferThrSuspendLoopCount) {

        os::NakedYield() ;

      } else {

        os::yield_all(steps) ;

        // Alternately, the VM thread could transiently depress its scheduling priority or

        // transiently increase the priority of the tardy mutator(s).

      }

      iterations ++ ;

    }

    assert(iterations < (uint)max_jint, "We have been iterating in the safepoint loop too long");

  }

  assert(still_running == 0, "sanity check");

  if (PrintSafepointStatistics) {

    update_statistics_on_spin_end();

  }

  // wait until all threads are stopped

  while (_waiting_to_block > 0) {

    if (TraceSafepoint) tty->print_cr("Waiting for %d thread(s) to block", _waiting_to_block);

    if (!SafepointTimeout || timeout_error_printed) {

      Safepoint_lock->wait(true);  // true, means with no safepoint checks

    } else {

      // Compute remaining time

      jlong remaining_time = safepoint_limit_time - os::javaTimeNanos();

      // If there is no remaining time, then there is an error

      if (remaining_time < 0 || Safepoint_lock->wait(true, remaining_time / MICROUNITS)) {

        print_safepoint_timeout(_blocking_timeout);

      }

    }

  }

  assert(_waiting_to_block == 0, "sanity check");

#ifndef PRODUCT

  if (SafepointTimeout) {

    jlong current_time = os::javaTimeNanos();

    if (safepoint_limit_time < current_time) {

      tty->print_cr("# SafepointSynchronize: Finished after "

                    INT64_FORMAT_W(6) " ms",

                    ((current_time - safepoint_limit_time) / MICROUNITS +

                     SafepointTimeoutDelay));

    }

  }

#endif

  assert((_safepoint_counter & 0x1) == 0, "must be even");

  assert(Threads_lock->owned_by_self(), "must hold Threads_lock");

  _safepoint_counter ++;

  // Record state

  _state = _synchronized;

  OrderAccess::fence();

  if (TraceSafepoint) {

    VM_Operation *op = VMThread::vm_operation();

    tty->print_cr("Entering safepoint region: %s", (op != NULL) ? op->name() : "no vm operation");

  }

  RuntimeService::record_safepoint_synchronized();

  if (PrintSafepointStatistics) {

    update_statistics_on_sync_end(os::javaTimeNanos());

  }

  // Call stuff that needs to be run when a safepoint is just about to be completed

  do_cleanup_tasks();

  }

}

// Wake up all threads, so they are ready to resume execution after the safepoint

// operation has been carried out

void SafepointSynchronize::end() {

  assert(Threads_lock->owned_by_self(), "must hold Threads_lock");

  assert((_safepoint_counter & 0x1) == 1, "must be odd");

  _safepoint_counter ++;

  // memory fence isn't required here since an odd _safepoint_counter

  // value can do no harm and a fence is issued below anyway.

  DEBUG_ONLY(Thread* myThread = Thread::current();)

  assert(myThread->is_VM_thread(), "Only VM thread can execute a safepoint");

  if (PrintSafepointStatistics) {

    end_statistics(os::javaTimeNanos());

  }

#ifdef ASSERT

  // A pending_exception cannot be installed during a safepoint.  The threads

  // may install an async exception after they come back from a safepoint into

  // pending_exception after they unblock.  But that should happen later.

  for(JavaThread *cur = Threads::first(); cur; cur = cur->next()) {

    assert (!(cur->has_pending_exception() &&

              cur->safepoint_state()->is_at_poll_safepoint()),

            "safepoint installed a pending exception");

  }

#endif // ASSERT

  if (PageArmed) {

    // Make polling safepoint aware

    os::make_polling_page_readable();

    PageArmed = 0 ;

  }

  // Remove safepoint check from interpreter

  Interpreter::ignore_safepoints();

  {

    MutexLocker mu(Safepoint_lock);

    assert(_state == _synchronized, "must be synchronized before ending safepoint synchronization");

    // Set to not synchronized, so the threads will not go into the signal_thread_blocked method

    // when they get restarted.

    _state = _not_synchronized;

    OrderAccess::fence();

    if (TraceSafepoint) {

       tty->print_cr("Leaving safepoint region");

    }

    // Start suspended threads

    for(JavaThread *current = Threads::first(); current; current = current->next()) {

      // A problem occuring on Solaris is when attempting to restart threads

      // the first #cpus - 1 go well, but then the VMThread is preempted when we get

      // to the next one (since it has been running the longest).  We then have

      // to wait for a cpu to become available before we can continue restarting

      // threads.

      // FIXME: This causes the performance of the VM to degrade when active and with

      // large numbers of threads.  Apparently this is due to the synchronous nature

      // of suspending threads.

      //

      // TODO-FIXME: the comments above are vestigial and no longer apply.

      // Furthermore, using solaris' schedctl in this particular context confers no benefit

      if (VMThreadHintNoPreempt) {

        os::hint_no_preempt();

      }

      ThreadSafepointState* cur_state = current->safepoint_state();

      assert(cur_state->type() != ThreadSafepointState::_running, "Thread not suspended at safepoint");

      cur_state->restart();

      assert(cur_state->is_running(), "safepoint state has not been reset");

    }

    RuntimeService::record_safepoint_end();

    // Release threads lock, so threads can be created/destroyed again. It will also starts all threads

    // blocked in signal_thread_blocked

    Threads_lock->unlock();

  }

#ifndef SERIALGC

  // If there are any concurrent GC threads resume them.

  if (UseConcMarkSweepGC) {

    ConcurrentMarkSweepThread::desynchronize(false);

  } else {

    ConcurrentGCThread::safepoint_desynchronize();

  }

#endif // SERIALGC

}

bool SafepointSynchronize::is_cleanup_needed() {

  // Need a safepoint if some inline cache buffers is non-empty

  if (!InlineCacheBuffer::is_empty()) return true;

  return false;

}

jlong CounterDecay::_last_timestamp = 0;

static void do_method(methodOop m) {

  m->invocation_counter()->decay();

}

void CounterDecay::decay() {

  _last_timestamp = os::javaTimeMillis();

  // This operation is going to be performed only at the end of a safepoint

  // and hence GC's will not be going on, all Java mutators are suspended

  // at this point and hence SystemDictionary_lock is also not needed.

  assert(SafepointSynchronize::is_at_safepoint(), "can only be executed at a safepoint");

  int nclasses = SystemDictionary::number_of_classes();

  double classes_per_tick = nclasses * (CounterDecayMinIntervalLength * 1e-3 /

                                        CounterHalfLifeTime);

  for (int i = 0; i < classes_per_tick; i++) {

    klassOop k = SystemDictionary::try_get_next_class();

    if (k != NULL && k->klass_part()->oop_is_instance()) {

      instanceKlass::cast(k)->methods_do(do_method);

    }

  }

}

// Various cleaning tasks that should be done periodically at safepoints

void SafepointSynchronize::do_cleanup_tasks() {

  jlong cleanup_time;

  // Update fat-monitor pool, since this is a safepoint.

  if (TraceSafepoint) {

    cleanup_time = os::javaTimeNanos();

  }

  ObjectSynchronizer::deflate_idle_monitors();

  InlineCacheBuffer::update_inline_caches();

  if(UseCounterDecay && CounterDecay::is_decay_needed()) {

    CounterDecay::decay();

  }

  NMethodSweeper::sweep();

  if (TraceSafepoint) {

    tty->print_cr("do_cleanup_tasks takes "INT64_FORMAT_W(6) "ms",

                  (os::javaTimeNanos() - cleanup_time) / MICROUNITS);

  }

}

bool SafepointSynchronize::safepoint_safe(JavaThread *thread, JavaThreadState state) {

  switch(state) {

  case _thread_in_native:

    // native threads are safe if they have no java stack or have walkable stack

    return !thread->has_last_Java_frame() || thread->frame_anchor()->walkable();

   // blocked threads should have already have walkable stack

  case _thread_blocked:

    assert(!thread->has_last_Java_frame() || thread->frame_anchor()->walkable(), "blocked and not walkable");

    return true;

  default:

    return false;

  }

}

// -------------------------------------------------------------------------------------------------------

// Implementation of Safepoint callback point

void SafepointSynchronize::block(JavaThread *thread) {

  assert(thread != NULL, "thread must be set");

  assert(thread->is_Java_thread(), "not a Java thread");

  // Threads shouldn't block if they are in the middle of printing, but...

  ttyLocker::break_tty_lock_for_safepoint(os::current_thread_id());

  // Only bail from the block() call if the thread is gone from the

  // thread list; starting to exit should still block.

  if (thread->is_terminated()) {

     // block current thread if we come here from native code when VM is gone

     thread->block_if_vm_exited();

     // otherwise do nothing

     return;

  }

  JavaThreadState state = thread->thread_state();

  thread->frame_anchor()->make_walkable(thread);

  // Check that we have a valid thread_state at this point

  switch(state) {

    case _thread_in_vm_trans:

    case _thread_in_Java:        // From compiled code

      // We are highly likely to block on the Safepoint_lock. In order to avoid blocking in this case,

      // we pretend we are still in the VM.

      thread->set_thread_state(_thread_in_vm);

      if (is_synchronizing()) {

         Atomic::inc (&TryingToBlock) ;

      }

      // We will always be holding the Safepoint_lock when we are examine the state

      // of a thread. Hence, the instructions between the Safepoint_lock->lock() and

      // Safepoint_lock->unlock() are happening atomic with regards to the safepoint code

      Safepoint_lock->lock_without_safepoint_check();

      if (is_synchronizing()) {

        // Decrement the number of threads to wait for and signal vm thread

        assert(_waiting_to_block > 0, "sanity check");

        _waiting_to_block--;

        thread->safepoint_state()->set_has_called_back(true);

        // Consider (_waiting_to_block < 2) to pipeline the wakeup of the VM thread

        if (_waiting_to_block == 0) {

          Safepoint_lock->notify_all();

        }

      }