// This exists only as a workaround of dtrace bug 6254741

int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {

  DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);

  return 0;

}

#define NINFLATIONLOCKS 256

static volatile intptr_t InflationLocks [NINFLATIONLOCKS] ;

ObjectMonitor * ObjectSynchronizer::gBlockList = NULL ;

ObjectMonitor * volatile ObjectSynchronizer::gFreeList  = NULL ;

ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList  = NULL ;

int ObjectSynchronizer::gOmInUseCount = 0;

static volatile intptr_t ListLock = 0 ;      // protects global monitor free-list cache

static volatile int MonitorFreeCount  = 0 ;      // # on gFreeList

static volatile int MonitorPopulation = 0 ;      // # Extant -- in circulation

#define CHAINMARKER ((oop)-1)

// -----------------------------------------------------------------------------

//  Fast Monitor Enter/Exit

// This the fast monitor enter. The interpreter and compiler use

// some assembly copies of this code. Make sure update those code

// if the following function is changed. The implementation is

// extremely sensitive to race condition. Be careful.

void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock, bool attempt_rebias, TRAPS) {

 if (UseBiasedLocking) {

    if (!SafepointSynchronize::is_at_safepoint()) {

      BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);

      if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) {

        return;

      }

    } else {

      assert(!attempt_rebias, "can not rebias toward VM thread");

      BiasedLocking::revoke_at_safepoint(obj);

    }

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

 }

 slow_enter (obj, lock, THREAD) ;

}

void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {

  assert(!object->mark()->has_bias_pattern(), "should not see bias pattern here");

  // if displaced header is null, the previous enter is recursive enter, no-op

  markOop dhw = lock->displaced_header();

  markOop mark ;

  if (dhw == NULL) {

     // Recursive stack-lock.

     // Diagnostics -- Could be: stack-locked, inflating, inflated.

     mark = object->mark() ;

     assert (!mark->is_neutral(), "invariant") ;

     if (mark->has_locker() && mark != markOopDesc::INFLATING()) {

        assert(THREAD->is_lock_owned((address)mark->locker()), "invariant") ;

     }

     if (mark->has_monitor()) {

        ObjectMonitor * m = mark->monitor() ;

        assert(((oop)(m->object()))->mark() == mark, "invariant") ;

        assert(m->is_entered(THREAD), "invariant") ;

     }

     return ;

  }

  mark = object->mark() ;

  // If the object is stack-locked by the current thread, try to

  // swing the displaced header from the box back to the mark.

  if (mark == (markOop) lock) {

     assert (dhw->is_neutral(), "invariant") ;

     if ((markOop) Atomic::cmpxchg_ptr (dhw, object->mark_addr(), mark) == mark) {

        TEVENT (fast_exit: release stacklock) ;

        return;

     }

  }

  ObjectSynchronizer::inflate(THREAD, object)->exit (THREAD) ;

}

// -----------------------------------------------------------------------------

// Interpreter/Compiler Slow Case

// This routine is used to handle interpreter/compiler slow case

// We don't need to use fast path here, because it must have been

// failed in the interpreter/compiler code.

void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {

  markOop mark = obj->mark();

  assert(!mark->has_bias_pattern(), "should not see bias pattern here");

  if (mark->is_neutral()) {

    // Anticipate successful CAS -- the ST of the displaced mark must

    // be visible <= the ST performed by the CAS.

    lock->set_displaced_header(mark);

    if (mark == (markOop) Atomic::cmpxchg_ptr(lock, obj()->mark_addr(), mark)) {

      TEVENT (slow_enter: release stacklock) ;

      return ;

    }

    // Fall through to inflate() ...

  } else

  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {

    assert(lock != mark->locker(), "must not re-lock the same lock");

    assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock");

    lock->set_displaced_header(NULL);

    return;

  }

#if 0

  // The following optimization isn't particularly useful.

  if (mark->has_monitor() && mark->monitor()->is_entered(THREAD)) {

    lock->set_displaced_header (NULL) ;

    return ;

  }

#endif

  // The object header will never be displaced to this lock,

  // so it does not matter what the value is, except that it

  // must be non-zero to avoid looking like a re-entrant lock,

  // and must not look locked either.

  lock->set_displaced_header(markOopDesc::unused_mark());

  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);

}

// This routine is used to handle interpreter/compiler slow case

// We don't need to use fast path here, because it must have

// failed in the interpreter/compiler code. Simply use the heavy

// weight monitor should be ok, unless someone find otherwise.

void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {

  fast_exit (object, lock, THREAD) ;

}

// -----------------------------------------------------------------------------

// Class Loader  support to workaround deadlocks on the class loader lock objects

// Also used by GC

// complete_exit()/reenter() are used to wait on a nested lock

// i.e. to give up an outer lock completely and then re-enter

// Used when holding nested locks - lock acquisition order: lock1 then lock2

//  1) complete_exit lock1 - saving recursion count

//  2) wait on lock2

//  3) when notified on lock2, unlock lock2

//  4) reenter lock1 with original recursion count

//  5) lock lock2

// NOTE: must use heavy weight monitor to handle complete_exit/reenter()

intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {

  TEVENT (complete_exit) ;

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());

  return monitor->complete_exit(THREAD);

}

// NOTE: must use heavy weight monitor to handle complete_exit/reenter()

void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {

  TEVENT (reenter) ;

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());

  monitor->reenter(recursion, THREAD);

}

// -----------------------------------------------------------------------------

// JNI locks on java objects

// NOTE: must use heavy weight monitor to handle jni monitor enter

void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) { // possible entry from jni enter

  // the current locking is from JNI instead of Java code

  TEVENT (jni_enter) ;

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  THREAD->set_current_pending_monitor_is_from_java(false);

  ObjectSynchronizer::inflate(THREAD, obj())->enter(THREAD);

  THREAD->set_current_pending_monitor_is_from_java(true);

}

// NOTE: must use heavy weight monitor to handle jni monitor enter

bool ObjectSynchronizer::jni_try_enter(Handle obj, Thread* THREAD) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  ObjectMonitor* monitor = ObjectSynchronizer::inflate_helper(obj());

  return monitor->try_enter(THREAD);

}

// NOTE: must use heavy weight monitor to handle jni monitor exit

void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {

  TEVENT (jni_exit) ;

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

  }

  assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj);

  // If this thread has locked the object, exit the monitor.  Note:  can't use

  // monitor->check(CHECK); must exit even if an exception is pending.

  if (monitor->check(THREAD)) {

     monitor->exit(THREAD);

}

// -----------------------------------------------------------------------------

// Internal VM locks on java objects

// standard constructor, allows locking failures

ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool doLock) {

  _dolock = doLock;

  _thread = thread;

  debug_only(if (StrictSafepointChecks) _thread->check_for_valid_safepoint_state(false);)

  _obj = obj;

  if (_dolock) {

    TEVENT (ObjectLocker) ;

    ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);

  }

}

ObjectLocker::~ObjectLocker() {

  if (_dolock) {

    ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);

  }

}

// -----------------------------------------------------------------------------

//  Wait/Notify/NotifyAll

// NOTE: must use heavy weight monitor to handle wait()

void ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  if (millis < 0) {

    TEVENT (wait - throw IAX) ;

    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");

  }

  ObjectMonitor* monitor = ObjectSynchronizer::inflate(THREAD, obj());

  DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);

  monitor->wait(millis, true, THREAD);

  /* This dummy call is in place to get around dtrace bug 6254741.  Once

     that's fixed we can uncomment the following line and remove the call */

  // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);

  dtrace_waited_probe(monitor, obj, THREAD);

}

void ObjectSynchronizer::waitUninterruptibly (Handle obj, jlong millis, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  if (millis < 0) {

    TEVENT (wait - throw IAX) ;

    THROW_MSG(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");

  }

  ObjectSynchronizer::inflate(THREAD, obj()) -> wait(millis, false, THREAD) ;

}

void ObjectSynchronizer::notify(Handle obj, TRAPS) {

 if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  markOop mark = obj->mark();

  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {

    return;

  }

  ObjectSynchronizer::inflate(THREAD, obj())->notify(THREAD);

}

// NOTE: see comment of notify()

void ObjectSynchronizer::notifyall(Handle obj, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  markOop mark = obj->mark();

  if (mark->has_locker() && THREAD->is_lock_owned((address)mark->locker())) {

    return;

  }

  ObjectSynchronizer::inflate(THREAD, obj())->notifyAll(THREAD);

}

// -----------------------------------------------------------------------------

// Hash Code handling

//

static markOop ReadStableMark (oop obj) {

  markOop mark = obj->mark() ;

  if (!mark->is_being_inflated()) {

    return mark ;       // normal fast-path return

  }

  int its = 0 ;

  for (;;) {

    markOop mark = obj->mark() ;

    if (!mark->is_being_inflated()) {

      return mark ;    // normal fast-path return

    }

    // The object is being inflated by some other thread.

    // The caller of ReadStableMark() must wait for inflation to complete.

    // Avoid live-lock

    // TODO: consider calling SafepointSynchronize::do_call_back() while

    // spinning to see if there's a safepoint pending.  If so, immediately

    // yielding or blocking would be appropriate.  Avoid spinning while

    // there is a safepoint pending.

    // TODO: add inflation contention performance counters.

    // TODO: restrict the aggregate number of spinners.

    ++its ;

    if (its > 10000 || !os::is_MP()) {

       if (its & 1) {

         os::NakedYield() ;

         TEVENT (Inflate: INFLATING - yield) ;

       } else {

         // Note that the following code attenuates the livelock problem but is not

         // a complete remedy.  A more complete solution would require that the inflating

         // thread hold the associated inflation lock.  The following code simply restricts

         // the number of spinners to at most one.  We'll have N-2 threads blocked

         // on the inflationlock, 1 thread holding the inflation lock and using

         // a yield/park strategy, and 1 thread in the midst of inflation.

         // A more refined approach would be to change the encoding of INFLATING

         // to allow encapsulation of a native thread pointer.  Threads waiting for

         // inflation to complete would use CAS to push themselves onto a singly linked

         // list rooted at the markword.  Once enqueued, they'd loop, checking a per-thread flag

         // and calling park().  When inflation was complete the thread that accomplished inflation

         // would detach the list and set the markword to inflated with a single CAS and

         // then for each thread on the list, set the flag and unpark() the thread.

         // This is conceptually similar to muxAcquire-muxRelease, except that muxRelease

         // wakes at most one thread whereas we need to wake the entire list.

         int ix = (intptr_t(obj) >> 5) & (NINFLATIONLOCKS-1) ;

         int YieldThenBlock = 0 ;

         assert (ix >= 0 && ix < NINFLATIONLOCKS, "invariant") ;

         assert ((NINFLATIONLOCKS & (NINFLATIONLOCKS-1)) == 0, "invariant") ;

         Thread::muxAcquire (InflationLocks + ix, "InflationLock") ;

         while (obj->mark() == markOopDesc::INFLATING()) {

           // Beware: NakedYield() is advisory and has almost no effect on some platforms

           // so we periodically call Self->_ParkEvent->park(1).

           // We use a mixed spin/yield/block mechanism.

           if ((YieldThenBlock++) >= 16) {

              Thread::current()->_ParkEvent->park(1) ;

           } else {

              os::NakedYield() ;

           }

         }

         Thread::muxRelease (InflationLocks + ix ) ;

         TEVENT (Inflate: INFLATING - yield/park) ;

       }

    } else {

       SpinPause() ;       // SMP-polite spinning

    }

  }

}