/*

 * Copyright (c) 1998, 2019, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "classfile/vmSymbols.hpp"

#include "logging/log.hpp"

#include "logging/logStream.hpp"

#include "jfr/jfrEvents.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/metaspaceShared.hpp"

#include "memory/padded.hpp"

#include "memory/resourceArea.hpp"

#include "oops/markOop.hpp"

#include "oops/oop.inline.hpp"

#include "runtime/atomic.hpp"

#include "runtime/biasedLocking.hpp"

#include "runtime/handles.inline.hpp"

#include "runtime/interfaceSupport.inline.hpp"

#include "runtime/mutexLocker.hpp"

#include "runtime/objectMonitor.hpp"

#include "runtime/objectMonitor.inline.hpp"

#include "runtime/osThread.hpp"

#include "runtime/safepointVerifiers.hpp"

#include "runtime/sharedRuntime.hpp"

#include "runtime/stubRoutines.hpp"

#include "runtime/synchronizer.hpp"

#include "runtime/thread.inline.hpp"

#include "runtime/timer.hpp"

#include "runtime/vframe.hpp"

#include "runtime/vmThread.hpp"

#include "utilities/align.hpp"

#include "utilities/dtrace.hpp"

#include "utilities/events.hpp"

#include "utilities/preserveException.hpp"

// The "core" versions of monitor enter and exit reside in this file.

// The interpreter and compilers contain specialized transliterated

// variants of the enter-exit fast-path operations.  See i486.ad fast_lock(),

// for instance.  If you make changes here, make sure to modify the

// interpreter, and both C1 and C2 fast-path inline locking code emission.

//

// -----------------------------------------------------------------------------

#ifdef DTRACE_ENABLED

// Only bother with this argument setup if dtrace is available

// TODO-FIXME: probes should not fire when caller is _blocked.  assert() accordingly.

#define DTRACE_MONITOR_PROBE_COMMON(obj, thread)                           \

  char* bytes = NULL;                                                      \

  int len = 0;                                                             \

  jlong jtid = SharedRuntime::get_java_tid(thread);                        \

  Symbol* klassname = ((oop)(obj))->klass()->name();                       \

  if (klassname != NULL) {                                                 \

    bytes = (char*)klassname->bytes();                                     \

    len = klassname->utf8_length();                                        \

  }

#define DTRACE_MONITOR_WAIT_PROBE(monitor, obj, thread, millis)            \

  {                                                                        \

    if (DTraceMonitorProbes) {                                             \

      DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \

      HOTSPOT_MONITOR_WAIT(jtid,                                           \

                           (uintptr_t)(monitor), bytes, len, (millis));    \

    }                                                                      \

  }

#define HOTSPOT_MONITOR_PROBE_notify HOTSPOT_MONITOR_NOTIFY

#define HOTSPOT_MONITOR_PROBE_notifyAll HOTSPOT_MONITOR_NOTIFYALL

#define HOTSPOT_MONITOR_PROBE_waited HOTSPOT_MONITOR_WAITED

#define DTRACE_MONITOR_PROBE(probe, monitor, obj, thread)                  \

  {                                                                        \

    if (DTraceMonitorProbes) {                                             \

      DTRACE_MONITOR_PROBE_COMMON(obj, thread);                            \

      HOTSPOT_MONITOR_PROBE_##probe(jtid, /* probe = waited */             \

                                    (uintptr_t)(monitor), bytes, len);     \

    }                                                                      \

  }

#else //  ndef DTRACE_ENABLED

#define DTRACE_MONITOR_WAIT_PROBE(obj, thread, millis, mon)    {;}

#define DTRACE_MONITOR_PROBE(probe, obj, thread, mon)          {;}

#endif // ndef DTRACE_ENABLED

// This exists only as a workaround of dtrace bug 6254741

int dtrace_waited_probe(ObjectMonitor* monitor, Handle obj, Thread* thr) {

  DTRACE_MONITOR_PROBE(waited, monitor, obj(), thr);

  return 0;

}

#define NINFLATIONLOCKS 256

static volatile intptr_t gInflationLocks[NINFLATIONLOCKS];

// global list of blocks of monitors

PaddedEnd<ObjectMonitor> * volatile ObjectSynchronizer::gBlockList = NULL;

// global monitor free list

ObjectMonitor * volatile ObjectSynchronizer::gFreeList  = NULL;

// global monitor in-use list, for moribund threads,

// monitors they inflated need to be scanned for deflation

ObjectMonitor * volatile ObjectSynchronizer::gOmInUseList  = NULL;

// count of entries in gOmInUseList

int ObjectSynchronizer::gOmInUseCount = 0;

static volatile intptr_t gListLock = 0;      // protects global monitor lists

static volatile int gMonitorFreeCount  = 0;  // # on gFreeList

static volatile int gMonitorPopulation = 0;  // # Extant -- in circulation

#define CHAINMARKER (cast_to_oop<intptr_t>(-1))

// =====================> Quick functions

// The quick_* forms are special fast-path variants used to improve

// performance.  In the simplest case, a "quick_*" implementation could

// simply return false, in which case the caller will perform the necessary

// state transitions and call the slow-path form.

// The fast-path is designed to handle frequently arising cases in an efficient

// manner and is just a degenerate "optimistic" variant of the slow-path.

// returns true  -- to indicate the call was satisfied.

// returns false -- to indicate the call needs the services of the slow-path.

// A no-loitering ordinance is in effect for code in the quick_* family

// operators: safepoints or indefinite blocking (blocking that might span a

// safepoint) are forbidden. Generally the thread_state() is _in_Java upon

// entry.

//

// Consider: An interesting optimization is to have the JIT recognize the

// following common idiom:

//   synchronized (someobj) { .... ; notify(); }

// That is, we find a notify() or notifyAll() call that immediately precedes

// the monitorexit operation.  In that case the JIT could fuse the operations

// into a single notifyAndExit() runtime primitive.

bool ObjectSynchronizer::quick_notify(oopDesc * obj, Thread * self, bool all) {

  assert(!SafepointSynchronize::is_at_safepoint(), "invariant");

  assert(self->is_Java_thread(), "invariant");

  assert(((JavaThread *) self)->thread_state() == _thread_in_Java, "invariant");

  NoSafepointVerifier nsv;

  if (obj == NULL) return false;  // slow-path for invalid obj

  const markOop mark = obj->mark();

  if (mark->has_locker() && self->is_lock_owned((address)mark->locker())) {

    // Degenerate notify

    // stack-locked by caller so by definition the implied waitset is empty.

    return true;

  }

  if (mark->has_monitor()) {

    ObjectMonitor * const mon = mark->monitor();

    assert(oopDesc::equals((oop) mon->object(), obj), "invariant");

    if (mon->owner() != self) return false;  // slow-path for IMS exception

    if (mon->first_waiter() != NULL) {

      // We have one or more waiters. Since this is an inflated monitor

      // that we own, we can transfer one or more threads from the waitset

      // to the entrylist here and now, avoiding the slow-path.

      if (all) {

        DTRACE_MONITOR_PROBE(notifyAll, mon, obj, self);

      } else {

        DTRACE_MONITOR_PROBE(notify, mon, obj, self);

      }

      int tally = 0;

      do {

        mon->INotify(self);

        ++tally;

      } while (mon->first_waiter() != NULL && all);

      OM_PERFDATA_OP(Notifications, inc(tally));

    }

    return true;

  }

  // biased locking and any other IMS exception states take the slow-path

  return false;

}

// The LockNode emitted directly at the synchronization site would have

// been too big if it were to have included support for the cases of inflated

// recursive enter and exit, so they go here instead.

// Note that we can't safely call AsyncPrintJavaStack() from within

// quick_enter() as our thread state remains _in_Java.

bool ObjectSynchronizer::quick_enter(oop obj, Thread * Self,

                                     BasicLock * lock) {

  assert(!SafepointSynchronize::is_at_safepoint(), "invariant");

  assert(Self->is_Java_thread(), "invariant");

  assert(((JavaThread *) Self)->thread_state() == _thread_in_Java, "invariant");

  NoSafepointVerifier nsv;

  if (obj == NULL) return false;       // Need to throw NPE

  const markOop mark = obj->mark();

  if (mark->has_monitor()) {

    ObjectMonitor * const m = mark->monitor();

    assert(oopDesc::equals((oop) m->object(), obj), "invariant");

    Thread * const owner = (Thread *) m->_owner;

    // Lock contention and Transactional Lock Elision (TLE) diagnostics

    // and observability

    // Case: light contention possibly amenable to TLE

    // Case: TLE inimical operations such as nested/recursive synchronization

    if (owner == Self) {

      m->_recursions++;

      return true;

    }

    // This Java Monitor is inflated so obj's header will never be

    // displaced to this thread's BasicLock. Make the displaced header

    // non-NULL so this BasicLock is not seen as recursive nor as

    // being locked. We do this unconditionally so that this thread's

    // BasicLock cannot be mis-interpreted by any stack walkers. For

    // performance reasons, stack walkers generally first check for

    // Biased Locking in the object's header, the second check is for

    // stack-locking in the object's header, the third check is for

    // recursive stack-locking in the displaced header in the BasicLock,

    // and last are the inflated Java Monitor (ObjectMonitor) checks.

    lock->set_displaced_header(markOopDesc::unused_mark());

    if (owner == NULL && Atomic::replace_if_null(Self, &(m->_owner))) {

      assert(m->_recursions == 0, "invariant");

      assert(m->_owner == Self, "invariant");

      return true;

    }

  }

  // Note that we could inflate in quick_enter.

  // This is likely a useful optimization

  // Critically, in quick_enter() we must not:

  // -- perform bias revocation, or

  // -- block indefinitely, or

  // -- reach a safepoint

  return false;        // revert to slow-path

}

// -----------------------------------------------------------------------------

//  Fast Monitor Enter/Exit

// This the fast monitor enter. The interpreter and compiler use

// some assembly copies of this code. Make sure update those code

// if the following function is changed. The implementation is

// extremely sensitive to race condition. Be careful.

void ObjectSynchronizer::fast_enter(Handle obj, BasicLock* lock,

                                    bool attempt_rebias, TRAPS) {

  if (UseBiasedLocking) {

    if (!SafepointSynchronize::is_at_safepoint()) {

      BiasedLocking::Condition cond = BiasedLocking::revoke_and_rebias(obj, attempt_rebias, THREAD);

      if (cond == BiasedLocking::BIAS_REVOKED_AND_REBIASED) {

        return;

      }

    } else {

      assert(!attempt_rebias, "can not rebias toward VM thread");

      BiasedLocking::revoke_at_safepoint(obj);

    }

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  slow_enter(obj, lock, THREAD);

}

void ObjectSynchronizer::fast_exit(oop object, BasicLock* lock, TRAPS) {

  markOop mark = object->mark();

  // We cannot check for Biased Locking if we are racing an inflation.

  assert(mark == markOopDesc::INFLATING() ||

         !mark->has_bias_pattern(), "should not see bias pattern here");

  markOop dhw = lock->displaced_header();

  if (dhw == NULL) {

    // If the displaced header is NULL, then this exit matches up with

    // a recursive enter. No real work to do here except for diagnostics.

#ifndef PRODUCT

    if (mark != markOopDesc::INFLATING()) {

      // Only do diagnostics if we are not racing an inflation. Simply

      // exiting a recursive enter of a Java Monitor that is being

      // inflated is safe; see the has_monitor() comment below.

      assert(!mark->is_neutral(), "invariant");

      assert(!mark->has_locker() ||

             THREAD->is_lock_owned((address)mark->locker()), "invariant");

      if (mark->has_monitor()) {

        // The BasicLock's displaced_header is marked as a recursive

        // enter and we have an inflated Java Monitor (ObjectMonitor).

        // This is a special case where the Java Monitor was inflated

        // after this thread entered the stack-lock recursively. When a

        // Java Monitor is inflated, we cannot safely walk the Java

        // Monitor owner's stack and update the BasicLocks because a

        // Java Monitor can be asynchronously inflated by a thread that

        // does not own the Java Monitor.

        ObjectMonitor * m = mark->monitor();

        assert(((oop)(m->object()))->mark() == mark, "invariant");

        assert(m->is_entered(THREAD), "invariant");

      }

    }

#endif

    return;

  }

  if (mark == (markOop) lock) {

    // If the object is stack-locked by the current thread, try to

    // swing the displaced header from the BasicLock back to the mark.

    assert(dhw->is_neutral(), "invariant");

    if (object->cas_set_mark(dhw, mark) == mark) {

      return;

    }

  }

  // We have to take the slow-path of possible inflation and then exit.

  inflate(THREAD, object, inflate_cause_vm_internal)->exit(true, THREAD);

}

// -----------------------------------------------------------------------------

// Interpreter/Compiler Slow Case

// This routine is used to handle interpreter/compiler slow case

// We don't need to use fast path here, because it must have been

// failed in the interpreter/compiler code.

void ObjectSynchronizer::slow_enter(Handle obj, BasicLock* lock, TRAPS) {

  markOop mark = obj->mark();

  assert(!mark->has_bias_pattern(), "should not see bias pattern here");

  if (mark->is_neutral()) {

    // Anticipate successful CAS -- the ST of the displaced mark must

    // be visible <= the ST performed by the CAS.

    lock->set_displaced_header(mark);

    if (mark == obj()->cas_set_mark((markOop) lock, mark)) {

      return;

    }

    // Fall through to inflate() ...

  } else if (mark->has_locker() &&

             THREAD->is_lock_owned((address)mark->locker())) {

    assert(lock != mark->locker(), "must not re-lock the same lock");

    assert(lock != (BasicLock*)obj->mark(), "don't relock with same BasicLock");

    lock->set_displaced_header(NULL);

    return;

  }

  // The object header will never be displaced to this lock,

  // so it does not matter what the value is, except that it

  // must be non-zero to avoid looking like a re-entrant lock,

  // and must not look locked either.

  lock->set_displaced_header(markOopDesc::unused_mark());

  inflate(THREAD, obj(), inflate_cause_monitor_enter)->enter(THREAD);

}

// This routine is used to handle interpreter/compiler slow case

// We don't need to use fast path here, because it must have

// failed in the interpreter/compiler code. Simply use the heavy

// weight monitor should be ok, unless someone find otherwise.

void ObjectSynchronizer::slow_exit(oop object, BasicLock* lock, TRAPS) {

  fast_exit(object, lock, THREAD);

}

// -----------------------------------------------------------------------------

// Class Loader  support to workaround deadlocks on the class loader lock objects

// Also used by GC

// complete_exit()/reenter() are used to wait on a nested lock

// i.e. to give up an outer lock completely and then re-enter

// Used when holding nested locks - lock acquisition order: lock1 then lock2

//  1) complete_exit lock1 - saving recursion count

//  2) wait on lock2

//  3) when notified on lock2, unlock lock2

//  4) reenter lock1 with original recursion count

//  5) lock lock2

// NOTE: must use heavy weight monitor to handle complete_exit/reenter()

intptr_t ObjectSynchronizer::complete_exit(Handle obj, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);

  return monitor->complete_exit(THREAD);

}

// NOTE: must use heavy weight monitor to handle complete_exit/reenter()

void ObjectSynchronizer::reenter(Handle obj, intptr_t recursion, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_vm_internal);

  monitor->reenter(recursion, THREAD);

}

// -----------------------------------------------------------------------------

// JNI locks on java objects

// NOTE: must use heavy weight monitor to handle jni monitor enter

void ObjectSynchronizer::jni_enter(Handle obj, TRAPS) {

  // the current locking is from JNI instead of Java code

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  THREAD->set_current_pending_monitor_is_from_java(false);

  inflate(THREAD, obj(), inflate_cause_jni_enter)->enter(THREAD);

  THREAD->set_current_pending_monitor_is_from_java(true);

}

// NOTE: must use heavy weight monitor to handle jni monitor exit

void ObjectSynchronizer::jni_exit(oop obj, Thread* THREAD) {

  if (UseBiasedLocking) {

    Handle h_obj(THREAD, obj);

    BiasedLocking::revoke_and_rebias(h_obj, false, THREAD);

    obj = h_obj();

  }

  assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  ObjectMonitor* monitor = inflate(THREAD, obj, inflate_cause_jni_exit);

  // If this thread has locked the object, exit the monitor.  Note:  can't use

  // monitor->check(CHECK); must exit even if an exception is pending.

  if (monitor->check(THREAD)) {

    monitor->exit(true, THREAD);

  }

}

// -----------------------------------------------------------------------------

// Internal VM locks on java objects

// standard constructor, allows locking failures

ObjectLocker::ObjectLocker(Handle obj, Thread* thread, bool doLock) {

  _dolock = doLock;

  _thread = thread;

  debug_only(if (StrictSafepointChecks) _thread->check_for_valid_safepoint_state(false);)

  _obj = obj;

  if (_dolock) {

    ObjectSynchronizer::fast_enter(_obj, &_lock, false, _thread);

  }

}

ObjectLocker::~ObjectLocker() {

  if (_dolock) {

    ObjectSynchronizer::fast_exit(_obj(), &_lock, _thread);

  }

}

// -----------------------------------------------------------------------------

//  Wait/Notify/NotifyAll

// NOTE: must use heavy weight monitor to handle wait()

int ObjectSynchronizer::wait(Handle obj, jlong millis, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  if (millis < 0) {

    THROW_MSG_0(vmSymbols::java_lang_IllegalArgumentException(), "timeout value is negative");

  }

  ObjectMonitor* monitor = inflate(THREAD, obj(), inflate_cause_wait);

  DTRACE_MONITOR_WAIT_PROBE(monitor, obj(), THREAD, millis);

  monitor->wait(millis, true, THREAD);

  // This dummy call is in place to get around dtrace bug 6254741.  Once

  // that's fixed we can uncomment the following line, remove the call

  // and change this function back into a "void" func.

  // DTRACE_MONITOR_PROBE(waited, monitor, obj(), THREAD);

  return dtrace_waited_probe(monitor, obj, THREAD);

}

void ObjectSynchronizer::waitUninterruptibly(Handle obj, jlong millis, TRAPS) {

  if (UseBiasedLocking) {

    BiasedLocking::revoke_and_rebias(obj, false, THREAD);

    assert(!obj->mark()->has_bias_pattern(), "biases should be revoked by now");

  }

  if (millis < 0) {