/*

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "logging/logStream.hpp"

#include "memory/allocation.inline.hpp"

#include "runtime/thread.inline.hpp"

#include "runtime/threadSMR.inline.hpp"

#include "services/threadService.hpp"

#include "utilities/globalDefinitions.hpp"

#include "utilities/resourceHash.hpp"

Monitor*              ThreadsSMRSupport::_delete_lock =

                          new Monitor(Monitor::special, "Thread_SMR_delete_lock",

                                      false /* allow_vm_block */,

                                      Monitor::_safepoint_check_never);

// The '_cnt', '_max' and '_times" fields are enabled via

// -XX:+EnableThreadSMRStatistics:

// # of parallel threads in _delete_lock->wait().

// Impl note: Hard to imagine > 64K waiting threads so this could be 16-bit,

// but there is no nice 16-bit _FORMAT support.

uint                  ThreadsSMRSupport::_delete_lock_wait_cnt = 0;

// Max # of parallel threads in _delete_lock->wait().

// Impl note: See _delete_lock_wait_cnt note.

uint                  ThreadsSMRSupport::_delete_lock_wait_max = 0;

// Flag to indicate when an _delete_lock->notify() is needed.

// Impl note: See _delete_lock_wait_cnt note.

volatile uint         ThreadsSMRSupport::_delete_notify = 0;

// # of threads deleted over VM lifetime.

// Impl note: Atomically incremented over VM lifetime so use unsigned for more

// range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc

// isn't available everywhere (or is it?).

volatile uint         ThreadsSMRSupport::_deleted_thread_cnt = 0;

// Max time in millis to delete a thread.

// Impl note: 16-bit might be too small on an overloaded machine. Use

// unsigned since this is a time value. Set via Atomic::cmpxchg() in a

// loop for correctness.

volatile uint         ThreadsSMRSupport::_deleted_thread_time_max = 0;

// Cumulative time in millis to delete threads.

// Impl note: Atomically added to over VM lifetime so use unsigned for more

// range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc

// isn't available everywhere (or is it?).

volatile uint         ThreadsSMRSupport::_deleted_thread_times = 0;

ThreadsList* volatile ThreadsSMRSupport::_java_thread_list = new ThreadsList(0);

// # of ThreadsLists allocated over VM lifetime.

// Impl note: We allocate a new ThreadsList for every thread create and

// every thread delete so we need a bigger type than the

// _deleted_thread_cnt field.

uint64_t              ThreadsSMRSupport::_java_thread_list_alloc_cnt = 1;

// # of ThreadsLists freed over VM lifetime.

// Impl note: See _java_thread_list_alloc_cnt note.

uint64_t              ThreadsSMRSupport::_java_thread_list_free_cnt = 0;

// Max size ThreadsList allocated.

// Impl note: Max # of threads alive at one time should fit in unsigned 32-bit.

uint                  ThreadsSMRSupport::_java_thread_list_max = 0;

// Max # of nested ThreadsLists for a thread.

// Impl note: Hard to imagine > 64K nested ThreadsLists so this could be

// 16-bit, but there is no nice 16-bit _FORMAT support.

uint                  ThreadsSMRSupport::_nested_thread_list_max = 0;

// # of ThreadsListHandles deleted over VM lifetime.

// Impl note: Atomically incremented over VM lifetime so use unsigned for

// more range. There will be fewer ThreadsListHandles than threads so

// unsigned 32-bit should be fine.

volatile uint         ThreadsSMRSupport::_tlh_cnt = 0;

// Max time in millis to delete a ThreadsListHandle.

// Impl note: 16-bit might be too small on an overloaded machine. Use

// unsigned since this is a time value. Set via Atomic::cmpxchg() in a

// loop for correctness.

volatile uint         ThreadsSMRSupport::_tlh_time_max = 0;

// Cumulative time in millis to delete ThreadsListHandles.

// Impl note: Atomically added to over VM lifetime so use unsigned for more

// range. Unsigned 64-bit would be more future proof, but 64-bit atomic inc

// isn't available everywhere (or is it?).

volatile uint         ThreadsSMRSupport::_tlh_times = 0;

ThreadsList*          ThreadsSMRSupport::_to_delete_list = NULL;

// # of parallel ThreadsLists on the to-delete list.

// Impl note: Hard to imagine > 64K ThreadsLists needing to be deleted so

// this could be 16-bit, but there is no nice 16-bit _FORMAT support.

uint                  ThreadsSMRSupport::_to_delete_list_cnt = 0;

// Max # of parallel ThreadsLists on the to-delete list.

// Impl note: See _to_delete_list_cnt note.

uint                  ThreadsSMRSupport::_to_delete_list_max = 0;

// 'inline' functions first so the definitions are before first use:

inline void ThreadsSMRSupport::add_deleted_thread_times(uint add_value) {

  Atomic::add(add_value, &_deleted_thread_times);

}

inline void ThreadsSMRSupport::inc_deleted_thread_cnt() {

  Atomic::inc(&_deleted_thread_cnt);

}

inline void ThreadsSMRSupport::inc_java_thread_list_alloc_cnt() {

  _java_thread_list_alloc_cnt++;

}

inline void ThreadsSMRSupport::update_deleted_thread_time_max(uint new_value) {

  while (true) {

    uint cur_value = _deleted_thread_time_max;

    if (new_value <= cur_value) {

      // No need to update max value so we're done.

      break;

    }

    if (Atomic::cmpxchg(new_value, &_deleted_thread_time_max, cur_value) == cur_value) {

      // Updated max value so we're done. Otherwise try it all again.

      break;

    }

  }

}

inline void ThreadsSMRSupport::update_java_thread_list_max(uint new_value) {

  if (new_value > _java_thread_list_max) {

    _java_thread_list_max = new_value;

  }

}

inline ThreadsList* ThreadsSMRSupport::xchg_java_thread_list(ThreadsList* new_list) {

  return (ThreadsList*)Atomic::xchg(new_list, &_java_thread_list);

}

// Hash table of pointers found by a scan. Used for collecting hazard

// pointers (ThreadsList references). Also used for collecting JavaThreads

// that are indirectly referenced by hazard ptrs. An instance of this

// class only contains one type of pointer.

//

class ThreadScanHashtable : public CHeapObj<mtThread> {

 private:

  static bool ptr_equals(void * const& s1, void * const& s2) {

    return s1 == s2;

  }

  static unsigned int ptr_hash(void * const& s1) {

    // 2654435761 = 2^32 * Phi (golden ratio)

    return (unsigned int)(((uint32_t)(uintptr_t)s1) * 2654435761u);

  }

  int _table_size;

  // ResourceHashtable SIZE is specified at compile time so our

  // dynamic _table_size is unused for now; 1031 is the first prime

  // after 1024.

  typedef ResourceHashtable<void *, int, &ThreadScanHashtable::ptr_hash,

                            &ThreadScanHashtable::ptr_equals, 1031,

                            ResourceObj::C_HEAP, mtThread> PtrTable;

  PtrTable * _ptrs;

 public:

  // ResourceHashtable is passed to various functions and populated in

  // different places so we allocate it using C_HEAP to make it immune

  // from any ResourceMarks that happen to be in the code paths.

  ThreadScanHashtable(int table_size) : _table_size(table_size), _ptrs(new (ResourceObj::C_HEAP, mtThread) PtrTable()) {}

  ~ThreadScanHashtable() { delete _ptrs; }

  bool has_entry(void *pointer) {

    int *val_ptr = _ptrs->get(pointer);

    return val_ptr != NULL && *val_ptr == 1;

  }

  void add_entry(void *pointer) {

    _ptrs->put(pointer, 1);

  }

};

// Closure to gather JavaThreads indirectly referenced by hazard ptrs

// (ThreadsList references) into a hash table. This closure handles part 2

// of the dance - adding all the JavaThreads referenced by the hazard

// pointer (ThreadsList reference) to the hash table.

//

class AddThreadHazardPointerThreadClosure : public ThreadClosure {

 private:

  ThreadScanHashtable *_table;

 public:

  AddThreadHazardPointerThreadClosure(ThreadScanHashtable *table) : _table(table) {}

  virtual void do_thread(Thread *thread) {

    if (!_table->has_entry((void*)thread)) {

      // The same JavaThread might be on more than one ThreadsList or

      // more than one thread might be using the same ThreadsList. In

      // either case, we only need a single entry for a JavaThread.

      _table->add_entry((void*)thread);

    }

  }

};

// Closure to gather JavaThreads indirectly referenced by hazard ptrs

// (ThreadsList references) into a hash table. This closure handles part 1

// of the dance - hazard ptr chain walking and dispatch to another

// closure.

//

class ScanHazardPtrGatherProtectedThreadsClosure : public ThreadClosure {

 private:

  ThreadScanHashtable *_table;

 public:

  ScanHazardPtrGatherProtectedThreadsClosure(ThreadScanHashtable *table) : _table(table) {}

  virtual void do_thread(Thread *thread) {

    assert_locked_or_safepoint(Threads_lock);

    if (thread == NULL) return;

    // This code races with ThreadsSMRSupport::acquire_stable_list() which

    // is lock-free so we have to handle some special situations.

    //

    ThreadsList *current_list = NULL;

    while (true) {

      current_list = thread->get_threads_hazard_ptr();

      // No hazard ptr so nothing more to do.

      if (current_list == NULL) {

        assert(thread->get_nested_threads_hazard_ptr() == NULL,

               "cannot have a nested hazard ptr with a NULL regular hazard ptr");

        return;

      }

      // If the hazard ptr is verified as stable (since it is not tagged),

      // then it is safe to use.

      if (!Thread::is_hazard_ptr_tagged(current_list)) break;

      // The hazard ptr is tagged as not yet verified as being stable

      // so we are racing with acquire_stable_list(). This exchange

      // attempts to invalidate the hazard ptr. If we win the race,

      // then we can ignore this unstable hazard ptr and the other

      // thread will retry the attempt to publish a stable hazard ptr.

      // If we lose the race, then we retry our attempt to look at the

      // hazard ptr.

      if (thread->cmpxchg_threads_hazard_ptr(NULL, current_list) == current_list) return;

    }

    // The current JavaThread has a hazard ptr (ThreadsList reference)

    // which might be _java_thread_list or it might be an older

    // ThreadsList that has been removed but not freed. In either case,

    // the hazard ptr is protecting all the JavaThreads on that

    // ThreadsList.

    AddThreadHazardPointerThreadClosure add_cl(_table);

    current_list->threads_do(&add_cl);

    // Any NestedThreadsLists are also protecting JavaThreads so

    // gather those also; the ThreadsLists may be different.

    for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();

         node != NULL; node = node->next()) {

      node->t_list()->threads_do(&add_cl);

    }

  }

};

// Closure to gather hazard ptrs (ThreadsList references) into a hash table.

//

class ScanHazardPtrGatherThreadsListClosure : public ThreadClosure {

 private:

  ThreadScanHashtable *_table;

 public:

  ScanHazardPtrGatherThreadsListClosure(ThreadScanHashtable *table) : _table(table) {}

  virtual void do_thread(Thread* thread) {

    assert_locked_or_safepoint(Threads_lock);

    if (thread == NULL) return;

    ThreadsList *threads = thread->get_threads_hazard_ptr();

    if (threads == NULL) {

      assert(thread->get_nested_threads_hazard_ptr() == NULL,

             "cannot have a nested hazard ptr with a NULL regular hazard ptr");

      return;

    }

    // In this closure we always ignore the tag that might mark this

    // hazard ptr as not yet verified. If we happen to catch an

    // unverified hazard ptr that is subsequently discarded (not

    // published), then the only side effect is that we might keep a

    // to-be-deleted ThreadsList alive a little longer.

    threads = Thread::untag_hazard_ptr(threads);

    if (!_table->has_entry((void*)threads)) {

      _table->add_entry((void*)threads);

    }

    // Any NestedThreadsLists are also protecting JavaThreads so

    // gather those also; the ThreadsLists may be different.

    for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();

         node != NULL; node = node->next()) {

      threads = node->t_list();

      if (!_table->has_entry((void*)threads)) {

        _table->add_entry((void*)threads);

      }

    }

  }

};

// Closure to print JavaThreads that have a hazard ptr (ThreadsList

// reference) that contains an indirect reference to a specific JavaThread.

//

class ScanHazardPtrPrintMatchingThreadsClosure : public ThreadClosure {

 private:

  JavaThread *_thread;

 public:

  ScanHazardPtrPrintMatchingThreadsClosure(JavaThread *thread) : _thread(thread) {}

  virtual void do_thread(Thread *thread) {

    assert_locked_or_safepoint(Threads_lock);

    if (thread == NULL) return;

    ThreadsList *current_list = thread->get_threads_hazard_ptr();

    if (current_list == NULL) {

      assert(thread->get_nested_threads_hazard_ptr() == NULL,

             "cannot have a nested hazard ptr with a NULL regular hazard ptr");

      return;

    }

    // If the hazard ptr is unverified, then ignore it.

    if (Thread::is_hazard_ptr_tagged(current_list)) return;

    // The current JavaThread has a hazard ptr (ThreadsList reference)

    // which might be _java_thread_list or it might be an older

    // ThreadsList that has been removed but not freed. In either case,

    // the hazard ptr is protecting all the JavaThreads on that

    // ThreadsList, but we only care about matching a specific JavaThread.

    JavaThreadIterator jti(current_list);

    for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {

      if (p == _thread) {

        log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));

        break;

      }

    }

    // Any NestedThreadsLists are also protecting JavaThreads so

    // check those also; the ThreadsLists may be different.

    for (NestedThreadsList* node = thread->get_nested_threads_hazard_ptr();

         node != NULL; node = node->next()) {

      JavaThreadIterator jti(node->t_list());

      for (JavaThread *p = jti.first(); p != NULL; p = jti.next()) {

        if (p == _thread) {

          log_debug(thread, smr)("tid=" UINTX_FORMAT ": ThreadsSMRSupport::smr_delete: thread1=" INTPTR_FORMAT " has a nested hazard pointer for thread2=" INTPTR_FORMAT, os::current_thread_id(), p2i(thread), p2i(_thread));

          return;

        }

      }

    }

  }

};

// 'entries + 1' so we always have at least one entry.

ThreadsList::ThreadsList(int entries) : _length(entries), _threads(NEW_C_HEAP_ARRAY(JavaThread*, entries + 1, mtThread)), _next_list(NULL) {

  *(JavaThread**)(_threads + entries) = NULL;  // Make sure the extra entry is NULL.

}

ThreadsList::~ThreadsList() {

  FREE_C_HEAP_ARRAY(JavaThread*, _threads);

}

// Add a JavaThread to a ThreadsList. The returned ThreadsList is a

// new copy of the specified ThreadsList with the specified JavaThread

// appended to the end.

ThreadsList *ThreadsList::add_thread(ThreadsList *list, JavaThread *java_thread) {

  const uint index = list->_length;

  const uint new_length = index + 1;

  const uint head_length = index;

  ThreadsList *const new_list = new ThreadsList(new_length);

  if (head_length > 0) {

    Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);

  }

  *(JavaThread**)(new_list->_threads + index) = java_thread;

  return new_list;

}

int ThreadsList::find_index_of_JavaThread(JavaThread *target) {

  if (target == NULL) {

    return -1;

  }

  for (uint i = 0; i < length(); i++) {

    if (target == thread_at(i)) {

      return (int)i;

    }

  }

  return -1;

}

JavaThread* ThreadsList::find_JavaThread_from_java_tid(jlong java_tid) const {

  for (uint i = 0; i < length(); i++) {

    JavaThread* thread = thread_at(i);

    oop tobj = thread->threadObj();

    // Ignore the thread if it hasn't run yet, has exited

    // or is starting to exit.

    if (tobj != NULL && !thread->is_exiting() &&

        java_tid == java_lang_Thread::thread_id(tobj)) {

      // found a match

      return thread;

    }

  }

  return NULL;

}

bool ThreadsList::includes(const JavaThread * const p) const {

  if (p == NULL) {

    return false;

  }

  for (uint i = 0; i < length(); i++) {

    if (thread_at(i) == p) {

      return true;

    }

  }

  return false;

}

// Remove a JavaThread from a ThreadsList. The returned ThreadsList is a

// new copy of the specified ThreadsList with the specified JavaThread

// removed.

ThreadsList *ThreadsList::remove_thread(ThreadsList* list, JavaThread* java_thread) {

  assert(list->_length > 0, "sanity");

  uint i = (uint)list->find_index_of_JavaThread(java_thread);

  assert(i < list->_length, "did not find JavaThread on the list");

  const uint index = i;

  const uint new_length = list->_length - 1;

  const uint head_length = index;

  const uint tail_length = (new_length >= index) ? (new_length - index) : 0;

  ThreadsList *const new_list = new ThreadsList(new_length);

  if (head_length > 0) {

    Copy::disjoint_words((HeapWord*)list->_threads, (HeapWord*)new_list->_threads, head_length);

  }

  if (tail_length > 0) {

    Copy::disjoint_words((HeapWord*)list->_threads + index + 1, (HeapWord*)new_list->_threads + index, tail_length);

  }

  return new_list;

}

ThreadsListHandle::ThreadsListHandle(Thread *self) : _list(ThreadsSMRSupport::acquire_stable_list(self, /* is_ThreadsListSetter */ false)), _self(self) {

  assert(self == Thread::current(), "sanity check");

  if (EnableThreadSMRStatistics) {

    _timer.start();

  }

}

ThreadsListHandle::~ThreadsListHandle() {

  ThreadsSMRSupport::release_stable_list(_self);

  if (EnableThreadSMRStatistics) {

    _timer.stop();

    uint millis = (uint)_timer.milliseconds();

    ThreadsSMRSupport::update_tlh_stats(millis);

  }

}

// Convert an internal thread reference to a JavaThread found on the

// associated ThreadsList. This ThreadsListHandle "protects" the

// returned JavaThread *.

//

// If thread_oop_p is not NULL, then the caller wants to use the oop

// after this call so the oop is returned. On success, *jt_pp is set

// to the converted JavaThread * and true is returned. On error,

// returns false.

//

bool ThreadsListHandle::cv_internal_thread_to_JavaThread(jobject jthread,

                                                         JavaThread ** jt_pp,

                                                         oop * thread_oop_p) {