# HG changeset patch
# User rehn
# Date 1524036351 -7200
# Node ID 69d7398038c54774d9395b6810e0cca335edc02c
# Parent  33dcb9c42f55e3c791ec3d2be3e81c9584b10e83
8195099: Concurrent safe-memory-reclamation mechanism
Summary: This implement a globalcounter with RCU semantics.
Reviewed-by: acorn, coleenp, dcubed, eosterlund, gziemski, mlarsson, kbarrett, dholmes

diff -r 33dcb9c42f55 -r 69d7398038c5 src/hotspot/share/runtime/thread.cpp
--- a/src/hotspot/share/runtime/thread.cpp	Tue Apr 17 23:27:41 2018 -0700
+++ b/src/hotspot/share/runtime/thread.cpp	Wed Apr 18 09:25:51 2018 +0200
@@ -246,6 +246,7 @@
   _threads_hazard_ptr = NULL;
   _nested_threads_hazard_ptr = NULL;
   _nested_threads_hazard_ptr_cnt = 0;
+  _rcu_counter = 0;
 
   // the handle mark links itself to last_handle_mark
   new HandleMark(this);
diff -r 33dcb9c42f55 -r 69d7398038c5 src/hotspot/share/runtime/thread.hpp
--- a/src/hotspot/share/runtime/thread.hpp	Tue Apr 17 23:27:41 2018 -0700
+++ b/src/hotspot/share/runtime/thread.hpp	Wed Apr 18 09:25:51 2018 +0200
@@ -305,6 +305,14 @@
   // claimed as a task.
   int _oops_do_parity;
 
+  // Support for GlobalCounter
+ private:
+  volatile uintx _rcu_counter;
+ public:
+  volatile uintx* get_rcu_counter() {
+    return &_rcu_counter;
+  }
+
  public:
   void set_last_handle_mark(HandleMark* mark)   { _last_handle_mark = mark; }
   HandleMark* last_handle_mark() const          { return _last_handle_mark; }
@@ -378,7 +386,7 @@
   void initialize_thread_current();
   void clear_thread_current(); // TLS cleanup needed before threads terminate
 
-  public:
+ public:
   // thread entry point
   virtual void run();
 
diff -r 33dcb9c42f55 -r 69d7398038c5 src/hotspot/share/utilities/globalCounter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/utilities/globalCounter.cpp	Wed Apr 18 09:25:51 2018 +0200
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "utilities/globalCounter.hpp"
+#include "runtime/orderAccess.inline.hpp"
+#include "runtime/thread.hpp"
+#include "runtime/threadSMR.inline.hpp"
+#include "runtime/vmThread.hpp"
+#include "utilities/spinYield.hpp"
+
+GlobalCounter::PaddedCounter GlobalCounter::_global_counter;
+
+class GlobalCounter::CounterThreadCheck : public ThreadClosure {
+ private:
+  uintx _gbl_cnt;
+ public:
+  CounterThreadCheck(uintx gbl_cnt) : _gbl_cnt(gbl_cnt) {}
+  void do_thread(Thread* thread) {
+    SpinYield yield;
+    // Loops on this thread until it has exited the critical read section.
+    while(true) {
+      uintx cnt = OrderAccess::load_acquire(thread->get_rcu_counter());
+      // This checks if the thread's counter is active. And if so is the counter
+      // for a pre-existing reader (belongs to this grace period). A pre-existing
+      // reader will have a lower counter than the global counter version for this
+      // generation. If the counter is larger than the global counter version this
+      //  is a new reader and we can continue.
+      if (((cnt & COUNTER_ACTIVE) != 0) && (cnt - _gbl_cnt) > (max_uintx / 2)) {
+        yield.wait();
+      } else {
+        break;
+      }
+    }
+  }
+};
+
+void GlobalCounter::write_synchronize() {
+  assert((*Thread::current()->get_rcu_counter() & COUNTER_ACTIVE) == 0x0, "must be outside a critcal section");
+  // Atomic::add must provide fence since we have storeload dependency.
+  volatile uintx gbl_cnt = Atomic::add((uintx)COUNTER_INCREMENT, &_global_counter._counter);
+  // Do all RCU threads.
+  CounterThreadCheck ctc(gbl_cnt);
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread *thread = jtiwh.next(); ) {
+    ctc.do_thread(thread);
+  }
+  ctc.do_thread(VMThread::vm_thread());
+}
diff -r 33dcb9c42f55 -r 69d7398038c5 src/hotspot/share/utilities/globalCounter.hpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/utilities/globalCounter.hpp	Wed Apr 18 09:25:51 2018 +0200
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_UTILITIES_GLOBAL_COUNTER_HPP
+#define SHARE_UTILITIES_GLOBAL_COUNTER_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+
+class Thread;
+
+// The GlobalCounter provides a synchronization mechanism between threads for
+// safe memory reclamation and other ABA problems. All readers must call
+// critical_section_begin before reading the volatile data and
+// critical_section_end afterwards. The write side must call write_synchronize
+// before reclaming the memory. The read-path only does an uncontented store
+// to a thread-local-storage and fence to stop any loads from floating up, thus
+// light weight and wait-free. The write-side is more heavy since it must check
+// all readers and wait until they have left the generation. (a system memory
+// barrier can be used on write-side to remove fence in read-side,
+// not implemented).
+class GlobalCounter : public AllStatic {
+ private:
+  // Since do not know what we will end up next to in BSS, we make sure the
+  // counter is on a seperate cacheline.
+  struct PaddedCounter {
+    DEFINE_PAD_MINUS_SIZE(0, DEFAULT_CACHE_LINE_SIZE/2, 0);
+    volatile uintx _counter;
+    DEFINE_PAD_MINUS_SIZE(1, DEFAULT_CACHE_LINE_SIZE/2, sizeof(volatile uintx));
+  };
+
+  // The global counter
+  static PaddedCounter _global_counter;
+
+  // Bit 0 is active bit.
+  static const uintx COUNTER_ACTIVE = 1;
+  // Thus we increase counter by 2.
+  static const uintx COUNTER_INCREMENT = 2;
+
+  // The per thread scanning closure.
+  class CounterThreadCheck;
+
+ public:
+  // Must be called before accessing the data. Only threads accessible lock-free
+  // can used this. Those included now are all Threads on SMR ThreadsList and
+  // the VMThread. Nesting is not yet supported.
+  static void critical_section_begin(Thread *thread);
+
+  // Must be called after finished accessing the data.
+  // Do not provide fence, allows load/stores moving into the critical section.
+  static void critical_section_end(Thread *thread);
+
+  // Make the data inaccessible to readers before calling. When this call
+  // returns it's safe to reclaim the data.
+  static void write_synchronize();
+
+  // A scoped object for a reads-side critical-section.
+  class CriticalSection;
+};
+
+#endif // include guard
diff -r 33dcb9c42f55 -r 69d7398038c5 src/hotspot/share/utilities/globalCounter.inline.hpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/utilities/globalCounter.inline.hpp	Wed Apr 18 09:25:51 2018 +0200
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_UTILITIES_GLOBAL_COUNTER_INLINE_HPP
+#define SHARE_UTILITIES_GLOBAL_COUNTER_INLINE_HPP
+
+#include "runtime/orderAccess.inline.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/globalCounter.hpp"
+
+inline void GlobalCounter::critical_section_begin(Thread *thread) {
+  assert(thread == Thread::current(), "must be current thread");
+  assert(thread->is_VM_thread() || thread->is_Java_thread(), "must be VMThread or JavaThread");
+  assert((*thread->get_rcu_counter() & COUNTER_ACTIVE) == 0x0, "nestled critical sections, not supported yet");
+  uintx gbl_cnt = OrderAccess::load_acquire(&_global_counter._counter);
+  OrderAccess::release_store_fence(thread->get_rcu_counter(), gbl_cnt | COUNTER_ACTIVE);
+}
+
+inline void GlobalCounter::critical_section_end(Thread *thread) {
+  assert(thread == Thread::current(), "must be current thread");
+  assert(thread->is_VM_thread() || thread->is_Java_thread(), "must be VMThread or JavaThread");
+  assert((*thread->get_rcu_counter() & COUNTER_ACTIVE) == COUNTER_ACTIVE, "must be in ctitical section");
+  // Mainly for debugging we set it to 'now'.
+  uintx gbl_cnt = OrderAccess::load_acquire(&_global_counter._counter);
+  OrderAccess::release_store(thread->get_rcu_counter(), gbl_cnt);
+}
+
+class GlobalCounter::CriticalSection {
+ private:
+  Thread* _thread;
+ public:
+  inline CriticalSection(Thread* thread) : _thread(thread) {
+    GlobalCounter::critical_section_begin(_thread);
+  }
+  inline  ~CriticalSection() {
+    GlobalCounter::critical_section_end(_thread);
+  }
+};
+
+#endif // include guard
diff -r 33dcb9c42f55 -r 69d7398038c5 test/hotspot/gtest/utilities/test_globalCounter.cpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/gtest/utilities/test_globalCounter.cpp	Wed Apr 18 09:25:51 2018 +0200
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/orderAccess.inline.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalCounter.hpp"
+#include "utilities/globalCounter.inline.hpp"
+#include "utilitiesHelper.inline.hpp"
+
+#define GOOD 1337
+#define BAD  4711
+
+struct TestData {
+  long test_value;
+};
+
+class RCUReaderThread : public JavaTestThread {
+public:
+  static volatile bool _exit;
+  volatile TestData** _test;
+  Semaphore* _wrt_start;
+  RCUReaderThread(Semaphore* post, volatile TestData** test, Semaphore* wrt_start)
+    : JavaTestThread(post), _test(test), _wrt_start(wrt_start) {};
+  virtual ~RCUReaderThread(){}
+  void main_run() {
+    _wrt_start->signal();
+    while (!_exit) {
+      GlobalCounter::critical_section_begin(this);
+      volatile TestData* test = OrderAccess::load_acquire(_test);
+      long value = OrderAccess::load_acquire(&test->test_value);
+      ASSERT_EQ(value, GOOD);
+      GlobalCounter::critical_section_end(this);
+      {
+        GlobalCounter::CriticalSection cs(this);
+        volatile TestData* test = OrderAccess::load_acquire(_test);
+        long value = OrderAccess::load_acquire(&test->test_value);
+        ASSERT_EQ(value, GOOD);
+      }
+    }
+  }
+};
+
+volatile bool RCUReaderThread::_exit = false;
+
+class RCUWriterThread : public JavaTestThread {
+public:
+  RCUWriterThread(Semaphore* post) : JavaTestThread(post) {
+  };
+  virtual ~RCUWriterThread(){}
+  void main_run() {
+    static const int NUMBER_OF_READERS = 4;
+    Semaphore post;
+    Semaphore wrt_start;
+    volatile TestData* test = NULL;
+
+    RCUReaderThread* reader1 = new RCUReaderThread(&post, &test, &wrt_start);
+    RCUReaderThread* reader2 = new RCUReaderThread(&post, &test, &wrt_start);
+    RCUReaderThread* reader3 = new RCUReaderThread(&post, &test, &wrt_start);
+    RCUReaderThread* reader4 = new RCUReaderThread(&post, &test, &wrt_start);
+
+    TestData* tmp = new TestData();
+    tmp->test_value = GOOD;
+    OrderAccess::release_store_fence(&test, tmp);
+
+    reader1->doit();
+    reader2->doit();
+    reader3->doit();
+    reader4->doit();
+
+    int nw = NUMBER_OF_READERS;
+    while (nw > 0) {
+      wrt_start.wait();
+      --nw;
+    }
+    jlong stop_ms = os::javaTimeMillis() + 1000; // 1 seconds max test time
+    for (int i = 0; i < 100000 && stop_ms > os::javaTimeMillis(); i++) {
+      volatile TestData* free_tmp = test;
+      tmp = new TestData();
+      tmp->test_value = GOOD;
+      OrderAccess::release_store(&test, tmp);
+      GlobalCounter::write_synchronize();
+      free_tmp->test_value = BAD;
+      delete free_tmp;
+    }
+    RCUReaderThread::_exit = true;
+    for (int i = 0; i < NUMBER_OF_READERS; i++) {
+      post.wait();
+    }
+  }
+};
+
+TEST_VM(GlobalCounter, critical_section) {
+  RCUReaderThread::_exit = false;
+  mt_test_doer<RCUWriterThread>();
+}
diff -r 33dcb9c42f55 -r 69d7398038c5 test/hotspot/gtest/utilities/utilitiesHelper.inline.hpp
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/gtest/utilities/utilitiesHelper.inline.hpp	Wed Apr 18 09:25:51 2018 +0200
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef GTEST_UTILITIES_HELPER_INLINE_HPP
+#define GTEST_UTILITIES_HELPER_INLINE_HPP
+
+#include "runtime/mutex.hpp"
+#include "runtime/semaphore.hpp"
+#include "runtime/thread.hpp"
+#include "runtime/vmThread.hpp"
+#include "runtime/vm_operations.hpp"
+#include "unittest.hpp"
+
+class VM_StopSafepoint : public VM_Operation {
+public:
+  Semaphore* _test_complete;
+  VM_StopSafepoint(Semaphore* wait_for) : _test_complete(wait_for) {}
+  VMOp_Type type() const          { return VMOp_Dummy; }
+  Mode evaluation_mode() const    { return _no_safepoint; }
+  bool is_cheap_allocated() const { return false; }
+  void doit()                     { _test_complete->wait(); }
+};
+
+// This class and thread keep the non-safepoint op running while we do our testing.
+class VMThreadBlocker : public JavaThread {
+public:
+  Semaphore* _unblock;
+  Semaphore* _done;
+  VMThreadBlocker(Semaphore* ub, Semaphore* done) : _unblock(ub), _done(done) {
+  }
+  virtual ~VMThreadBlocker() {}
+  void run() {
+    this->set_thread_state(_thread_in_vm);
+    {
+      MutexLocker ml(Threads_lock);
+      Threads::add(this);
+    }
+    VM_StopSafepoint ss(_unblock);
+    VMThread::execute(&ss);
+    _done->signal();
+    Threads::remove(this);
+    this->smr_delete();
+  }
+  void doit() {
+    if (os::create_thread(this, os::os_thread)) {
+      os::start_thread(this);
+    } else {
+      ASSERT_TRUE(false);
+    }
+  }
+};
+
+// For testing in a real JavaThread.
+class JavaTestThread : public JavaThread {
+public:
+  Semaphore* _post;
+  JavaTestThread(Semaphore* post)
+    : _post(post) {
+  }
+  virtual ~JavaTestThread() {}
+
+  void prerun() {
+    this->set_thread_state(_thread_in_vm);
+    {
+      MutexLocker ml(Threads_lock);
+      Threads::add(this);
+    }
+    {
+      MutexLockerEx ml(SR_lock(), Mutex::_no_safepoint_check_flag);
+    }
+  }
+
+  virtual void main_run() = 0;
+
+  void run() {
+    prerun();
+    main_run();
+    postrun();
+  }
+
+  void postrun() {
+    Threads::remove(this);
+    _post->signal();
+    this->smr_delete();
+  }
+
+  void doit() {
+    if (os::create_thread(this, os::os_thread)) {
+      os::start_thread(this);
+    } else {
+      ASSERT_TRUE(false);
+    }
+  }
+};
+
+template <typename FUNC>
+class SingleTestThread : public JavaTestThread {
+public:
+  FUNC& _f;
+  SingleTestThread(Semaphore* post, FUNC& f)
+    : JavaTestThread(post), _f(f) {
+  }
+
+  virtual ~SingleTestThread(){}
+
+  virtual void main_run() {
+    _f(this);
+  }
+};
+
+template <typename TESTFUNC>
+static void nomt_test_doer(TESTFUNC &f) {
+  Semaphore post, block_done, vmt_done;
+  VMThreadBlocker* blocker = new VMThreadBlocker(&block_done, &vmt_done);
+  blocker->doit();
+  SingleTestThread<TESTFUNC>* stt = new SingleTestThread<TESTFUNC>(&post, f);
+  stt->doit();
+  post.wait();
+  block_done.signal();
+  vmt_done.wait();
+}
+
+template <typename RUNNER>
+static void mt_test_doer() {
+  Semaphore post, block_done, vmt_done;
+  VMThreadBlocker* blocker = new VMThreadBlocker(&block_done, &vmt_done);
+  blocker->doit();
+  RUNNER* runner = new RUNNER(&post);
+  runner->doit();
+  post.wait();
+  block_done.signal();
+  vmt_done.wait();
+}
+
+#endif // include guard