--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.cpp	Wed Jun 19 08:52:15 2019 -0400
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+
+bool ShenandoahConcurrentRoots::can_do_concurrent_roots() {
+  // Don't support traversal GC at this moment
+  return !ShenandoahHeap::heap()->is_concurrent_traversal_in_progress();
+}
+
+bool ShenandoahConcurrentRoots::should_do_concurrent_roots() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  bool stw_gc_in_progress = heap->is_full_gc_in_progress() ||
+                            heap->is_degenerated_gc_in_progress();
+  return can_do_concurrent_roots() &&
+         !stw_gc_in_progress;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentRoots.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2019, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTROOTS_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTROOTS_HPP
+
+#include "memory/allocation.hpp"
+
+class ShenandoahConcurrentRoots : public AllStatic {
+public:
+  // Can GC settings allow concurrent root processing
+  static bool can_do_concurrent_roots();
+  // If current GC cycle can process roots concurrently
+  static bool should_do_concurrent_roots();
+};
+
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTROOTS_HPP

--- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp	Wed Jun 19 08:52:15 2019 -0400
@@ -377,6 +377,9 @@
   // Complete marking under STW, and start evacuation
   heap->vmop_entry_final_mark();
 
+  // Evacuate concurrent roots
+  heap->entry_roots();
+
   // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
   // the space. This would be the last action if there is nothing to evacuate.
   heap->entry_cleanup();

--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp	Wed Jun 19 08:52:15 2019 -0400
@@ -38,6 +38,7 @@
 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp"
+#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahControlThread.hpp"
 #include "gc/shenandoah/shenandoahFreeSet.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
@@ -1071,9 +1072,11 @@
   DerivedPointerTable::clear();
 #endif
   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
-
   {
-    ShenandoahRootEvacuator rp(workers()->active_workers(), ShenandoahPhaseTimings::init_evac);
+    // Include concurrent roots if current cycle can not process those roots concurrently
+    ShenandoahRootEvacuator rp(workers()->active_workers(),
+                               ShenandoahPhaseTimings::init_evac,
+                               !ShenandoahConcurrentRoots::should_do_concurrent_roots());
     ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
     workers()->run_task(&roots_task);
   }
@@ -1517,7 +1520,11 @@
       }
 
       if (ShenandoahVerify) {
-        verifier()->verify_roots_no_forwarded();
+        if (ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+          verifier()->verify_roots_no_forwarded_except(ShenandoahRootVerifier::JNIHandleRoots);
+        } else {
+          verifier()->verify_roots_no_forwarded();
+        }
         verifier()->verify_during_evacuation();
       }
     } else {
@@ -1578,6 +1585,30 @@
   free_set()->recycle_trash();
 }
 
+class ShenandoahConcurrentRootsEvacUpdateTask : public AbstractGangTask {
+private:
+  ShenandoahJNIHandleRoots<true /*concurrent*/>         _jni_roots;
+
+public:
+  ShenandoahConcurrentRootsEvacUpdateTask() :
+    AbstractGangTask("Shenandoah Evacuate/Update Concurrent Roots Task") {
+  }
+
+  void work(uint worker_id) {
+    ShenandoahEvacOOMScope oom;
+    ShenandoahEvacuateUpdateRootsClosure cl;
+    _jni_roots.oops_do<ShenandoahEvacuateUpdateRootsClosure>(&cl);
+  }
+};
+
+void ShenandoahHeap::op_roots() {
+  if (is_evacuation_in_progress() &&
+      ShenandoahConcurrentRoots::should_do_concurrent_roots()) {
+    ShenandoahConcurrentRootsEvacUpdateTask task;
+    workers()->run_task(&task);
+  }
+}
+
 void ShenandoahHeap::op_reset() {
   reset_mark_bitmap();
 }
@@ -2559,6 +2590,22 @@
   try_inject_alloc_failure();
   op_updaterefs();
 }
+
+void ShenandoahHeap::entry_roots() {
+  ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_roots);
+
+  static const char* msg = "Concurrent roots processing";
+  GCTraceTime(Info, gc) time(msg, NULL, GCCause::_no_gc, true);
+  EventMark em("%s", msg);
+
+  ShenandoahWorkerScope scope(workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
+                              "concurrent root processing");
+
+  try_inject_alloc_failure();
+  op_roots();
+}
+
 void ShenandoahHeap::entry_cleanup() {
   ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_cleanup);
 

--- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -391,6 +391,7 @@
   void entry_reset();
   void entry_mark();
   void entry_preclean();
+  void entry_roots();
   void entry_cleanup();
   void entry_evac();
   void entry_updaterefs();
@@ -414,6 +415,7 @@
   void op_reset();
   void op_mark();
   void op_preclean();
+  void op_roots();
   void op_cleanup();
   void op_conc_evac();
   void op_stw_evac();

--- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -306,6 +306,7 @@
   f(conc_mark,                                       "Concurrent Marking")              \
   f(conc_termination,                                "  Termination")                   \
   f(conc_preclean,                                   "Concurrent Precleaning")          \
+  f(conc_roots,                                      "Concurrent Roots")                \
   f(conc_evac,                                       "Concurrent Evacuation")           \
   f(conc_update_refs,                                "Concurrent Update Refs")          \
   f(conc_cleanup,                                    "Concurrent Cleanup")              \

--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp	Wed Jun 19 08:52:15 2019 -0400
@@ -71,10 +71,6 @@
   _jvmti_root.oops_do(cl, worker_id);
 }
 
-ShenandoahJNIHandleRoots::ShenandoahJNIHandleRoots() :
-  ShenandoahSerialRoot(&JNIHandles::oops_do, ShenandoahPhaseTimings::JNIRoots) {
-}
-
 ShenandoahThreadRoots::ShenandoahThreadRoots(bool is_par) : _is_par(is_par) {
   Threads::change_thread_claim_token();
 }
@@ -148,21 +144,22 @@
   _heap->phase_timings()->record_workers_end(_phase);
 }
 
-ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase) :
+ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots) :
   ShenandoahRootProcessor(phase),
   _thread_roots(n_workers > 1),
-  _weak_roots(n_workers) {
+  _weak_roots(n_workers),
+  _include_concurrent_roots(include_concurrent_roots) {
 }
 
 void ShenandoahRootEvacuator::roots_do(uint worker_id, OopClosure* oops) {
   MarkingCodeBlobClosure blobsCl(oops, CodeBlobToOopClosure::FixRelocations);
   CLDToOopClosure clds(oops, ClassLoaderData::_claim_strong);
-  CLDToOopClosure* weak_clds = ShenandoahHeap::heap()->unload_classes() ? NULL : &clds;
-
   AlwaysTrueClosure always_true;
 
   _serial_roots.oops_do(oops, worker_id);
-  _jni_roots.oops_do(oops, worker_id);
+  if (_include_concurrent_roots) {
+    _jni_roots.oops_do<OopClosure>(oops, worker_id);
+  }
 
   _thread_roots.oops_do(oops, NULL, worker_id);
   _cld_roots.clds_do(&clds, &clds, worker_id);

--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -61,9 +61,15 @@
   void oops_do(OopClosure* cl, uint worker_id);
 };
 
-class ShenandoahJNIHandleRoots : public ShenandoahSerialRoot {
+template <bool CONCURRENT>
+class ShenandoahJNIHandleRoots {
+private:
+  OopStorage::ParState<CONCURRENT, false /* is_const */> _itr;
 public:
   ShenandoahJNIHandleRoots();
+
+  template <typename T>
+  void oops_do(T* cl, uint worker_id = 0);
 };
 
 class ShenandoahThreadRoots {
@@ -129,11 +135,11 @@
 template <typename ITR>
 class ShenandoahRootScanner : public ShenandoahRootProcessor {
 private:
-  ShenandoahSerialRoots          _serial_roots;
-  ShenandoahJNIHandleRoots       _jni_roots;
-  ShenandoahClassLoaderDataRoots _cld_roots;
-  ShenandoahThreadRoots          _thread_roots;
-  ShenandoahCodeCacheRoots<ITR>  _code_roots;
+  ShenandoahSerialRoots                           _serial_roots;
+  ShenandoahJNIHandleRoots<false /*concurrent*/>  _jni_roots;
+  ShenandoahClassLoaderDataRoots                  _cld_roots;
+  ShenandoahThreadRoots                           _thread_roots;
+  ShenandoahCodeCacheRoots<ITR>                   _code_roots;
 public:
   ShenandoahRootScanner(uint n_workers, ShenandoahPhaseTimings::Phase phase);
 
@@ -157,16 +163,17 @@
 // Evacuate all roots at a safepoint
 class ShenandoahRootEvacuator : public ShenandoahRootProcessor {
 private:
-  ShenandoahSerialRoots          _serial_roots;
-  ShenandoahJNIHandleRoots       _jni_roots;
-  ShenandoahClassLoaderDataRoots _cld_roots;
-  ShenandoahThreadRoots          _thread_roots;
-  ShenandoahWeakRoots            _weak_roots;
-  ShenandoahStringDedupRoots     _dedup_roots;
+  ShenandoahSerialRoots                                     _serial_roots;
+  ShenandoahJNIHandleRoots<false /*concurrent*/>            _jni_roots;
+  ShenandoahClassLoaderDataRoots                            _cld_roots;
+  ShenandoahThreadRoots                                     _thread_roots;
+  ShenandoahWeakRoots                                       _weak_roots;
+  ShenandoahStringDedupRoots                                _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
+  bool                                                      _include_concurrent_roots;
 
 public:
-  ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase);
+  ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool include_concurrent_roots);
 
   void roots_do(uint worker_id, OopClosure* oops);
 };
@@ -174,14 +181,14 @@
 // Update all roots at a safepoint
 class ShenandoahRootUpdater : public ShenandoahRootProcessor {
 private:
-  ShenandoahSerialRoots          _serial_roots;
-  ShenandoahJNIHandleRoots       _jni_roots;
-  ShenandoahClassLoaderDataRoots _cld_roots;
-  ShenandoahThreadRoots          _thread_roots;
-  ShenandoahWeakRoots            _weak_roots;
-  ShenandoahStringDedupRoots     _dedup_roots;
+  ShenandoahSerialRoots                                     _serial_roots;
+  ShenandoahJNIHandleRoots<false /*concurrent*/>            _jni_roots;
+  ShenandoahClassLoaderDataRoots                            _cld_roots;
+  ShenandoahThreadRoots                                     _thread_roots;
+  ShenandoahWeakRoots                                       _weak_roots;
+  ShenandoahStringDedupRoots                                _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahCsetCodeRootsIterator> _code_roots;
-  const bool                     _update_code_cache;
+  const bool                                                _update_code_cache;
 
 public:
   ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase, bool update_code_cache);
@@ -193,12 +200,12 @@
 // Adjuster all roots at a safepoint during full gc
 class ShenandoahRootAdjuster : public ShenandoahRootProcessor {
 private:
-  ShenandoahSerialRoots          _serial_roots;
-  ShenandoahJNIHandleRoots       _jni_roots;
-  ShenandoahClassLoaderDataRoots _cld_roots;
-  ShenandoahThreadRoots          _thread_roots;
-  ShenandoahWeakRoots            _weak_roots;
-  ShenandoahStringDedupRoots     _dedup_roots;
+  ShenandoahSerialRoots                                    _serial_roots;
+  ShenandoahJNIHandleRoots<false /*concurrent*/>           _jni_roots;
+  ShenandoahClassLoaderDataRoots                           _cld_roots;
+  ShenandoahThreadRoots                                    _thread_roots;
+  ShenandoahWeakRoots                                      _weak_roots;
+  ShenandoahStringDedupRoots                               _dedup_roots;
   ShenandoahCodeCacheRoots<ShenandoahAllCodeRootsIterator> _code_roots;
 
 public:

--- a/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.inline.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -24,12 +24,31 @@
 #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_INLINE_HPP
 #define SHARE_GC_SHENANDOAH_SHENANDOAHROOTPROCESSOR_INLINE_HPP
 
+#include "classfile/classLoaderDataGraph.hpp"
+#include "gc/shared/oopStorageParState.inline.hpp"
 #include "gc/shenandoah/shenandoahHeuristics.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.hpp"
 #include "gc/shenandoah/shenandoahTimingTracker.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "memory/resourceArea.hpp"
 
+template <bool CONCURRENT>
+ShenandoahJNIHandleRoots<CONCURRENT>::ShenandoahJNIHandleRoots() :
+  _itr(JNIHandles::global_handles()) {
+}
+
+template <bool CONCURRENT>
+template <typename T>
+void ShenandoahJNIHandleRoots<CONCURRENT>::oops_do(T* cl, uint worker_id) {
+  if (CONCURRENT) {
+    _itr.oops_do(cl);
+  } else {
+    ShenandoahWorkerTimings* worker_times = ShenandoahHeap::heap()->phase_timings()->worker_times();
+    ShenandoahWorkerTimingsTracker timer(worker_times, ShenandoahPhaseTimings::JNIRoots, worker_id);
+    _itr.oops_do(cl);
+  }
+}
+
 template <typename IsAlive, typename KeepAlive>
 void ShenandoahWeakRoots::oops_do(IsAlive* is_alive, KeepAlive* keep_alive, uint worker_id) {
   _task.work<IsAlive, KeepAlive>(worker_id, is_alive, keep_alive);

--- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp	Wed Jun 19 08:52:15 2019 -0400
@@ -30,6 +30,7 @@
 uint ShenandoahWorkerPolicy::_prev_par_marking     = 0;
 uint ShenandoahWorkerPolicy::_prev_conc_marking    = 0;
 uint ShenandoahWorkerPolicy::_prev_conc_evac       = 0;
+uint ShenandoahWorkerPolicy::_prev_conc_root_proc  = 0;
 uint ShenandoahWorkerPolicy::_prev_fullgc          = 0;
 uint ShenandoahWorkerPolicy::_prev_degengc         = 0;
 uint ShenandoahWorkerPolicy::_prev_stw_traversal   = 0;
@@ -63,6 +64,16 @@
   return _prev_par_marking;
 }
 
+// Calculate workers for concurrent root processing
+uint ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing() {
+  uint active_workers = (_prev_conc_root_proc == 0) ? ConcGCThreads : _prev_conc_root_proc;
+  _prev_conc_root_proc =
+    WorkerPolicy::calc_active_conc_workers(ConcGCThreads,
+                                           active_workers,
+                                           Threads::number_of_non_daemon_threads());
+  return _prev_conc_root_proc;
+}
+
 // Calculate workers for concurrent evacuation (concurrent GC)
 uint ShenandoahWorkerPolicy::calc_workers_for_conc_evac() {
   uint active_workers = (_prev_conc_evac == 0) ? ConcGCThreads : _prev_conc_evac;

--- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp	Wed Jun 19 12:17:22 2019 +0100
+++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp	Wed Jun 19 08:52:15 2019 -0400
@@ -30,6 +30,7 @@
 private:
   static uint _prev_par_marking;
   static uint _prev_conc_marking;
+  static uint _prev_conc_root_proc;
   static uint _prev_conc_evac;
   static uint _prev_fullgc;
   static uint _prev_degengc;
@@ -50,6 +51,9 @@
   // Calculate the number of workers for final marking
   static uint calc_workers_for_final_marking();
 
+  // Calculate workers for concurrent root processing
+  static uint calc_workers_for_conc_root_processing();
+
   // Calculate workers for concurrent evacuation (concurrent GC)
   static uint calc_workers_for_conc_evac();