--- a/src/hotspot/share/memory/iterator.inline.hpp	Mon Jun 25 12:44:52 2018 +0200
+++ b/src/hotspot/share/memory/iterator.inline.hpp	Sat May 26 06:59:49 2018 +0200
@@ -38,21 +38,21 @@
 #include "oops/typeArrayKlass.inline.hpp"
 #include "utilities/debug.hpp"
 
-inline void MetadataAwareOopClosure::do_cld_nv(ClassLoaderData* cld) {
+inline void MetadataVisitingOopIterateClosure::do_cld(ClassLoaderData* cld) {
   bool claim = true;  // Must claim the class loader data before processing.
   cld->oops_do(this, claim);
 }
 
-inline void MetadataAwareOopClosure::do_klass_nv(Klass* k) {
+inline void MetadataVisitingOopIterateClosure::do_klass(Klass* k) {
   ClassLoaderData* cld = k->class_loader_data();
-  do_cld_nv(cld);
+  MetadataVisitingOopIterateClosure::do_cld(cld);
 }
 
 #ifdef ASSERT
 // This verification is applied to all visited oops.
 // The closures can turn is off by overriding should_verify_oops().
 template <typename T>
-void ExtendedOopClosure::verify(T* p) {
+void OopIterateClosure::verify(T* p) {
   if (should_verify_oops()) {
     T heap_oop = RawAccess<>::oop_load(p);
     if (!CompressedOops::is_null(heap_oop)) {
@@ -65,54 +65,360 @@
 #endif
 
 // Implementation of the non-virtual do_oop dispatch.
+//
+// The same implementation is used for do_metadata, do_klass, and do_cld.
+//
+// Preconditions:
+//  - Base has a pure virtual do_oop
+//  - Only one of the classes in the inheritance chain from OopClosureType to
+//    Base implements do_oop.
+//
+// Given the preconditions:
+//  - If &OopClosureType::do_oop is resolved to &Base::do_oop, then there is no
+//    implementation of do_oop between Base and OopClosureType. However, there
+//    must be one implementation in one of the subclasses of OopClosureType.
+//    In this case we take the virtual call.
+//
+//  - Conversely, if &OopClosureType::do_oop is not resolved to &Base::do_oop,
+//    then we've found the one and only concrete implementation. In this case we
+//    take a non-virtual call.
+//
+// Because of this it's clear when we should call the virtual call and
+//   when the non-virtual call should be made.
+//
+// The way we find if &OopClosureType::do_oop is resolved to &Base::do_oop is to
+//   check if the resulting type of the class of a member-function pointer to
+//   &OopClosureType::do_oop is equal to the type of the class of a
+//   &Base::do_oop member-function pointer. Template parameter deduction is used
+//   to find these types, and then the IsSame trait is used to check if they are
+//   equal. Finally, SFINAE is used to select the appropriate implementation.
+//
+// Template parameters:
+//   T              - narrowOop or oop
+//   Receiver       - the resolved type of the class of the
+//                    &OopClosureType::do_oop member-function pointer. That is,
+//                    the klass with the do_oop member function.
+//   Base           - klass with the pure virtual do_oop member function.
+//   OopClosureType - The dynamic closure type
+//
+// Parameters:
+//   closure - The closure to call
+//   p       - The oop (or narrowOop) field to pass to the closure
 
-template <class OopClosureType, typename T>
-inline void Devirtualizer<true>::do_oop(OopClosureType* closure, T* p) {
-  debug_only(closure->verify(p));
-  closure->do_oop_nv(p);
-}
-template <class OopClosureType>
-inline void Devirtualizer<true>::do_klass(OopClosureType* closure, Klass* k) {
-  closure->do_klass_nv(k);
-}
-template <class OopClosureType>
-void Devirtualizer<true>::do_cld(OopClosureType* closure, ClassLoaderData* cld) {
-  closure->do_cld_nv(cld);
-}
-template <class OopClosureType>
-inline bool Devirtualizer<true>::do_metadata(OopClosureType* closure) {
-  // Make sure the non-virtual and the virtual versions match.
-  assert(closure->do_metadata_nv() == closure->do_metadata(), "Inconsistency in do_metadata");
-  return closure->do_metadata_nv();
+template <typename T, typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<IsSame<Receiver, Base>::value, void>::type
+call_do_oop(void (Receiver::*)(T*), void (Base::*)(T*), OopClosureType* closure, T* p) {
+  closure->do_oop(p);
 }
 
-// Implementation of the virtual do_oop dispatch.
+template <typename T, typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<!IsSame<Receiver, Base>::value, void>::type
+call_do_oop(void (Receiver::*)(T*), void (Base::*)(T*), OopClosureType* closure, T* p) {
+  // Sanity check
+  STATIC_ASSERT((!IsSame<OopClosureType, OopIterateClosure>::value));
+  closure->OopClosureType::do_oop(p);
+}
 
-template <class OopClosureType, typename T>
-void Devirtualizer<false>::do_oop(OopClosureType* closure, T* p) {
-  debug_only(closure->verify(p));
-  closure->do_oop(p);
+template <typename OopClosureType, typename T>
+inline void Devirtualizer::do_oop_no_verify(OopClosureType* closure, T* p) {
+  call_do_oop<T>(&OopClosureType::do_oop, &OopClosure::do_oop, closure, p);
 }
-template <class OopClosureType>
-void Devirtualizer<false>::do_klass(OopClosureType* closure, Klass* k) {
-  closure->do_klass(k);
+
+template <typename OopClosureType, typename T>
+inline void Devirtualizer::do_oop(OopClosureType* closure, T* p) {
+  debug_only(closure->verify(p));
+
+  do_oop_no_verify(closure, p);
 }
-template <class OopClosureType>
-void Devirtualizer<false>::do_cld(OopClosureType* closure, ClassLoaderData* cld) {
-  closure->do_cld(cld);
-}
-template <class OopClosureType>
-bool Devirtualizer<false>::do_metadata(OopClosureType* closure) {
+
+// Implementation of the non-virtual do_metadata dispatch.
+
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<IsSame<Receiver, Base>::value, bool>::type
+call_do_metadata(bool (Receiver::*)(), bool (Base::*)(), OopClosureType* closure) {
   return closure->do_metadata();
 }
 
-// The list of all "specializable" oop_oop_iterate function definitions.
-#define ALL_KLASS_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix)                  \
-  ALL_INSTANCE_KLASS_OOP_OOP_ITERATE_DEFN(             OopClosureType, nv_suffix)  \
-  ALL_INSTANCE_REF_KLASS_OOP_OOP_ITERATE_DEFN(         OopClosureType, nv_suffix)  \
-  ALL_INSTANCE_MIRROR_KLASS_OOP_OOP_ITERATE_DEFN(      OopClosureType, nv_suffix)  \
-  ALL_INSTANCE_CLASS_LOADER_KLASS_OOP_OOP_ITERATE_DEFN(OopClosureType, nv_suffix)  \
-  ALL_OBJ_ARRAY_KLASS_OOP_OOP_ITERATE_DEFN(            OopClosureType, nv_suffix)  \
-  ALL_TYPE_ARRAY_KLASS_OOP_OOP_ITERATE_DEFN(           OopClosureType, nv_suffix)
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<!IsSame<Receiver, Base>::value, bool>::type
+call_do_metadata(bool (Receiver::*)(), bool (Base::*)(), OopClosureType* closure) {
+  return closure->OopClosureType::do_metadata();
+}
+
+template <typename OopClosureType>
+inline bool Devirtualizer::do_metadata(OopClosureType* closure) {
+  return call_do_metadata(&OopClosureType::do_metadata, &OopIterateClosure::do_metadata, closure);
+}
+
+// Implementation of the non-virtual do_klass dispatch.
+
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<IsSame<Receiver, Base>::value, void>::type
+call_do_klass(void (Receiver::*)(Klass*), void (Base::*)(Klass*), OopClosureType* closure, Klass* k) {
+  closure->do_klass(k);
+}
+
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<!IsSame<Receiver, Base>::value, void>::type
+call_do_klass(void (Receiver::*)(Klass*), void (Base::*)(Klass*), OopClosureType* closure, Klass* k) {
+  closure->OopClosureType::do_klass(k);
+}
+
+template <typename OopClosureType>
+inline void Devirtualizer::do_klass(OopClosureType* closure, Klass* k) {
+  call_do_klass(&OopClosureType::do_klass, &OopIterateClosure::do_klass, closure, k);
+}
+
+// Implementation of the non-virtual do_cld dispatch.
+
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<IsSame<Receiver, Base>::value, void>::type
+call_do_cld(void (Receiver::*)(ClassLoaderData*), void (Base::*)(ClassLoaderData*), OopClosureType* closure, ClassLoaderData* cld) {
+  closure->do_cld(cld);
+}
+
+template <typename Receiver, typename Base, typename OopClosureType>
+static typename EnableIf<!IsSame<Receiver, Base>::value, void>::type
+call_do_cld(void (Receiver::*)(ClassLoaderData*), void (Base::*)(ClassLoaderData*), OopClosureType* closure, ClassLoaderData* cld) {
+  closure->OopClosureType::do_cld(cld);
+}
+
+template <typename OopClosureType>
+void Devirtualizer::do_cld(OopClosureType* closure, ClassLoaderData* cld) {
+  call_do_cld(&OopClosureType::do_cld, &OopIterateClosure::do_cld, closure, cld);
+}
+
+// Dispatch table implementation for *Klass::oop_oop_iterate
+//
+// It allows for a single call to do a multi-dispatch to an optimized version
+//   of oop_oop_iterate that statically know all these types:
+//   - OopClosureType    : static type give at call site
+//   - Klass*            : dynamic to static type through Klass::id() -> table index
+//   - UseCompressedOops : dynamic to static value determined once
+//
+// when users call obj->oop_iterate(&cl).
+//
+// oopDesc::oop_iterate() calls OopOopIterateDispatch::function(klass)(cl, obj, klass),
+//   which dispatches to an optimized version of
+//   [Instance, ObjArry, etc]Klass::oop_oop_iterate(oop, OopClosureType)
+//
+// OopClosureType :
+//   If OopClosureType has an implementation of do_oop (and do_metadata et.al.),
+//   then the static type of OopClosureType will be used to allow inlining of
+//   do_oop (even though do_oop is virtual). Otherwise, a virtual call will be
+//   used when calling do_oop.
+//
+// Klass* :
+//   A table mapping from *Klass::ID to function is setup. This happens once
+//   when the program starts, when the static _table instance is initialized for
+//   the OopOopIterateDispatch specialized with the OopClosureType.
+//
+// UseCompressedOops :
+//   Initially the table is populated with an init function, and not the actual
+//   oop_oop_iterate function. This is done, so that the first time we dispatch
+//   through the init function we check what the value of UseCompressedOops
+//   became, and use that to determine if we should install an optimized
+//   narrowOop version or optimized oop version of oop_oop_iterate. The appropriate
+//   oop_oop_iterate function replaces the init function in the table, and
+//   succeeding calls will jump directly to oop_oop_iterate.
+
+
+template <typename OopClosureType>
+class OopOopIterateDispatch : public AllStatic {
+private:
+  class Table {
+  private:
+    template <typename KlassType, typename T>
+    static void oop_oop_iterate(OopClosureType* cl, oop obj, Klass* k) {
+      ((KlassType*)k)->KlassType::template oop_oop_iterate<T>(obj, cl);
+    }
+
+    template <typename KlassType>
+    static void init(OopClosureType* cl, oop obj, Klass* k) {
+      OopOopIterateDispatch<OopClosureType>::_table.set_resolve_function_and_execute<KlassType>(cl, obj, k);
+    }
+
+    template <typename KlassType>
+    void set_init_function() {
+      _function[KlassType::ID] = &init<KlassType>;
+    }
+
+    template <typename KlassType>
+    void set_resolve_function() {
+      // Size requirement to prevent word tearing
+      // when functions pointers are updated.
+      STATIC_ASSERT(sizeof(_function[0]) == sizeof(void*));
+      if (UseCompressedOops) {
+        _function[KlassType::ID] = &oop_oop_iterate<KlassType, narrowOop>;
+      } else {
+        _function[KlassType::ID] = &oop_oop_iterate<KlassType, oop>;
+      }
+    }
+
+    template <typename KlassType>
+    void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k) {
+      set_resolve_function<KlassType>();
+      _function[KlassType::ID](cl, obj, k);
+    }
+
+  public:
+    void (*_function[KLASS_ID_COUNT])(OopClosureType*, oop, Klass*);
+
+    Table(){
+      set_init_function<InstanceKlass>();
+      set_init_function<InstanceRefKlass>();
+      set_init_function<InstanceMirrorKlass>();
+      set_init_function<InstanceClassLoaderKlass>();
+      set_init_function<ObjArrayKlass>();
+      set_init_function<TypeArrayKlass>();
+    }
+  };
+
+  static Table _table;
+public:
+
+  static void (*function(Klass* klass))(OopClosureType*, oop, Klass*) {
+    return _table._function[klass->id()];
+  }
+};
+
+template <typename OopClosureType>
+typename OopOopIterateDispatch<OopClosureType>::Table OopOopIterateDispatch<OopClosureType>::_table;
+
+
+template <typename OopClosureType>
+class OopOopIterateBoundedDispatch {
+private:
+  class Table {
+  private:
+    template <typename KlassType, typename T>
+    static void oop_oop_iterate_bounded(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) {
+      ((KlassType*)k)->KlassType::template oop_oop_iterate_bounded<T>(obj, cl, mr);
+    }
+
+    template <typename KlassType>
+    static void init(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) {
+      OopOopIterateBoundedDispatch<OopClosureType>::_table.set_resolve_function_and_execute<KlassType>(cl, obj, k, mr);
+    }
+
+    template <typename KlassType>
+    void set_init_function() {
+      _function[KlassType::ID] = &init<KlassType>;
+    }
+
+    template <typename KlassType>
+    void set_resolve_function() {
+      if (UseCompressedOops) {
+        _function[KlassType::ID] = &oop_oop_iterate_bounded<KlassType, narrowOop>;
+      } else {
+        _function[KlassType::ID] = &oop_oop_iterate_bounded<KlassType, oop>;
+      }
+    }
+
+    template <typename KlassType>
+    void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k, MemRegion mr) {
+      set_resolve_function<KlassType>();
+      _function[KlassType::ID](cl, obj, k, mr);
+    }
+
+  public:
+    void (*_function[KLASS_ID_COUNT])(OopClosureType*, oop, Klass*, MemRegion);
+
+    Table(){
+      set_init_function<InstanceKlass>();
+      set_init_function<InstanceRefKlass>();
+      set_init_function<InstanceMirrorKlass>();
+      set_init_function<InstanceClassLoaderKlass>();
+      set_init_function<ObjArrayKlass>();
+      set_init_function<TypeArrayKlass>();
+    }
+  };
+
+  static Table _table;
+public:
+
+  static void (*function(Klass* klass))(OopClosureType*, oop, Klass*, MemRegion) {
+    return _table._function[klass->id()];
+  }
+};
+
+template <typename OopClosureType>
+typename OopOopIterateBoundedDispatch<OopClosureType>::Table OopOopIterateBoundedDispatch<OopClosureType>::_table;
+
+
+template <typename OopClosureType>
+class OopOopIterateBackwardsDispatch {
+private:
+  class Table {
+  private:
+    template <typename KlassType, typename T>
+    static void oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* k) {
+      ((KlassType*)k)->KlassType::template oop_oop_iterate_reverse<T>(obj, cl);
+    }
+
+    template <typename KlassType>
+    static void init(OopClosureType* cl, oop obj, Klass* k) {
+      OopOopIterateBackwardsDispatch<OopClosureType>::_table.set_resolve_function_and_execute<KlassType>(cl, obj, k);
+    }
+
+    template <typename KlassType>
+    void set_init_function() {
+      _function[KlassType::ID] = &init<KlassType>;
+    }
+
+    template <typename KlassType>
+    void set_resolve_function() {
+      if (UseCompressedOops) {
+        _function[KlassType::ID] = &oop_oop_iterate_backwards<KlassType, narrowOop>;
+      } else {
+        _function[KlassType::ID] = &oop_oop_iterate_backwards<KlassType, oop>;
+      }
+    }
+
+    template <typename KlassType>
+    void set_resolve_function_and_execute(OopClosureType* cl, oop obj, Klass* k) {
+      set_resolve_function<KlassType>();
+      _function[KlassType::ID](cl, obj, k);
+    }
+
+  public:
+    void (*_function[KLASS_ID_COUNT])(OopClosureType*, oop, Klass*);
+
+    Table(){
+      set_init_function<InstanceKlass>();
+      set_init_function<InstanceRefKlass>();
+      set_init_function<InstanceMirrorKlass>();
+      set_init_function<InstanceClassLoaderKlass>();
+      set_init_function<ObjArrayKlass>();
+      set_init_function<TypeArrayKlass>();
+    }
+  };
+
+  static Table _table;
+public:
+
+  static void (*function(Klass* klass))(OopClosureType*, oop, Klass*) {
+    return _table._function[klass->id()];
+  }
+};
+
+template <typename OopClosureType>
+typename OopOopIterateBackwardsDispatch<OopClosureType>::Table OopOopIterateBackwardsDispatch<OopClosureType>::_table;
+
+
+template <typename OopClosureType>
+void OopIteratorClosureDispatch::oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass) {
+  OopOopIterateDispatch<OopClosureType>::function(klass)(cl, obj, klass);
+}
+
+template <typename OopClosureType>
+void OopIteratorClosureDispatch::oop_oop_iterate(OopClosureType* cl, oop obj, Klass* klass, MemRegion mr) {
+  OopOopIterateBoundedDispatch<OopClosureType>::function(klass)(cl, obj, klass, mr);
+}
+
+template <typename OopClosureType>
+void OopIteratorClosureDispatch::oop_oop_iterate_backwards(OopClosureType* cl, oop obj, Klass* klass) {
+  OopOopIterateBackwardsDispatch<OopClosureType>::function(klass)(cl, obj, klass);
+}
 
 #endif // SHARE_VM_MEMORY_ITERATOR_INLINE_HPP