--- a/src/hotspot/share/code/nmethod.cpp	Wed May 01 12:41:26 2019 -0400
+++ b/src/hotspot/share/code/nmethod.cpp	Wed May 01 12:31:29 2019 -0700
@@ -64,7 +64,7 @@
 #include "utilities/resourceHash.hpp"
 #include "utilities/xmlstream.hpp"
 #if INCLUDE_JVMCI
-#include "jvmci/jvmciJavaClasses.hpp"
+#include "jvmci/jvmciRuntime.hpp"
 #endif
 
 #ifdef DTRACE_ENABLED
@@ -112,6 +112,10 @@
   int dependencies_size;
   int handler_table_size;
   int nul_chk_table_size;
+#if INCLUDE_JVMCI
+  int speculations_size;
+  int jvmci_data_size;
+#endif
   int oops_size;
   int metadata_size;
 
@@ -129,6 +133,10 @@
     dependencies_size   += nm->dependencies_size();
     handler_table_size  += nm->handler_table_size();
     nul_chk_table_size  += nm->nul_chk_table_size();
+#if INCLUDE_JVMCI
+    speculations_size   += nm->speculations_size();
+    jvmci_data_size     += nm->jvmci_data_size();
+#endif
   }
   void print_nmethod_stats(const char* name) {
     if (nmethod_count == 0)  return;
@@ -146,6 +154,10 @@
     if (dependencies_size != 0)   tty->print_cr(" dependencies   = %d", dependencies_size);
     if (handler_table_size != 0)  tty->print_cr(" handler table  = %d", handler_table_size);
     if (nul_chk_table_size != 0)  tty->print_cr(" nul chk table  = %d", nul_chk_table_size);
+#if INCLUDE_JVMCI
+    if (speculations_size != 0)   tty->print_cr(" speculations   = %d", speculations_size);
+    if (jvmci_data_size != 0)     tty->print_cr(" JVMCI data     = %d", jvmci_data_size);
+#endif
   }
 };
 
@@ -426,11 +438,6 @@
 #if INCLUDE_RTM_OPT
   _rtm_state               = NoRTM;
 #endif
-#if INCLUDE_JVMCI
-  _jvmci_installed_code   = NULL;
-  _speculation_log        = NULL;
-  _jvmci_installed_code_triggers_invalidation = false;
-#endif
 }
 
 nmethod* nmethod::new_native_nmethod(const methodHandle& method,
@@ -483,8 +490,11 @@
   AbstractCompiler* compiler,
   int comp_level
 #if INCLUDE_JVMCI
-  , jweak installed_code,
-  jweak speculationLog
+  , char* speculations,
+  int speculations_len,
+  int nmethod_mirror_index,
+  const char* nmethod_mirror_name,
+  FailedSpeculation** failed_speculations
 #endif
 )
 {
@@ -493,12 +503,19 @@
   // create nmethod
   nmethod* nm = NULL;
   { MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+#if INCLUDE_JVMCI
+    int jvmci_data_size = !compiler->is_jvmci() ? 0 : JVMCINMethodData::compute_size(nmethod_mirror_name);
+#endif
     int nmethod_size =
       CodeBlob::allocation_size(code_buffer, sizeof(nmethod))
       + adjust_pcs_size(debug_info->pcs_size())
       + align_up((int)dependencies->size_in_bytes(), oopSize)
       + align_up(handler_table->size_in_bytes()    , oopSize)
       + align_up(nul_chk_table->size_in_bytes()    , oopSize)
+#if INCLUDE_JVMCI
+      + align_up(speculations_len                  , oopSize)
+      + align_up(jvmci_data_size                   , oopSize)
+#endif
       + align_up(debug_info->data_size()           , oopSize);
 
     nm = new (nmethod_size, comp_level)
@@ -510,12 +527,19 @@
             compiler,
             comp_level
 #if INCLUDE_JVMCI
-            , installed_code,
-            speculationLog
+            , speculations,
+            speculations_len,
+            jvmci_data_size
 #endif
             );
 
     if (nm != NULL) {
+#if INCLUDE_JVMCI
+      if (compiler->is_jvmci()) {
+        // Initialize the JVMCINMethodData object inlined into nm
+        nm->jvmci_nmethod_data()->initialize(nmethod_mirror_index, nmethod_mirror_name, failed_speculations);
+      }
+#endif
       // To make dependency checking during class loading fast, record
       // the nmethod dependencies in the classes it is dependent on.
       // This allows the dependency checking code to simply walk the
@@ -591,7 +615,13 @@
     _dependencies_offset     = _scopes_pcs_offset;
     _handler_table_offset    = _dependencies_offset;
     _nul_chk_table_offset    = _handler_table_offset;
+#if INCLUDE_JVMCI
+    _speculations_offset     = _nul_chk_table_offset;
+    _jvmci_data_offset       = _speculations_offset;
+    _nmethod_end_offset      = _jvmci_data_offset;
+#else
     _nmethod_end_offset      = _nul_chk_table_offset;
+#endif
     _compile_id              = compile_id;
     _comp_level              = CompLevel_none;
     _entry_point             = code_begin()          + offsets->value(CodeOffsets::Entry);
@@ -667,8 +697,9 @@
   AbstractCompiler* compiler,
   int comp_level
 #if INCLUDE_JVMCI
-  , jweak installed_code,
-  jweak speculation_log
+  , char* speculations,
+  int speculations_len,
+  int jvmci_data_size
 #endif
   )
   : CompiledMethod(method, "nmethod", type, nmethod_size, sizeof(nmethod), code_buffer, offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false),
@@ -697,15 +728,6 @@
     set_ctable_begin(header_begin() + _consts_offset);
 
 #if INCLUDE_JVMCI
-    _jvmci_installed_code = installed_code;
-    _speculation_log = speculation_log;
-    oop obj = JNIHandles::resolve(installed_code);
-    if (obj == NULL || (obj->is_a(HotSpotNmethod::klass()) && HotSpotNmethod::isDefault(obj))) {
-      _jvmci_installed_code_triggers_invalidation = false;
-    } else {
-      _jvmci_installed_code_triggers_invalidation = true;
-    }
-
     if (compiler->is_jvmci()) {
       // JVMCI might not produce any stub sections
       if (offsets->value(CodeOffsets::Exceptions) != -1) {
@@ -735,10 +757,10 @@
       _deopt_mh_handler_begin  = (address) this + _stub_offset          + offsets->value(CodeOffsets::DeoptMH);
     } else {
       _deopt_mh_handler_begin  = NULL;
+    }
 #if INCLUDE_JVMCI
     }
 #endif
-    }
     if (offsets->value(CodeOffsets::UnwindHandler) != -1) {
       _unwind_handler_offset = code_offset()         + offsets->value(CodeOffsets::UnwindHandler);
     } else {
@@ -753,7 +775,13 @@
     _dependencies_offset     = _scopes_pcs_offset    + adjust_pcs_size(debug_info->pcs_size());
     _handler_table_offset    = _dependencies_offset  + align_up((int)dependencies->size_in_bytes (), oopSize);
     _nul_chk_table_offset    = _handler_table_offset + align_up(handler_table->size_in_bytes(), oopSize);
+#if INCLUDE_JVMCI
+    _speculations_offset     = _nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize);
+    _jvmci_data_offset       = _speculations_offset  + align_up(speculations_len, oopSize);
+    _nmethod_end_offset      = _jvmci_data_offset    + align_up(jvmci_data_size, oopSize);
+#else
     _nmethod_end_offset      = _nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize);
+#endif
     _entry_point             = code_begin()          + offsets->value(CodeOffsets::Entry);
     _verified_entry_point    = code_begin()          + offsets->value(CodeOffsets::Verified_Entry);
     _osr_entry_point         = code_begin()          + offsets->value(CodeOffsets::OSR_Entry);
@@ -779,6 +807,13 @@
     handler_table->copy_to(this);
     nul_chk_table->copy_to(this);
 
+#if INCLUDE_JVMCI
+    // Copy speculations to nmethod
+    if (speculations_size() != 0) {
+      memcpy(speculations_begin(), speculations, speculations_len);
+    }
+#endif
+
     // we use the information of entry points to find out if a method is
     // static or non static
     assert(compiler->is_c2() || compiler->is_jvmci() ||
@@ -798,13 +833,14 @@
     log->print(" level='%d'", comp_level());
   }
 #if INCLUDE_JVMCI
-    char buffer[O_BUFLEN];
-    char* jvmci_name = jvmci_installed_code_name(buffer, O_BUFLEN);
+  if (jvmci_nmethod_data() != NULL) {
+    const char* jvmci_name = jvmci_nmethod_data()->name();
     if (jvmci_name != NULL) {
-      log->print(" jvmci_installed_code_name='");
+      log->print(" jvmci_mirror_name='");
       log->text("%s", jvmci_name);
       log->print("'");
     }
+  }
 #endif
 }
 
@@ -1115,13 +1151,6 @@
   // Log the unloading.
   log_state_change();
 
-#if INCLUDE_JVMCI
-  // The method can only be unloaded after the pointer to the installed code
-  // Java wrapper is no longer alive. Here we need to clear out this weak
-  // reference to the dead object.
-  maybe_invalidate_installed_code();
-#endif
-
   // The Method* is gone at this point
   assert(_method == NULL, "Tautology");
 
@@ -1134,6 +1163,15 @@
   // concurrent nmethod unloading. Therefore, there is no need for
   // acquire on the loader side.
   OrderAccess::release_store(&_state, (signed char)unloaded);
+
+#if INCLUDE_JVMCI
+  // Clear the link between this nmethod and a HotSpotNmethod mirror
+  JVMCINMethodData* nmethod_data = jvmci_nmethod_data();
+  if (nmethod_data != NULL) {
+    nmethod_data->invalidate_nmethod_mirror(this);
+    nmethod_data->clear_nmethod_mirror(this);
+  }
+#endif
 }
 
 void nmethod::invalidate_osr_method() {
@@ -1265,13 +1303,18 @@
     // Log the transition once
     log_state_change();
 
-    // Invalidate while holding the patching lock
-    JVMCI_ONLY(maybe_invalidate_installed_code());
-
     // Remove nmethod from method.
     unlink_from_method(false /* already owns Patching_lock */);
   } // leave critical region under Patching_lock
 
+#if INCLUDE_JVMCI
+  // Invalidate can't occur while holding the Patching lock
+  JVMCINMethodData* nmethod_data = jvmci_nmethod_data();
+  if (nmethod_data != NULL) {
+    nmethod_data->invalidate_nmethod_mirror(this);
+  }
+#endif
+
 #ifdef ASSERT
   if (is_osr_method() && method() != NULL) {
     // Make sure osr nmethod is invalidated, i.e. not on the list
@@ -1297,6 +1340,14 @@
       flush_dependencies(/*delete_immediately*/true);
     }
 
+#if INCLUDE_JVMCI
+    // Now that the nmethod has been unregistered, it's
+    // safe to clear the HotSpotNmethod mirror oop.
+    if (nmethod_data != NULL) {
+      nmethod_data->clear_nmethod_mirror(this);
+    }
+#endif
+
     // Clear ICStubs to prevent back patching stubs of zombie or flushed
     // nmethods during the next safepoint (see ICStub::finalize), as well
     // as to free up CompiledICHolder resources.
@@ -1324,7 +1375,7 @@
     assert(state == not_entrant, "other cases may need to be handled differently");
   }
 
-  if (TraceCreateZombies) {
+  if (TraceCreateZombies && state == zombie) {
     ResourceMark m;
     tty->print_cr("nmethod <" INTPTR_FORMAT "> %s code made %s", p2i(this), this->method() ? this->method()->name_and_sig_as_C_string() : "null", (state == not_entrant) ? "not entrant" : "zombie");
   }
@@ -1362,11 +1413,6 @@
     ec = next;
   }
 
-#if INCLUDE_JVMCI
-  assert(_jvmci_installed_code == NULL, "should have been nulled out when transitioned to zombie");
-  assert(_speculation_log == NULL, "should have been nulled out when transitioned to zombie");
-#endif
-
   Universe::heap()->flush_nmethod(this);
 
   CodeBlob::flush();
@@ -1660,17 +1706,6 @@
   if (is_unloading()) {
     make_unloaded();
   } else {
-#if INCLUDE_JVMCI
-    if (_jvmci_installed_code != NULL) {
-      if (JNIHandles::is_global_weak_cleared(_jvmci_installed_code)) {
-        if (_jvmci_installed_code_triggers_invalidation) {
-          make_not_entrant();
-        }
-        clear_jvmci_installed_code();
-      }
-    }
-#endif
-
     guarantee(unload_nmethod_caches(unloading_occurred),
               "Should not need transition stubs");
   }
@@ -2066,7 +2101,7 @@
   if (cm->is_aot()) return;  // FIXME: Revisit once _lock_count is added to aot_method
   nmethod* nm = cm->as_nmethod();
   Atomic::inc(&nm->_lock_count);
-  assert(zombie_ok || !nm->is_zombie(), "cannot lock a zombie method");
+  assert(zombie_ok || !nm->is_zombie(), "cannot lock a zombie method: %p", nm);
 }
 
 void nmethodLocker::unlock_nmethod(CompiledMethod* cm) {
@@ -2275,6 +2310,16 @@
                                               p2i(nul_chk_table_begin()),
                                               p2i(nul_chk_table_end()),
                                               nul_chk_table_size());
+#if INCLUDE_JVMCI
+  if (speculations_size () > 0) tty->print_cr(" speculations   [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
+                                              p2i(speculations_begin()),
+                                              p2i(speculations_end()),
+                                              speculations_size());
+  if (jvmci_data_size   () > 0) tty->print_cr(" JVMCI data     [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
+                                              p2i(jvmci_data_begin()),
+                                              p2i(jvmci_data_end()),
+                                              jvmci_data_size());
+#endif
 }
 
 #ifndef PRODUCT
@@ -2857,115 +2902,18 @@
 #endif // !PRODUCT
 
 #if INCLUDE_JVMCI
-void nmethod::clear_jvmci_installed_code() {
-  assert_locked_or_safepoint(Patching_lock);
-  if (_jvmci_installed_code != NULL) {
-    JNIHandles::destroy_weak_global(_jvmci_installed_code);
-    _jvmci_installed_code = NULL;
-  }
-}
-
-void nmethod::clear_speculation_log() {
-  assert_locked_or_safepoint(Patching_lock);
-  if (_speculation_log != NULL) {
-    JNIHandles::destroy_weak_global(_speculation_log);
-    _speculation_log = NULL;
-  }
-}
-
-void nmethod::maybe_invalidate_installed_code() {
-  if (!is_compiled_by_jvmci()) {
-    return;
-  }
-
-  assert(Patching_lock->is_locked() ||
-         SafepointSynchronize::is_at_safepoint(), "should be performed under a lock for consistency");
-  oop installed_code = JNIHandles::resolve(_jvmci_installed_code);
-  if (installed_code != NULL) {
-    // Update the values in the InstalledCode instance if it still refers to this nmethod
-    nmethod* nm = (nmethod*)InstalledCode::address(installed_code);
-    if (nm == this) {
-      if (!is_alive() || is_unloading()) {
-        // Break the link between nmethod and InstalledCode such that the nmethod
-        // can subsequently be flushed safely.  The link must be maintained while
-        // the method could have live activations since invalidateInstalledCode
-        // might want to invalidate all existing activations.
-        InstalledCode::set_address(installed_code, 0);
-        InstalledCode::set_entryPoint(installed_code, 0);
-      } else if (is_not_entrant()) {
-        // Remove the entry point so any invocation will fail but keep
-        // the address link around that so that existing activations can
-        // be invalidated.
-        InstalledCode::set_entryPoint(installed_code, 0);
-      }
-    }
-  }
-  if (!is_alive() || is_unloading()) {
-    // Clear these out after the nmethod has been unregistered and any
-    // updates to the InstalledCode instance have been performed.
-    clear_jvmci_installed_code();
-    clear_speculation_log();
+void nmethod::update_speculation(JavaThread* thread) {
+  jlong speculation = thread->pending_failed_speculation();
+  if (speculation != 0) {
+    guarantee(jvmci_nmethod_data() != NULL, "failed speculation in nmethod without failed speculation list");
+    jvmci_nmethod_data()->add_failed_speculation(this, speculation);
+    thread->set_pending_failed_speculation(0);
   }
 }
 
-void nmethod::invalidate_installed_code(Handle installedCode, TRAPS) {
-  if (installedCode() == NULL) {
-    THROW(vmSymbols::java_lang_NullPointerException());
-  }
-  jlong nativeMethod = InstalledCode::address(installedCode);
-  nmethod* nm = (nmethod*)nativeMethod;
-  if (nm == NULL) {
-    // Nothing to do
-    return;
-  }
-
-  nmethodLocker nml(nm);
-#ifdef ASSERT
-  {
-    MutexLocker pl(Patching_lock, Mutex::_no_safepoint_check_flag);
-    // This relationship can only be checked safely under a lock
-    assert(!nm->is_alive() || nm->is_unloading() || nm->jvmci_installed_code() == installedCode(), "sanity check");
-  }
-#endif
-
-  if (nm->is_alive()) {
-    // Invalidating the InstalledCode means we want the nmethod
-    // to be deoptimized.
-    nm->mark_for_deoptimization();
-    VM_Deoptimize op;
-    VMThread::execute(&op);
-  }
-
-  // Multiple threads could reach this point so we now need to
-  // lock and re-check the link to the nmethod so that only one
-  // thread clears it.
-  MutexLocker pl(Patching_lock, Mutex::_no_safepoint_check_flag);
-  if (InstalledCode::address(installedCode) == nativeMethod) {
-      InstalledCode::set_address(installedCode, 0);
-  }
-}
-
-oop nmethod::jvmci_installed_code() {
-  return JNIHandles::resolve(_jvmci_installed_code);
-}
-
-oop nmethod::speculation_log() {
-  return JNIHandles::resolve(_speculation_log);
-}
-
-char* nmethod::jvmci_installed_code_name(char* buf, size_t buflen) const {
-  if (!this->is_compiled_by_jvmci()) {
-    return NULL;
-  }
-  oop installed_code = JNIHandles::resolve(_jvmci_installed_code);
-  if (installed_code != NULL) {
-    oop installed_code_name = NULL;
-    if (installed_code->is_a(InstalledCode::klass())) {
-      installed_code_name = InstalledCode::name(installed_code);
-    }
-    if (installed_code_name != NULL) {
-      return java_lang_String::as_utf8_string(installed_code_name, buf, (int)buflen);
-    }
+const char* nmethod::jvmci_name() {
+  if (jvmci_nmethod_data() != NULL) {
+    return jvmci_nmethod_data()->name();
   }
   return NULL;
 }