--- a/make/autoconf/hotspot.m4	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/autoconf/hotspot.m4	Mon Dec 10 15:47:44 2018 +0100
@@ -25,7 +25,7 @@
 
 # All valid JVM features, regardless of platform
 VALID_JVM_FEATURES="compiler1 compiler2 zero minimal dtrace jvmti jvmci \
-    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc zgc nmt cds \
+    graal vm-structs jni-check services management cmsgc epsilongc g1gc parallelgc serialgc shenandoahgc zgc nmt cds \
     static-build link-time-opt aot jfr"
 
 # Deprecated JVM features (these are ignored, but with a warning)
@@ -325,6 +325,15 @@
     fi
   fi
 
+  # Only enable Shenandoah on supported arches
+  AC_MSG_CHECKING([if shenandoah can be built])
+  if test "x$OPENJDK_TARGET_CPU_ARCH" = "xx86" || test "x$OPENJDK_TARGET_CPU" = "xaarch64" ; then
+    AC_MSG_RESULT([yes])
+  else
+    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES shenandoahgc"
+    AC_MSG_RESULT([no, platform not supported])
+  fi
+
   # Only enable ZGC on supported platforms
   AC_MSG_CHECKING([if zgc can be built])
   if test "x$OPENJDK_TARGET_OS" = "xlinux" && test "x$OPENJDK_TARGET_CPU" = "xx86_64"; then
@@ -336,7 +345,7 @@
 
   # Disable unsupported GCs for Zero
   if HOTSPOT_CHECK_JVM_VARIANT(zero); then
-    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc zgc"
+    DISABLED_JVM_FEATURES="$DISABLED_JVM_FEATURES epsilongc g1gc zgc shenandoahgc"
   fi
 
   # Turn on additional features based on other parts of configure
@@ -470,7 +479,7 @@
   fi
 
   # All variants but minimal (and custom) get these features
-  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc jni-check jvmti management nmt services vm-structs zgc"
+  NON_MINIMAL_FEATURES="$NON_MINIMAL_FEATURES cmsgc g1gc parallelgc serialgc epsilongc shenandoahgc jni-check jvmti management nmt services vm-structs zgc"
 
   # Disable CDS on AIX.
   if test "x$OPENJDK_TARGET_OS" = "xaix"; then

--- a/make/hotspot/gensrc/GensrcAdlc.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -136,6 +136,12 @@
       $d/os_cpu/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH)/$(HOTSPOT_TARGET_OS)_$(HOTSPOT_TARGET_CPU_ARCH).ad \
     )))
 
+  ifeq ($(call check-jvm-feature, shenandoahgc), true)
+    AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/shenandoah/shenandoah_$(HOTSPOT_TARGET_CPU).ad \
+      )))
+  endif
+
   SINGLE_AD_SRCFILE := $(ADLC_SUPPORT_DIR)/all-ad-src.ad
 
   INSERT_FILENAME_AWK_SCRIPT := \

--- a/make/hotspot/lib/JvmFeatures.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/lib/JvmFeatures.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -166,6 +166,13 @@
   JVM_EXCLUDE_PATTERNS += gc/z
 endif
 
+ifneq ($(call check-jvm-feature, shenandoahgc), true)
+  JVM_CFLAGS_FEATURES += -DINCLUDE_SHENANDOAHGC=0
+  JVM_EXCLUDE_PATTERNS += gc/shenandoah
+else
+  JVM_CFLAGS_FEATURES += -DSUPPORT_BARRIER_ON_PRIMITIVES -DSUPPORT_NOT_TO_SPACE_INVARIANT
+endif
+
 ifneq ($(call check-jvm-feature, jfr), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_JFR=0
   JVM_EXCLUDE_PATTERNS += jfr

--- a/make/hotspot/lib/JvmOverrideFiles.gmk	Mon Dec 10 17:34:49 2018 +0300
+++ b/make/hotspot/lib/JvmOverrideFiles.gmk	Mon Dec 10 15:47:44 2018 +0100
@@ -36,6 +36,11 @@
   BUILD_LIBJVM_assembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_cardTableBarrierSetAssembler_x86.cpp_CXXFLAGS := -Wno-maybe-uninitialized
   BUILD_LIBJVM_interp_masm_x86.cpp_CXXFLAGS := -Wno-uninitialized
+  ifeq ($(DEBUG_LEVEL), release)
+    # Need extra inlining to collapse all marking code into the hot marking loop
+    BUILD_LIBJVM_shenandoahConcurrentMark.cpp_CXXFLAGS := --param inline-unit-growth=1000
+    BUILD_LIBJVM_shenandoahTraversalGC.cpp_CXXFLAGS := --param inline-unit-growth=1000
+  endif
 endif
 
 LIBJVM_FDLIBM_COPY_OPT_FLAG := $(CXX_O_FLAG_NONE)

--- a/src/hotspot/cpu/aarch64/aarch64.ad	Mon Dec 10 17:34:49 2018 +0300
+++ b/src/hotspot/cpu/aarch64/aarch64.ad	Mon Dec 10 15:47:44 2018 +0100
@@ -1,6 +1,6 @@
 //
 // Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2014, Red Hat Inc. All rights reserved.
+// Copyright (c) 2014, 2018, Red Hat, Inc. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -1272,6 +1272,8 @@
     case Op_CompareAndSwapL:
     case Op_CompareAndSwapP:
     case Op_CompareAndSwapN:
+    case Op_ShenandoahCompareAndSwapP:
+    case Op_ShenandoahCompareAndSwapN:
     case Op_CompareAndSwapB:
     case Op_CompareAndSwapS:
     case Op_GetAndSetI:
@@ -1293,6 +1295,10 @@
     case Op_WeakCompareAndSwapL:
     case Op_WeakCompareAndSwapP:
     case Op_WeakCompareAndSwapN:
+    case Op_ShenandoahWeakCompareAndSwapP:
+    case Op_ShenandoahWeakCompareAndSwapN:
+    case Op_ShenandoahCompareAndExchangeP:
+    case Op_ShenandoahCompareAndExchangeN:
       return maybe_volatile;
     default:
       return false;

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,671 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register addr, Register count, RegSet saved_regs) {
+  if (is_oop) {
+    bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+    if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
+      __ push(saved_regs, sp);
+      if (count == c_rarg0) {
+        if (addr == c_rarg1) {
+          // exactly backwards!!
+          __ mov(rscratch1, c_rarg0);
+          __ mov(c_rarg0, c_rarg1);
+          __ mov(c_rarg1, rscratch1);
+        } else {
+          __ mov(c_rarg1, count);
+          __ mov(c_rarg0, addr);
+        }
+      } else {
+        __ mov(c_rarg0, addr);
+        __ mov(c_rarg1, count);
+      }
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
+      } else {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
+      }
+      __ pop(saved_regs, sp);
+    }
+  }
+}
+
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                                       Register start, Register end, Register scratch, RegSet saved_regs) {
+  if (is_oop) {
+    __ push(saved_regs, sp);
+    // must compute element count unless barrier set interface is changed (other platforms supply count)
+    assert_different_registers(start, end, scratch);
+    __ lea(scratch, Address(end, BytesPerHeapOop));
+    __ sub(scratch, scratch, start);               // subtract start to get #bytes
+    __ lsr(scratch, scratch, LogBytesPerHeapOop);  // convert to element count
+    __ mov(c_rarg0, start);
+    __ mov(c_rarg1, scratch);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
+    __ pop(saved_regs, sp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+  assert(thread == rthread, "must be");
+
+  Label done;
+  Label runtime;
+
+  assert_different_registers(obj, pre_val, tmp, rscratch1);
+  assert(pre_val != noreg &&  tmp != noreg, "expecting a register");
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  // Is marking active?
+  if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+    __ ldrw(tmp, in_progress);
+  } else {
+    assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+    __ ldrb(tmp, in_progress);
+  }
+  __ cbzw(tmp, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cbz(pre_val, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ ldr(tmp, index);                      // tmp := *index_adr
+  __ cbz(tmp, runtime);                    // tmp == 0?
+                                        // If yes, goto runtime
+
+  __ sub(tmp, tmp, wordSize);              // tmp := tmp - wordSize
+  __ str(tmp, index);                      // *index_adr := tmp
+  __ ldr(rscratch1, buffer);
+  __ add(tmp, tmp, rscratch1);             // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ str(pre_val, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  // save the live input values
+  RegSet saved = RegSet::of(pre_val);
+  if (tosca_live) saved += RegSet::of(r0);
+  if (obj != noreg) saved += RegSet::of(obj);
+
+  __ push(saved, sp);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(rfp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  if (expand_call) {
+    assert(pre_val != c_rarg1, "smashed arg");
+    __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  }
+
+  __ pop(saved, sp);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  Label is_null;
+  __ cbz(dst, is_null);
+  read_barrier_not_null_impl(masm, dst);
+  __ bind(is_null);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_not_null_impl(masm, dst);
+  }
+}
+
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahWriteBarrier) {
+    write_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+  assert(dst != rscratch1, "need rscratch1");
+  assert(dst != rscratch2, "need rscratch2");
+
+  Label done;
+
+  Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(rscratch1, gc_state);
+
+  // Check for heap stability
+  __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ tst(rscratch1, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  // Heap is unstable, need to perform the read-barrier even if WB is inactive
+  __ ldr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+
+  // Check for evacuation-in-progress and jump to WB slow-path if needed
+  __ mov(rscratch2, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ tst(rscratch1, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  RegSet to_save = RegSet::of(r0);
+  if (dst != r0) {
+    __ push(to_save, sp);
+    __ mov(r0, dst);
+  }
+
+  __ far_call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+
+  if (dst != r0) {
+    __ mov(dst, r0);
+    __ pop(to_save, sp);
+  }
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValEnqueueBarrier) {
+    Label is_null;
+    __ cbz(dst, is_null);
+    write_barrier_impl(masm, dst);
+    __ bind(is_null);
+    // Save possibly live regs.
+    RegSet live_regs = RegSet::range(r0, r4) - dst;
+    __ push(live_regs, sp);
+    __ strd(v0, __ pre(sp, 2 * -wordSize));
+
+    satb_write_barrier_pre(masm, noreg, dst, rthread, tmp, true, false);
+
+    // Restore possibly live regs.
+    __ ldrd(v0, __ post(sp, 2 * wordSize));
+    __ pop(live_regs, sp);
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                            Register dst, Address src, Register tmp1, Register tmp_thread) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  bool on_reference = on_weak || on_phantom;
+
+  if (in_heap) {
+    read_barrier_not_null(masm, src.base());
+  }
+
+  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
+    __ enter();
+    satb_write_barrier_pre(masm /* masm */,
+                           noreg /* obj */,
+                           dst /* pre_val */,
+                           rthread /* thread */,
+                           tmp1 /* tmp */,
+                           true /* tosca_live */,
+                           true /* expand_call */);
+    __ leave();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                             Address dst, Register val, Register tmp1, Register tmp2) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  if (in_heap) {
+    write_barrier(masm, dst.base());
+  }
+  if (!on_oop) {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+    return;
+  }
+
+  // flatten object address if needed
+  if (dst.index() == noreg && dst.offset() == 0) {
+    if (dst.base() != r3) {
+      __ mov(r3, dst.base());
+    }
+  } else {
+    __ lea(r3, dst);
+  }
+
+  shenandoah_write_barrier_pre(masm,
+                               r3 /* obj */,
+                               tmp2 /* pre_val */,
+                               rthread /* thread */,
+                               tmp1  /* tmp */,
+                               val != noreg /* tosca_live */,
+                               false /* expand_call */);
+
+  if (val == noreg) {
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), noreg, noreg, noreg);
+  } else {
+    storeval_barrier(masm, val, tmp1);
+    // G1 barrier needs uncompressed oop for region cross check.
+    Register new_val = val;
+    if (UseCompressedOops) {
+      new_val = rscratch2;
+      __ mov(new_val, val);
+    }
+    BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg);
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
+  __ cmp(op1, op2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ br(Assembler::EQ, done);
+    // The object may have been evacuated, but we won't see it without a
+    // membar here.
+    __ membar(Assembler::LoadStore| Assembler::LoadLoad);
+    read_barrier(masm, op1);
+    read_barrier(masm, op2);
+    __ cmp(op1, op2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm, Register obj,
+                                                  Register var_size_in_bytes,
+                                                  int con_size_in_bytes,
+                                                  Register t1,
+                                                  Register t2,
+                                                  Label& slow_case) {
+
+  assert_different_registers(obj, t2);
+  assert_different_registers(obj, var_size_in_bytes);
+  Register end = t2;
+
+  __ ldr(obj, Address(rthread, JavaThread::tlab_top_offset()));
+  if (var_size_in_bytes == noreg) {
+    __ lea(end, Address(obj, (int) (con_size_in_bytes + ShenandoahBrooksPointer::byte_size())));
+  } else {
+    __ add(var_size_in_bytes, var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
+    __ lea(end, Address(obj, var_size_in_bytes));
+  }
+  __ ldr(rscratch1, Address(rthread, JavaThread::tlab_end_offset()));
+  __ cmp(end, rscratch1);
+  __ br(Assembler::HI, slow_case);
+
+  // update the tlab top pointer
+  __ str(end, Address(rthread, JavaThread::tlab_top_offset()));
+
+  __ add(obj, obj, ShenandoahBrooksPointer::byte_size());
+  __ str(obj, Address(obj, ShenandoahBrooksPointer::byte_offset()));
+
+  // recover var_size_in_bytes if necessary
+  if (var_size_in_bytes == end) {
+    __ sub(var_size_in_bytes, var_size_in_bytes, obj);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
+  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+  bool is_write = (decorators & ACCESS_WRITE) != 0;
+  if (is_write) {
+    if (oop_not_null) {
+      write_barrier(masm, obj);
+    } else {
+      Label done;
+      __ cbz(obj, done);
+      write_barrier(masm, obj);
+      __ bind(done);
+    }
+  } else {
+    if (oop_not_null) {
+      read_barrier_not_null(masm, obj);
+    } else {
+      read_barrier(masm, obj);
+    }
+  }
+}
+
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                                                bool acquire, bool release, bool weak, bool encode,
+                                                Register tmp1, Register tmp2, Register tmp3,
+                                                Register result) {
+
+  if (!ShenandoahCASBarrier) {
+    if (UseCompressedOops) {
+      if (encode) {
+        __ encode_heap_oop(tmp1, expected);
+        expected = tmp1;
+        __ encode_heap_oop(tmp3, new_val);
+        new_val = tmp3;
+      }
+      __ cmpxchg(addr, expected, new_val, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
+      __ membar(__ AnyAny);
+    } else {
+      __ cmpxchg(addr, expected, new_val, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
+      __ membar(__ AnyAny);
+    }
+    return;
+  }
+
+  if (encode) {
+    storeval_barrier(masm, new_val, tmp3);
+  }
+
+  if (UseCompressedOops) {
+    if (encode) {
+      __ encode_heap_oop(tmp1, expected);
+      expected = tmp1;
+      __ encode_heap_oop(tmp2, new_val);
+      new_val = tmp2;
+    }
+  }
+  bool is_cae = (result != noreg);
+  bool is_narrow = UseCompressedOops;
+  Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
+  if (! is_cae) result = rscratch1;
+
+  assert_different_registers(addr, expected, new_val, result, tmp3);
+
+  Label retry, done, fail;
+
+  // CAS, using LL/SC pair.
+  __ bind(retry);
+  __ load_exclusive(result, addr, size, acquire);
+  if (is_narrow) {
+    __ cmpw(result, expected);
+  } else {
+    __ cmp(result, expected);
+  }
+  __ br(Assembler::NE, fail);
+  __ store_exclusive(tmp3, new_val, addr, size, release);
+  if (weak) {
+    __ cmpw(tmp3, 0u); // If the store fails, return NE to our caller
+  } else {
+    __ cbnzw(tmp3, retry);
+  }
+  __ b(done);
+
+ __  bind(fail);
+  // Check if rb(expected)==rb(result)
+  // Shuffle registers so that we have memory value ready for next expected.
+  __ mov(tmp3, expected);
+  __ mov(expected, result);
+  if (is_narrow) {
+    __ decode_heap_oop(result, result);
+    __ decode_heap_oop(tmp3, tmp3);
+  }
+  read_barrier_impl(masm, result);
+  read_barrier_impl(masm, tmp3);
+  __ cmp(result, tmp3);
+  // Retry with expected now being the value we just loaded from addr.
+  __ br(Assembler::EQ, retry);
+  if (is_narrow && is_cae) {
+    // For cmp-and-exchange and narrow oops, we need to restore
+    // the compressed old-value. We moved it to 'expected' a few lines up.
+    __ mov(result, expected);
+  }
+  __ bind(done);
+
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+  __ cbz(pre_val_reg, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ b(*stub->continuation());
+}
+
+void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+
+  Label done;
+
+  __ bind(*stub->entry());
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+  // Check for null.
+  if (stub->needs_null_check()) {
+    __ cbz(res, done);
+  }
+
+  write_barrier(ce->masm(), res);
+
+  __ bind(done);
+  __ b(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+
+  // arg0 : previous value of memory
+
+  BarrierSet* bs = BarrierSet::barrier_set();
+
+  const Register pre_val = r0;
+  const Register thread = rthread;
+  const Register tmp = rscratch1;
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is marking still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ ldrb(tmp, gc_state);
+  __ mov(rscratch2, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ tst(tmp, rscratch2);
+  __ br(Assembler::EQ, done);
+
+  // Can we store original value in the thread's buffer?
+  __ ldr(tmp, queue_index);
+  __ cbz(tmp, runtime);
+
+  __ sub(tmp, tmp, wordSize);
+  __ str(tmp, queue_index);
+  __ ldr(rscratch2, buffer);
+  __ add(tmp, tmp, rscratch2);
+  __ load_parameter(0, rscratch2);
+  __ str(rscratch2, Address(tmp, 0));
+  __ b(done);
+
+  __ bind(runtime);
+  __ push_call_clobbered_registers();
+  __ load_parameter(0, pre_val);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), pre_val, thread);
+  __ pop_call_clobbered_registers();
+  __ bind(done);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_wb() {
+  assert(_shenandoah_wb != NULL, "need write barrier stub");
+  return _shenandoah_wb;
+}
+
+#define __ cgen->assembler()->
+
+// Shenandoah write barrier.
+//
+// Input:
+//   r0: OOP to evacuate.  Not null.
+//
+// Output:
+//   r0: Pointer to evacuated OOP.
+//
+// Trash rscratch1, rscratch2.  Preserve everything else.
+address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+
+  __ align(6);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+  address start = __ pc();
+
+  Label work;
+  __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
+  __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  __ ldrb(rscratch2, Address(rscratch2, rscratch1));
+  __ tbnz(rscratch2, 0, work);
+  __ ret(lr);
+  __ bind(work);
+
+  Register obj = r0;
+
+  __ enter(); // required for proper stackwalking of RuntimeStub frame
+
+  __ push_call_clobbered_registers();
+
+  __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT));
+  __ blrt(lr, 1, 0, MacroAssembler::ret_type_integral);
+  __ mov(rscratch1, obj);
+  __ pop_call_clobbered_registers();
+  __ mov(obj, rscratch1);
+
+  __ leave(); // required for proper stackwalking of RuntimeStub frame
+  __ ret(lr);
+
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+    int stub_code_size = 2048;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_wb = generate_shenandoah_wb(&cgen);
+  }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahWriteBarrierStub;
+class StubAssembler;
+class StubCodeGenerator;
+#endif
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_wb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void read_barrier(MacroAssembler* masm, Register dst);
+  void read_barrier_impl(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
+  void write_barrier(MacroAssembler* masm, Register dst);
+  void write_barrier_impl(MacroAssembler* masm, Register dst);
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+  void asm_acmp_barrier(MacroAssembler* masm, Register op1, Register op2);
+
+  address generate_shenandoah_wb(StubCodeGenerator* cgen);
+
+public:
+  static address shenandoah_wb();
+
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register addr, Register count, RegSet saved_regs);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
+                                  Register start, Register end, Register tmp, RegSet saved_regs);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
+  virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
+  virtual void tlab_allocate(MacroAssembler* masm, Register obj,
+                             Register var_size_in_bytes,
+                             int con_size_in_bytes,
+                             Register t1,
+                             Register t2,
+                             Label& slow_case);
+
+  void cmpxchg_oop(MacroAssembler* masm, Register addr, Register expected, Register new_val,
+                   bool acquire, bool release, bool weak, bool encode,
+                   Register tmp1, Register tmp2, Register tmp3 = rscratch2,
+                   Register result = noreg);
+
+  virtual void barrier_stubs_init();
+};
+
+#endif // CPU_AARCH64_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_AARCH64_HPP

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetC1_aarch64.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  Register addr = _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, true, tmp1, tmp2);
+}
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+  BasicType bt = access.type();
+  if (access.is_oop()) {
+    LIRGenerator *gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item();
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2,
+                                                   LIR_OprFact::illegalOpr));
+
+      LIR_Opr result = gen->new_register(T_INT);
+      __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+               result, T_INT);
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  LIR_Opr tmp = gen->new_register(T_INT);
+  __ xchg(access.resolved_addr(), value_opr, result, tmp);
+
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoah_aarch64.ad	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,281 @@
+//
+// Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+//
+
+source_hpp %{
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+%}
+
+encode %{
+  enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+  %}
+
+  enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp) %{
+    MacroAssembler _masm(&cbuf);
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+  %}
+%}
+
+instruct shenandoahRB(iRegPNoSp dst, iRegP src, rFlagsReg cr) %{
+  match(Set dst (ShenandoahReadBarrier src));
+  format %{ "shenandoah_rb $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ ldr(d, Address(s, ShenandoahBrooksPointer::byte_offset()));
+  %}
+  ins_pipe(pipe_class_memory);
+%}
+
+
+instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp),
+             aarch64_enc_cset_eq(res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchgw_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+    __ cset($res$$Register, Assembler::EQ);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+  format %{
+    "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+
+  ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp),
+             aarch64_enc_cset_eq(res));
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+
+  effect(TEMP tmp, KILL cr);
+
+ format %{
+    "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+    "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+ %}
+
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg);
+    __ cset($res$$Register, Assembler::EQ);
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeN_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeP_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangeNAcq_shenandoah(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndExchangeN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /* encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct compareAndExchangePAcq_shenandoah(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahCompareAndExchangeP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP_DEF res, TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_acq_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ false, /*encode*/ false, noreg, noreg, rscratch2, $res$$Register);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(2 * VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ false, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahWeakCompareAndSwapN mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchgw_acq_shenandoah $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+
+instruct weakCompareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
+  predicate(needs_acquiring_load_exclusive(n));
+  match(Set res (ShenandoahWeakCompareAndSwapP mem (Binary oldval newval)));
+  ins_cost(VOLATILE_REF_COST);
+  effect(TEMP tmp, KILL cr);
+  format %{
+    "cmpxchg_acq_shenandoah $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
+    "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
+  %}
+  ins_encode %{
+    Register tmp = $tmp$$Register;
+    __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
+    ShenandoahBarrierSet::assembler()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
+                                                   /*acquire*/ true, /*release*/ true, /*weak*/ true, /*encode*/ false, noreg, noreg);
+    __ csetw($res$$Register, Assembler::EQ);
+  %}
+  ins_pipe(pipe_slow);
+%}
+

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,1041 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/shenandoahHeap.hpp"
+#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+#include "gc/shenandoah/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahRuntime.hpp"
+#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
+#include "interpreter/interpreter.hpp"
+#include "interpreter/interp_masm.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+#endif
+
+#define __ masm->
+
+address ShenandoahBarrierSetAssembler::_shenandoah_wb = NULL;
+
+void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register src, Register dst, Register count) {
+
+  bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+  bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+  bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  if (type == T_OBJECT || type == T_ARRAY) {
+#ifdef _LP64
+    if (!checkcast && !obj_int) {
+      // Save count for barrier
+      __ movptr(r11, count);
+    } else if (disjoint && obj_int) {
+      // Save dst in r11 in the disjoint case
+      __ movq(r11, dst);
+    }
+#else
+    if (disjoint) {
+      __ mov(rdx, dst);          // save 'to'
+    }
+#endif
+
+    if (!dest_uninitialized && !ShenandoahHeap::heap()->heuristics()->can_do_traversal_gc()) {
+      Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+#ifndef _LP64
+      __ push(thread);
+      __ get_thread(thread);
+#endif
+
+      Label filtered;
+      Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+      // Is marking active?
+      if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
+        __ cmpl(in_progress, 0);
+      } else {
+        assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
+        __ cmpb(in_progress, 0);
+      }
+
+      NOT_LP64(__ pop(thread);)
+
+        __ jcc(Assembler::equal, filtered);
+
+      __ pusha();                      // push registers
+#ifdef _LP64
+      if (count == c_rarg0) {
+        if (dst == c_rarg1) {
+          // exactly backwards!!
+          __ xchgptr(c_rarg1, c_rarg0);
+        } else {
+          __ movptr(c_rarg1, count);
+          __ movptr(c_rarg0, dst);
+        }
+      } else {
+        __ movptr(c_rarg0, dst);
+        __ movptr(c_rarg1, count);
+      }
+      if (UseCompressedOops) {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_narrow_oop_entry), 2);
+      } else {
+        __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry), 2);
+      }
+#else
+      __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_pre_oop_entry),
+                      dst, count);
+#endif
+      __ popa();
+      __ bind(filtered);
+    }
+  }
+
+}
+
+void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                                       Register src, Register dst, Register count) {
+  bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+  bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+  bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+  Register tmp = rax;
+
+  if (type == T_OBJECT || type == T_ARRAY) {
+#ifdef _LP64
+    if (!checkcast && !obj_int) {
+      // Save count for barrier
+      count = r11;
+    } else if (disjoint && obj_int) {
+      // Use the saved dst in the disjoint case
+      dst = r11;
+    } else if (checkcast) {
+      tmp = rscratch1;
+    }
+#else
+    if (disjoint) {
+      __ mov(dst, rdx); // restore 'to'
+    }
+#endif
+
+    __ pusha();             // push registers (overkill)
+#ifdef _LP64
+    if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
+      assert_different_registers(c_rarg1, dst);
+      __ mov(c_rarg1, count);
+      __ mov(c_rarg0, dst);
+    } else {
+      assert_different_registers(c_rarg0, count);
+      __ mov(c_rarg0, dst);
+      __ mov(c_rarg1, count);
+    }
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry), 2);
+#else
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_array_post_entry),
+                    dst, count);
+#endif
+    __ popa();
+  }
+}
+
+void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                                                 Register obj,
+                                                                 Register pre_val,
+                                                                 Register thread,
+                                                                 Register tmp,
+                                                                 bool tosca_live,
+                                                                 bool expand_call) {
+
+  if (ShenandoahSATBBarrier) {
+    satb_write_barrier_pre(masm, obj, pre_val, thread, tmp, tosca_live, expand_call);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm,
+                                                           Register obj,
+                                                           Register pre_val,
+                                                           Register thread,
+                                                           Register tmp,
+                                                           bool tosca_live,
+                                                           bool expand_call) {
+  // If expand_call is true then we expand the call_VM_leaf macro
+  // directly to skip generating the check by
+  // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
+
+#ifdef _LP64
+  assert(thread == r15_thread, "must be");
+#endif // _LP64
+
+  Label done;
+  Label runtime;
+
+  assert(pre_val != noreg, "check this code");
+
+  if (obj != noreg) {
+    assert_different_registers(obj, pre_val, tmp);
+    assert(pre_val != rax, "check this code");
+  }
+
+  Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
+  Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ jcc(Assembler::zero, done);
+
+  // Do we need to load the previous value?
+  if (obj != noreg) {
+    __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
+  }
+
+  // Is the previous value null?
+  __ cmpptr(pre_val, (int32_t) NULL_WORD);
+  __ jcc(Assembler::equal, done);
+
+  // Can we store original value in the thread's buffer?
+  // Is index == 0?
+  // (The index field is typed as size_t.)
+
+  __ movptr(tmp, index);                   // tmp := *index_adr
+  __ cmpptr(tmp, 0);                       // tmp == 0?
+  __ jcc(Assembler::equal, runtime);       // If yes, goto runtime
+
+  __ subptr(tmp, wordSize);                // tmp := tmp - wordSize
+  __ movptr(index, tmp);                   // *index_adr := tmp
+  __ addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
+
+  // Record the previous value
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+  // save the live input values
+  if(tosca_live) __ push(rax);
+
+  if (obj != noreg && obj != rax)
+    __ push(obj);
+
+  if (pre_val != rax)
+    __ push(pre_val);
+
+  // Calling the runtime using the regular call_VM_leaf mechanism generates
+  // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
+  // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
+  //
+  // If we care generating the pre-barrier without a frame (e.g. in the
+  // intrinsified Reference.get() routine) then ebp might be pointing to
+  // the caller frame and so this check will most likely fail at runtime.
+  //
+  // Expanding the call directly bypasses the generation of the check.
+  // So when we do not have have a full interpreter frame on the stack
+  // expand_call should be passed true.
+
+  NOT_LP64( __ push(thread); )
+
+#ifdef _LP64
+  // We move pre_val into c_rarg0 early, in order to avoid smashing it, should
+  // pre_val be c_rarg1 (where the call prologue would copy thread argument).
+  // Note: this should not accidentally smash thread, because thread is always r15.
+  assert(thread != c_rarg0, "smashed arg");
+  if (c_rarg0 != pre_val) {
+    __ mov(c_rarg0, pre_val);
+  }
+#endif
+
+  if (expand_call) {
+    LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
+#ifdef _LP64
+    if (c_rarg1 != thread) {
+      __ mov(c_rarg1, thread);
+    }
+    // Already moved pre_val into c_rarg0 above
+#else
+    __ push(thread);
+    __ push(pre_val);
+#endif
+    __ MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), 2);
+  } else {
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), LP64_ONLY(c_rarg0) NOT_LP64(pre_val), thread);
+  }
+
+  NOT_LP64( __ pop(thread); )
+
+  // save the live input values
+  if (pre_val != rax)
+    __ pop(pre_val);
+
+  if (obj != noreg && obj != rax)
+    __ pop(obj);
+
+  if(tosca_live) __ pop(rax);
+
+  __ bind(done);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  Label is_null;
+  __ testptr(dst, dst);
+  __ jcc(Assembler::zero, is_null);
+  read_barrier_not_null_impl(masm, dst);
+  __ bind(is_null);
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null(MacroAssembler* masm, Register dst) {
+  if (ShenandoahReadBarrier) {
+    read_barrier_not_null_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::read_barrier_not_null_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || ShenandoahCASBarrier), "should be enabled");
+  __ movptr(dst, Address(dst, ShenandoahBrooksPointer::byte_offset()));
+}
+
+
+void ShenandoahBarrierSetAssembler::write_barrier(MacroAssembler* masm, Register dst) {
+  if (ShenandoahWriteBarrier) {
+    write_barrier_impl(masm, dst);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::write_barrier_impl(MacroAssembler* masm, Register dst) {
+  assert(UseShenandoahGC && (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier), "Should be enabled");
+#ifdef _LP64
+  Label done;
+
+  Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ jccb(Assembler::zero, done);
+
+  // Heap is unstable, need to perform the read-barrier even if WB is inactive
+  read_barrier_not_null(masm, dst);
+
+  __ testb(gc_state, ShenandoahHeap::EVACUATION | ShenandoahHeap::TRAVERSAL);
+  __ jccb(Assembler::zero, done);
+
+   if (dst != rax) {
+     __ xchgptr(dst, rax); // Move obj into rax and save rax into obj.
+   }
+
+   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_wb())));
+
+   if (dst != rax) {
+     __ xchgptr(rax, dst); // Swap back obj with rax.
+   }
+
+  __ bind(done);
+#else
+  Unimplemented();
+#endif
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier(MacroAssembler* masm, Register dst, Register tmp) {
+  if (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier) {
+    storeval_barrier_impl(masm, dst, tmp);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp) {
+  assert(UseShenandoahGC && (ShenandoahStoreValReadBarrier || ShenandoahStoreValEnqueueBarrier), "should be enabled");
+
+  if (dst == noreg) return;
+
+#ifdef _LP64
+  if (ShenandoahStoreValEnqueueBarrier) {
+    Label is_null;
+    __ testptr(dst, dst);
+    __ jcc(Assembler::zero, is_null);
+    write_barrier_impl(masm, dst);
+    __ bind(is_null);
+
+    // The set of registers to be saved+restored is the same as in the write-barrier above.
+    // Those are the commonly used registers in the interpreter.
+    __ pusha();
+    // __ push_callee_saved_registers();
+    __ subptr(rsp, 2 * Interpreter::stackElementSize);
+    __ movdbl(Address(rsp, 0), xmm0);
+
+    satb_write_barrier_pre(masm, noreg, dst, r15_thread, tmp, true, false);
+    __ movdbl(xmm0, Address(rsp, 0));
+    __ addptr(rsp, 2 * Interpreter::stackElementSize);
+    //__ pop_callee_saved_registers();
+    __ popa();
+  }
+  if (ShenandoahStoreValReadBarrier) {
+    read_barrier_impl(masm, dst);
+  }
+#else
+  Unimplemented();
+#endif
+}
+
+void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+             Register dst, Address src, Register tmp1, Register tmp_thread) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  bool on_reference = on_weak || on_phantom;
+  if (in_heap) {
+    read_barrier_not_null(masm, src.base());
+  }
+  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+  if (ShenandoahKeepAliveBarrier && on_oop && on_reference) {
+    const Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
+    NOT_LP64(__ get_thread(thread));
+
+    // Generate the SATB pre-barrier code to log the value of
+    // the referent field in an SATB buffer.
+    shenandoah_write_barrier_pre(masm /* masm */,
+                                 noreg /* obj */,
+                                 dst /* pre_val */,
+                                 thread /* thread */,
+                                 tmp1 /* tmp */,
+                                 true /* tosca_live */,
+                                 true /* expand_call */);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+              Address dst, Register val, Register tmp1, Register tmp2) {
+
+  bool in_heap = (decorators & IN_HEAP) != 0;
+  bool as_normal = (decorators & AS_NORMAL) != 0;
+  if (in_heap) {
+    write_barrier(masm, dst.base());
+  }
+  if (type == T_OBJECT || type == T_ARRAY) {
+    bool needs_pre_barrier = as_normal;
+
+    Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
+    Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
+    // flatten object address if needed
+    // We do it regardless of precise because we need the registers
+    if (dst.index() == noreg && dst.disp() == 0) {
+      if (dst.base() != tmp1) {
+        __ movptr(tmp1, dst.base());
+      }
+    } else {
+      __ lea(tmp1, dst);
+    }
+
+#ifndef _LP64
+    InterpreterMacroAssembler *imasm = static_cast<InterpreterMacroAssembler*>(masm);
+#endif
+
+    NOT_LP64(__ get_thread(rcx));
+    NOT_LP64(imasm->save_bcp());
+
+    if (needs_pre_barrier) {
+      shenandoah_write_barrier_pre(masm /*masm*/,
+                                   tmp1 /* obj */,
+                                   tmp2 /* pre_val */,
+                                   rthread /* thread */,
+                                   tmp3  /* tmp */,
+                                   val != noreg /* tosca_live */,
+                                   false /* expand_call */);
+    }
+    if (val == noreg) {
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    } else {
+      storeval_barrier(masm, val, tmp3);
+      BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg);
+    }
+    NOT_LP64(imasm->restore_bcp());
+  } else {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2);
+  }
+}
+
+#ifndef _LP64
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
+                                               Address obj1, jobject obj2) {
+  Unimplemented();
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm,
+                                               Register obj1, jobject obj2) {
+  Unimplemented();
+}
+#endif
+
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register op1, Register op2) {
+  __ cmpptr(op1, op2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ jccb(Assembler::equal, done);
+    read_barrier(masm, op1);
+    read_barrier(masm, op2);
+    __ cmpptr(op1, op2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::obj_equals(MacroAssembler* masm, Register src1, Address src2) {
+  __ cmpptr(src1, src2);
+  if (ShenandoahAcmpBarrier) {
+    Label done;
+    __ jccb(Assembler::equal, done);
+    __ movptr(rscratch2, src2);
+    read_barrier(masm, src1);
+    read_barrier(masm, rscratch2);
+    __ cmpptr(src1, rscratch2);
+    __ bind(done);
+  }
+}
+
+void ShenandoahBarrierSetAssembler::tlab_allocate(MacroAssembler* masm,
+                                                  Register thread, Register obj,
+                                                  Register var_size_in_bytes,
+                                                  int con_size_in_bytes,
+                                                  Register t1, Register t2,
+                                                  Label& slow_case) {
+  assert_different_registers(obj, t1, t2);
+  assert_different_registers(obj, var_size_in_bytes, t1);
+  Register end = t2;
+  if (!thread->is_valid()) {
+#ifdef _LP64
+    thread = r15_thread;
+#else
+    assert(t1->is_valid(), "need temp reg");
+    thread = t1;
+    __ get_thread(thread);
+#endif
+  }
+
+  __ verify_tlab();
+
+  __ movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
+  if (var_size_in_bytes == noreg) {
+    __ lea(end, Address(obj, con_size_in_bytes + ShenandoahBrooksPointer::byte_size()));
+  } else {
+    __ addptr(var_size_in_bytes, ShenandoahBrooksPointer::byte_size());
+    __ lea(end, Address(obj, var_size_in_bytes, Address::times_1));
+  }
+  __ cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
+  __ jcc(Assembler::above, slow_case);
+
+  // update the tlab top pointer
+  __ movptr(Address(thread, JavaThread::tlab_top_offset()), end);
+
+  // Initialize brooks pointer
+#ifdef _LP64
+  __ incrementq(obj, ShenandoahBrooksPointer::byte_size());
+#else
+  __ incrementl(obj, ShenandoahBrooksPointer::byte_size());
+#endif
+  __ movptr(Address(obj, ShenandoahBrooksPointer::byte_offset()), obj);
+
+  // recover var_size_in_bytes if necessary
+  if (var_size_in_bytes == end) {
+    __ subptr(var_size_in_bytes, obj);
+  }
+  __ verify_tlab();
+}
+
+void ShenandoahBarrierSetAssembler::resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj) {
+  bool oop_not_null = (decorators & IS_NOT_NULL) != 0;
+  bool is_write = (decorators & ACCESS_WRITE) != 0;
+  if (is_write) {
+    if (oop_not_null) {
+      write_barrier(masm, obj);
+    } else {
+      Label done;
+      __ testptr(obj, obj);
+      __ jcc(Assembler::zero, done);
+      write_barrier(masm, obj);
+      __ bind(done);
+    }
+  } else {
+    if (oop_not_null) {
+      read_barrier_not_null(masm, obj);
+    } else {
+      read_barrier(masm, obj);
+    }
+  }
+}
+
+// Special Shenandoah CAS implementation that handles false negatives
+// due to concurrent evacuation.
+#ifndef _LP64
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
+                                                Register res, Address addr, Register oldval, Register newval,
+                                                bool exchange, bool encode, Register tmp1, Register tmp2) {
+  // Shenandoah has no 32-bit version for this.
+  Unimplemented();
+}
+#else
+void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
+                                                Register res, Address addr, Register oldval, Register newval,
+                                                bool exchange, bool encode, Register tmp1, Register tmp2) {
+  if (!ShenandoahCASBarrier) {
+#ifdef _LP64
+    if (UseCompressedOops) {
+      if (encode) {
+        __ encode_heap_oop(oldval);
+        __ mov(rscratch1, newval);
+        __ encode_heap_oop(rscratch1);
+        newval = rscratch1;
+      }
+      if (os::is_MP()) {
+        __ lock();
+      }
+      // oldval (rax) is implicitly used by this instruction
+      __ cmpxchgl(newval, addr);
+    } else
+#endif
+      {
+        if (os::is_MP()) {
+          __ lock();
+        }
+        __ cmpxchgptr(newval, addr);
+      }
+
+    if (!exchange) {
+      assert(res != NULL, "need result register");
+      __ setb(Assembler::equal, res);
+      __ movzbl(res, res);
+    }
+    return;
+  }
+
+  assert(ShenandoahCASBarrier, "Should only be used when CAS barrier is enabled");
+  assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
+
+  Label retry, done;
+
+  // Apply storeval barrier to newval.
+  if (encode) {
+    storeval_barrier(masm, newval, tmp1);
+  }
+
+  if (UseCompressedOops) {
+    if (encode) {
+      __ encode_heap_oop(oldval);
+      __ mov(rscratch1, newval);
+      __ encode_heap_oop(rscratch1);
+      newval = rscratch1;
+    }
+  }
+
+  // Remember oldval for retry logic below
+  if (UseCompressedOops) {
+    __ movl(tmp1, oldval);
+  } else {
+    __ movptr(tmp1, oldval);
+  }
+
+  // Step 1. Try to CAS with given arguments. If successful, then we are done,
+  // and can safely return.
+  if (os::is_MP()) __ lock();
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else {
+    __ cmpxchgptr(newval, addr);
+  }
+  __ jcc(Assembler::equal, done, true);
+
+  // Step 2. CAS had failed. This may be a false negative.
+  //
+  // The trouble comes when we compare the to-space pointer with the from-space
+  // pointer to the same object. To resolve this, it will suffice to read both
+  // oldval and the value from memory through the read barriers -- this will give
+  // both to-space pointers. If they mismatch, then it was a legitimate failure.
+  //
+  if (UseCompressedOops) {
+    __ decode_heap_oop(tmp1);
+  }
+  read_barrier_impl(masm, tmp1);
+
+  if (UseCompressedOops) {
+    __ movl(tmp2, oldval);
+    __ decode_heap_oop(tmp2);
+  } else {
+    __ movptr(tmp2, oldval);
+  }
+  read_barrier_impl(masm, tmp2);
+
+  __ cmpptr(tmp1, tmp2);
+  __ jcc(Assembler::notEqual, done, true);
+
+  // Step 3. Try to CAS again with resolved to-space pointers.
+  //
+  // Corner case: it may happen that somebody stored the from-space pointer
+  // to memory while we were preparing for retry. Therefore, we can fail again
+  // on retry, and so need to do this in loop, always re-reading the failure
+  // witness through the read barrier.
+  __ bind(retry);
+  if (os::is_MP()) __ lock();
+  if (UseCompressedOops) {
+    __ cmpxchgl(newval, addr);
+  } else {
+    __ cmpxchgptr(newval, addr);
+  }
+  __ jcc(Assembler::equal, done, true);
+
+  if (UseCompressedOops) {
+    __ movl(tmp2, oldval);
+    __ decode_heap_oop(tmp2);
+  } else {
+    __ movptr(tmp2, oldval);
+  }
+  read_barrier_impl(masm, tmp2);
+
+  __ cmpptr(tmp1, tmp2);
+  __ jcc(Assembler::equal, retry, true);
+
+  // Step 4. If we need a boolean result out of CAS, check the flag again,
+  // and promote the result. Note that we handle the flag from both the CAS
+  // itself and from the retry loop.
+  __ bind(done);
+  if (!exchange) {
+    assert(res != NULL, "need result register");
+    __ setb(Assembler::equal, res);
+    __ movzbl(res, res);
+  }
+}
+#endif // LP64
+
+void ShenandoahBarrierSetAssembler::save_vector_registers(MacroAssembler* masm) {
+  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
+  if (UseAVX > 2) {
+    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
+  }
+
+  if (UseSSE == 1)  {
+    __ subptr(rsp, sizeof(jdouble)*8);
+    for (int n = 0; n < 8; n++) {
+      __ movflt(Address(rsp, n*sizeof(jdouble)), as_XMMRegister(n));
+    }
+  } else if (UseSSE >= 2)  {
+    if (UseAVX > 2) {
+      __ push(rbx);
+      __ movl(rbx, 0xffff);
+      __ kmovwl(k1, rbx);
+      __ pop(rbx);
+    }
+#ifdef COMPILER2
+    if (MaxVectorSize > 16) {
+      if(UseAVX > 2) {
+        // Save upper half of ZMM registers
+        __ subptr(rsp, 32*num_xmm_regs);
+        for (int n = 0; n < num_xmm_regs; n++) {
+          __ vextractf64x4_high(Address(rsp, n*32), as_XMMRegister(n));
+        }
+      }
+      assert(UseAVX > 0, "256 bit vectors are supported only with AVX");
+      // Save upper half of YMM registers
+      __ subptr(rsp, 16*num_xmm_regs);
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vextractf128_high(Address(rsp, n*16), as_XMMRegister(n));
+      }
+    }
+#endif
+    // Save whole 128bit (16 bytes) XMM registers
+    __ subptr(rsp, 16*num_xmm_regs);
+#ifdef _LP64
+    if (VM_Version::supports_evex()) {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vextractf32x4(Address(rsp, n*16), as_XMMRegister(n), 0);
+      }
+    } else {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
+      }
+    }
+#else
+    for (int n = 0; n < num_xmm_regs; n++) {
+      __ movdqu(Address(rsp, n*16), as_XMMRegister(n));
+    }
+#endif
+  }
+}
+
+void ShenandoahBarrierSetAssembler::restore_vector_registers(MacroAssembler* masm) {
+  int num_xmm_regs = LP64_ONLY(16) NOT_LP64(8);
+  if (UseAVX > 2) {
+    num_xmm_regs = LP64_ONLY(32) NOT_LP64(8);
+  }
+  if (UseSSE == 1)  {
+    for (int n = 0; n < 8; n++) {
+      __ movflt(as_XMMRegister(n), Address(rsp, n*sizeof(jdouble)));
+    }
+    __ addptr(rsp, sizeof(jdouble)*8);
+  } else if (UseSSE >= 2)  {
+    // Restore whole 128bit (16 bytes) XMM registers
+#ifdef _LP64
+    if (VM_Version::supports_evex()) {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vinsertf32x4(as_XMMRegister(n), as_XMMRegister(n), Address(rsp, n*16), 0);
+      }
+    } else {
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
+      }
+    }
+#else
+    for (int n = 0; n < num_xmm_regs; n++) {
+      __ movdqu(as_XMMRegister(n), Address(rsp, n*16));
+    }
+#endif
+    __ addptr(rsp, 16*num_xmm_regs);
+
+#ifdef COMPILER2
+    if (MaxVectorSize > 16) {
+      // Restore upper half of YMM registers.
+      for (int n = 0; n < num_xmm_regs; n++) {
+        __ vinsertf128_high(as_XMMRegister(n), Address(rsp, n*16));
+      }
+      __ addptr(rsp, 16*num_xmm_regs);
+      if (UseAVX > 2) {
+        for (int n = 0; n < num_xmm_regs; n++) {
+          __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, n*32));
+        }
+        __ addptr(rsp, 32*num_xmm_regs);
+      }
+    }
+#endif
+  }
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
+  ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+  // At this point we know that marking is in progress.
+  // If do_load() is true then we have to emit the
+  // load of the previous value; otherwise it has already
+  // been loaded into _pre_val.
+
+  __ bind(*stub->entry());
+  assert(stub->pre_val()->is_register(), "Precondition.");
+
+  Register pre_val_reg = stub->pre_val()->as_register();
+
+  if (stub->do_load()) {
+    ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/, false /*unaligned*/);
+  }
+
+  __ cmpptr(pre_val_reg, (int32_t)NULL_WORD);
+  __ jcc(Assembler::equal, *stub->continuation());
+  ce->store_parameter(stub->pre_val()->as_register(), 0);
+  __ call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
+  __ jmp(*stub->continuation());
+
+}
+
+void ShenandoahBarrierSetAssembler::gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub) {
+  __ bind(*stub->entry());
+
+  Label done;
+  Register obj = stub->obj()->as_register();
+  Register res = stub->result()->as_register();
+
+  if (res != obj) {
+    __ mov(res, obj);
+  }
+
+  // Check for null.
+  if (stub->needs_null_check()) {
+    __ testptr(res, res);
+    __ jcc(Assembler::zero, done);
+  }
+
+  write_barrier(ce->masm(), res);
+
+  __ bind(done);
+  __ jmp(*stub->continuation());
+}
+
+#undef __
+
+#define __ sasm->
+
+void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
+  __ prologue("shenandoah_pre_barrier", false);
+  // arg0 : previous value of memory
+
+  __ push(rax);
+  __ push(rdx);
+
+  const Register pre_val = rax;
+  const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
+  const Register tmp = rdx;
+
+  NOT_LP64(__ get_thread(thread);)
+
+  Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+  Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+  Label done;
+  Label runtime;
+
+  // Is SATB still active?
+  Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+  __ testb(gc_state, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL);
+  __ jcc(Assembler::zero, done);
+
+  // Can we store original value in the thread's buffer?
+
+  __ movptr(tmp, queue_index);
+  __ testptr(tmp, tmp);
+  __ jcc(Assembler::zero, runtime);
+  __ subptr(tmp, wordSize);
+  __ movptr(queue_index, tmp);
+  __ addptr(tmp, buffer);
+
+  // prev_val (rax)
+  __ load_parameter(0, pre_val);
+  __ movptr(Address(tmp, 0), pre_val);
+  __ jmp(done);
+
+  __ bind(runtime);
+
+  __ save_live_registers_no_oop_map(true);
+
+  // load the pre-value
+  __ load_parameter(0, rcx);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), rcx, thread);
+
+  __ restore_live_registers(true);
+
+  __ bind(done);
+
+  __ pop(rdx);
+  __ pop(rax);
+
+  __ epilogue();
+}
+
+#undef __
+
+#endif // COMPILER1
+
+address ShenandoahBarrierSetAssembler::shenandoah_wb() {
+  assert(_shenandoah_wb != NULL, "need write barrier stub");
+  return _shenandoah_wb;
+}
+
+#define __ cgen->assembler()->
+
+address ShenandoahBarrierSetAssembler::generate_shenandoah_wb(StubCodeGenerator* cgen) {
+  __ align(CodeEntryAlignment);
+  StubCodeMark mark(cgen, "StubRoutines", "shenandoah_wb");
+  address start = __ pc();
+
+#ifdef _LP64
+  Label not_done;
+
+  // We use RDI, which also serves as argument register for slow call.
+  // RAX always holds the src object ptr, except after the slow call and
+  // the cmpxchg, then it holds the result.
+  // R8 and RCX are used as temporary registers.
+  __ push(rdi);
+  __ push(r8);
+
+  // Check for object beeing in the collection set.
+  // TODO: Can we use only 1 register here?
+  // The source object arrives here in rax.
+  // live: rax
+  // live: rdi
+  __ mov(rdi, rax);
+  __ shrptr(rdi, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+  // live: r8
+  __ movptr(r8, (intptr_t) ShenandoahHeap::in_cset_fast_test_addr());
+  __ movbool(r8, Address(r8, rdi, Address::times_1));
+  // unlive: rdi
+  __ testbool(r8);
+  // unlive: r8
+  __ jccb(Assembler::notZero, not_done);
+
+  __ pop(r8);
+  __ pop(rdi);
+  __ ret(0);
+
+  __ bind(not_done);
+
+  __ push(rcx);
+  __ push(rdx);
+  __ push(rdi);
+  __ push(rsi);
+  __ push(r8);
+  __ push(r9);
+  __ push(r10);
+  __ push(r11);
+  __ push(r12);
+  __ push(r13);
+  __ push(r14);
+  __ push(r15);
+  save_vector_registers(cgen->assembler());
+  __ movptr(rdi, rax);
+  __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_JRT), rdi);
+  restore_vector_registers(cgen->assembler());
+  __ pop(r15);
+  __ pop(r14);
+  __ pop(r13);
+  __ pop(r12);
+  __ pop(r11);
+  __ pop(r10);
+  __ pop(r9);
+  __ pop(r8);
+  __ pop(rsi);
+  __ pop(rdi);
+  __ pop(rdx);
+  __ pop(rcx);
+
+  __ pop(r8);
+  __ pop(rdi);
+  __ ret(0);
+#else
+  ShouldNotReachHere();
+#endif
+  return start;
+}
+
+#undef __
+
+void ShenandoahBarrierSetAssembler::barrier_stubs_init() {
+  if (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) {
+    int stub_code_size = 4096;
+    ResourceMark rm;
+    BufferBlob* bb = BufferBlob::create("shenandoah_barrier_stubs", stub_code_size);
+    CodeBuffer buf(bb);
+    StubCodeGenerator cgen(&buf);
+    _shenandoah_wb = generate_shenandoah_wb(&cgen);
+  }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,119 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+#define CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP
+
+#include "asm/macroAssembler.hpp"
+#include "gc/shared/barrierSetAssembler.hpp"
+#ifdef COMPILER1
+class LIR_Assembler;
+class ShenandoahPreBarrierStub;
+class ShenandoahWriteBarrierStub;
+class StubAssembler;
+class StubCodeGenerator;
+#endif
+
+class ShenandoahBarrierSetAssembler: public BarrierSetAssembler {
+private:
+
+  static address _shenandoah_wb;
+
+  void satb_write_barrier_pre(MacroAssembler* masm,
+                              Register obj,
+                              Register pre_val,
+                              Register thread,
+                              Register tmp,
+                              bool tosca_live,
+                              bool expand_call);
+
+  void shenandoah_write_barrier_pre(MacroAssembler* masm,
+                                    Register obj,
+                                    Register pre_val,
+                                    Register thread,
+                                    Register tmp,
+                                    bool tosca_live,
+                                    bool expand_call);
+
+  void read_barrier(MacroAssembler* masm, Register dst);
+  void read_barrier_impl(MacroAssembler* masm, Register dst);
+
+  void read_barrier_not_null(MacroAssembler* masm, Register dst);
+  void read_barrier_not_null_impl(MacroAssembler* masm, Register dst);
+
+  void write_barrier(MacroAssembler* masm, Register dst);
+  void write_barrier_impl(MacroAssembler* masm, Register dst);
+
+  void storeval_barrier(MacroAssembler* masm, Register dst, Register tmp);
+  void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp);
+
+  address generate_shenandoah_wb(StubCodeGenerator* cgen);
+
+  void save_vector_registers(MacroAssembler* masm);
+  void restore_vector_registers(MacroAssembler* masm);
+
+public:
+  static address shenandoah_wb();
+
+#ifdef COMPILER1
+  void gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub);
+  void gen_write_barrier_stub(LIR_Assembler* ce, ShenandoahWriteBarrierStub* stub);
+  void generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm);
+#endif
+
+  void cmpxchg_oop(MacroAssembler* masm,
+                   Register res, Address addr, Register oldval, Register newval,
+                   bool exchange, bool encode, Register tmp1, Register tmp2);
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count);
+  virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count);
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register dst, Address src, Register tmp1, Register tmp_thread);
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Address dst, Register val, Register tmp1, Register tmp2);
+
+#ifndef _LP64
+  virtual void obj_equals(MacroAssembler* masm,
+                          Address obj1, jobject obj2);
+  virtual void obj_equals(MacroAssembler* masm,
+                          Register obj1, jobject obj2);
+#endif
+
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Register src2);
+  virtual void obj_equals(MacroAssembler* masm, Register src1, Address src2);
+
+  virtual void tlab_allocate(MacroAssembler* masm,
+                             Register thread, Register obj,
+                             Register var_size_in_bytes,
+                             int con_size_in_bytes,
+                             Register t1, Register t2,
+                             Label& slow_case);
+
+  virtual void resolve(MacroAssembler* masm, DecoratorSet decorators, Register obj);
+
+  virtual void barrier_stubs_init();
+
+};
+
+#endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetC1_x86.cpp	Mon Dec 10 15:47:44 2018 +0100
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2018, Red Hat, Inc. All rights reserved.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
+#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
+
+void LIR_OpShenandoahCompareAndSwap::emit_code(LIR_Assembler* masm) {
+  Register addr = _addr->as_register_lo();
+  Register newval = _new_value->as_register();
+  Register cmpval = _cmp_value->as_register();
+  Register tmp1 = _tmp1->as_register();
+  Register tmp2 = _tmp2->as_register();
+  assert(cmpval == rax, "wrong register");
+  assert(newval != NULL, "new val must be register");
+  assert(cmpval != newval, "cmp and new values must be in different registers");
+  assert(cmpval != addr, "cmp and addr must be in different registers");
+  assert(newval != addr, "new value and addr must be in different registers");
+  ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm->masm(), NULL, Address(addr, 0), cmpval, newval, true, true, tmp1, tmp2);
+}
+
+#ifdef ASSERT
+#define __ gen->lir(__FILE__, __LINE__)->
+#else
+#define __ gen->lir()->
+#endif
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+
+  if (access.is_oop()) {
+    LIRGenerator* gen = access.gen();
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(gen, access.access_emit_info(), access.decorators(), access.resolved_addr(),
+                  LIR_OprFact::illegalOpr /* pre_val */);
+    }
+    if (ShenandoahCASBarrier) {
+      cmp_value.load_item_force(FrameMap::rax_oop_opr);
+      new_value.load_item();
+
+      LIR_Opr t1 = gen->new_register(T_OBJECT);
+      LIR_Opr t2 = gen->new_register(T_OBJECT);
+      LIR_Opr addr = access.resolved_addr()->as_address_ptr()->base();
+
+      __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, LIR_OprFact::illegalOpr));
+
+      LIR_Opr result = gen->new_register(T_INT);
+      __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+               result, T_INT);
+      return result;
+    }
+  }
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  LIRGenerator* gen = access.gen();
+  BasicType type = access.type();
+
+  LIR_Opr result = gen->new_register(type);
+  value.load_item();
+  LIR_Opr value_opr = value.result();
+
+  if (access.is_oop()) {
+    value_opr = storeval_barrier(access.gen(), value_opr, access.access_emit_info(), access.decorators());
+  }
+
+  // Because we want a 2-arg form of xchg and xadd
+  __ move(value_opr, result);
+
+  assert(type == T_INT || type == T_OBJECT || type == T_ARRAY LP64_ONLY( || type == T_LONG ), "unexpected type");
+  __ xchg(access.resolved_addr(), result, result, LIR_OprFact::illegalOpr);
+
+  if (access.is_oop()) {
+    if (ShenandoahSATBBarrier) {
+      pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr,
+                  result /* pre_val */);
+    }
+  }
+
+  return result;
+}