--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -3647,6 +3647,12 @@
   srlx(r, LogMinObjAlignmentInBytes, r);
 }
 
+void MacroAssembler::encode_heap_oop_not_null(Register src, Register dst) {
+  assert (UseCompressedOops, "must be compressed");
+  sub(src, G6_heapbase, dst);
+  srlx(dst, LogMinObjAlignmentInBytes, dst);
+}
+
 // Same algorithm as oops.inline.hpp decode_heap_oop.
 void  MacroAssembler::decode_heap_oop(Register src, Register dst) {
   assert (UseCompressedOops, "must be compressed");
@@ -3665,6 +3671,14 @@
   add(r, G6_heapbase, r);
 }
 
+void  MacroAssembler::decode_heap_oop_not_null(Register src, Register dst) {
+  // Do not add assert code to this unless you change vtableStubs_sparc.cpp
+  // pd_code_size_limit.
+  assert (UseCompressedOops, "must be compressed");
+  sllx(src, LogMinObjAlignmentInBytes, dst);
+  add(dst, G6_heapbase, dst);
+}
+
 void MacroAssembler::reinit_heapbase() {
   if (UseCompressedOops) {
     // call indirectly to solve generation ordering problem

--- a/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/sparc/vm/assembler_sparc.hpp	Wed Apr 23 11:20:36 2008 -0700
@@ -1998,6 +1998,8 @@
   }
   void encode_heap_oop_not_null(Register r);
   void decode_heap_oop_not_null(Register r);
+  void encode_heap_oop_not_null(Register src, Register dst);
+  void decode_heap_oop_not_null(Register src, Register dst);
 
   // Support for managing the JavaThread pointer (i.e.; the reference to
   // thread-local information).

--- a/hotspot/src/cpu/sparc/vm/sparc.ad	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/sparc/vm/sparc.ad	Wed Apr 23 11:20:36 2008 -0700
@@ -5957,15 +5957,27 @@
 
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(iRegN dst, iRegP src) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (EncodeP src));
-  format %{ "SRL    $src,3,$dst\t encodeHeapOop" %}
+  format %{ "encode_heap_oop $src, $dst" %}
   ins_encode %{
     __ encode_heap_oop($src$$Register, $dst$$Register);
   %}
   ins_pipe(ialu_reg);
 %}
 
+instruct encodeHeapOop_not_null(iRegN dst, iRegP src) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (EncodeP src));
+  format %{ "encode_heap_oop_not_null $src, $dst" %}
+  ins_encode %{
+    __ encode_heap_oop_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
+
 instruct decodeHeapOop(iRegP dst, iRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (DecodeN src));
   format %{ "decode_heap_oop $src, $dst" %}
   ins_encode %{
@@ -5974,6 +5986,16 @@
   ins_pipe(ialu_reg);
 %}
 
+instruct decodeHeapOop_not_null(iRegP dst, iRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (DecodeN src));
+  format %{ "decode_heap_oop_not_null $src, $dst" %}
+  ins_encode %{
+    __ decode_heap_oop_not_null($src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg);
+%}
+
 
 // Store Zero into Aligned Packed Bytes
 instruct storeA8B0(memory mem, immI0 zero) %{

--- a/hotspot/src/cpu/x86/vm/assembler_x86_64.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/x86/vm/assembler_x86_64.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -4150,7 +4150,7 @@
   if (oop_result->is_valid()) {
     movq(oop_result, Address(r15_thread, JavaThread::vm_result_offset()));
     movptr(Address(r15_thread, JavaThread::vm_result_offset()), NULL_WORD);
-    verify_oop(oop_result);
+    verify_oop(oop_result, "broken oop in call_VM_base");
   }
 }
 
@@ -4689,6 +4689,10 @@
   popq(r12);
 }
 
+#ifndef PRODUCT
+extern "C" void findpc(intptr_t x);
+#endif
+
 void MacroAssembler::debug(char* msg, int64_t pc, int64_t regs[]) {
   // In order to get locks to work, we need to fake a in_VM state
   if (ShowMessageBoxOnError ) {
@@ -4707,6 +4711,11 @@
     if (os::message_box(msg, "Execution stopped, print registers?")) {
       ttyLocker ttyl;
       tty->print_cr("rip = 0x%016lx", pc);
+#ifndef PRODUCT
+      tty->cr();
+      findpc(pc);
+      tty->cr();
+#endif
       tty->print_cr("rax = 0x%016lx", regs[15]);
       tty->print_cr("rbx = 0x%016lx", regs[12]);
       tty->print_cr("rcx = 0x%016lx", regs[14]);
@@ -5187,7 +5196,7 @@
   bind(ok);
   popq(rscratch1);
 #endif
-  verify_oop(r);
+  verify_oop(r, "broken oop in encode_heap_oop");
   testq(r, r);
   cmovq(Assembler::equal, r, r12_heapbase);
   subq(r, r12_heapbase);
@@ -5203,11 +5212,28 @@
   stop("null oop passed to encode_heap_oop_not_null");
   bind(ok);
 #endif
-  verify_oop(r);
+  verify_oop(r, "broken oop in encode_heap_oop_not_null");
   subq(r, r12_heapbase);
   shrq(r, LogMinObjAlignmentInBytes);
 }
 
+void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
+  assert (UseCompressedOops, "should be compressed");
+#ifdef ASSERT
+  Label ok;
+  testq(src, src);
+  jcc(Assembler::notEqual, ok);
+  stop("null oop passed to encode_heap_oop_not_null2");
+  bind(ok);
+#endif
+  verify_oop(src, "broken oop in encode_heap_oop_not_null2");
+  if (dst != src) {
+    movq(dst, src);
+  }
+  subq(dst, r12_heapbase);
+  shrq(dst, LogMinObjAlignmentInBytes);
+}
+
 void  MacroAssembler::decode_heap_oop(Register r) {
   assert (UseCompressedOops, "should be compressed");
 #ifdef ASSERT
@@ -5232,7 +5258,7 @@
    leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
 #endif
   bind(done);
-  verify_oop(r);
+  verify_oop(r, "broken oop in decode_heap_oop");
 }
 
 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
@@ -5243,6 +5269,14 @@
   leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
 }
 
+void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
+  assert (UseCompressedOops, "should only be used for compressed headers");
+  // Cannot assert, unverified entry point counts instructions (see .ad file)
+  // vtableStubs also counts instructions in pd_code_size_limit.
+  assert(Address::times_8 == LogMinObjAlignmentInBytes, "decode alg wrong");
+  leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
+}
+
 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
   switch (cond) {
     // Note some conditions are synonyms for others

--- a/hotspot/src/cpu/x86/vm/assembler_x86_64.hpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/x86/vm/assembler_x86_64.hpp	Wed Apr 23 11:20:36 2008 -0700
@@ -1111,6 +1111,8 @@
   void decode_heap_oop(Register r);
   void encode_heap_oop_not_null(Register r);
   void decode_heap_oop_not_null(Register r);
+  void encode_heap_oop_not_null(Register dst, Register src);
+  void decode_heap_oop_not_null(Register dst, Register src);
 
   // Stack frame creation/removal
   void enter();

--- a/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/x86/vm/stubGenerator_x86_64.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -913,11 +913,12 @@
   // Stack after saving c_rarg3:
   //    [tos + 0]: saved c_rarg3
   //    [tos + 1]: saved c_rarg2
-  //    [tos + 2]: saved flags
-  //    [tos + 3]: return address
-  //  * [tos + 4]: error message (char*)
-  //  * [tos + 5]: object to verify (oop)
-  //  * [tos + 6]: saved rax - saved by caller and bashed
+  //    [tos + 2]: saved r12 (several TemplateTable methods use it)
+  //    [tos + 3]: saved flags
+  //    [tos + 4]: return address
+  //  * [tos + 5]: error message (char*)
+  //  * [tos + 6]: object to verify (oop)
+  //  * [tos + 7]: saved rax - saved by caller and bashed
   //  * = popped on exit
   address generate_verify_oop() {
     StubCodeMark mark(this, "StubRoutines", "verify_oop");
@@ -928,12 +929,24 @@
     __ pushfq();
     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 
+    __ pushq(r12);
+
     // save c_rarg2 and c_rarg3
     __ pushq(c_rarg2);
     __ pushq(c_rarg3);
 
+    enum {
+           // After previous pushes.
+           oop_to_verify = 6 * wordSize,
+           saved_rax     = 7 * wordSize,
+
+           // Before the call to MacroAssembler::debug(), see below.
+           return_addr   = 16 * wordSize,
+           error_msg     = 17 * wordSize
+    };
+
     // get object
-    __ movq(rax, Address(rsp, 5 * wordSize));
+    __ movq(rax, Address(rsp, oop_to_verify));
 
     // make sure object is 'reasonable'
     __ testq(rax, rax);
@@ -946,6 +959,9 @@
     __ cmpq(c_rarg2, c_rarg3);
     __ jcc(Assembler::notZero, error);
 
+    // set r12 to heapbase for load_klass()
+    __ reinit_heapbase();
+
     // make sure klass is 'reasonable'
     __ load_klass(rax, rax);  // get klass
     __ testq(rax, rax);
@@ -971,40 +987,45 @@
 
     // return if everything seems ok
     __ bind(exit);
-    __ movq(rax, Address(rsp, 6 * wordSize));    // get saved rax back
-    __ popq(c_rarg3);                              // restore c_rarg3
-    __ popq(c_rarg2);                              // restore c_rarg2
+    __ movq(rax, Address(rsp, saved_rax));       // get saved rax back
+    __ popq(c_rarg3);                            // restore c_rarg3
+    __ popq(c_rarg2);                            // restore c_rarg2
+    __ popq(r12);                                // restore r12
     __ popfq();                                  // restore flags
     __ ret(3 * wordSize);                        // pop caller saved stuff
 
     // handle errors
     __ bind(error);
-    __ movq(rax, Address(rsp, 6 * wordSize));    // get saved rax back
-    __ popq(c_rarg3);                              // get saved c_rarg3 back
-    __ popq(c_rarg2);                              // get saved c_rarg2 back
+    __ movq(rax, Address(rsp, saved_rax));       // get saved rax back
+    __ popq(c_rarg3);                            // get saved c_rarg3 back
+    __ popq(c_rarg2);                            // get saved c_rarg2 back
+    __ popq(r12);                                // get saved r12 back
     __ popfq();                                  // get saved flags off stack --
                                                  // will be ignored
 
     __ pushaq();                                 // push registers
                                                  // (rip is already
                                                  // already pushed)
-    // debug(char* msg, int64_t regs[])
+    // debug(char* msg, int64_t pc, int64_t regs[])
     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
     // pushed all the registers, so now the stack looks like:
     //     [tos +  0] 16 saved registers
     //     [tos + 16] return address
-    //     [tos + 17] error message (char*)
-
-    __ movq(c_rarg0, Address(rsp, 17 * wordSize)); // pass address of error message
-    __ movq(c_rarg1, rsp);                         // pass address of regs on stack
+    //   * [tos + 17] error message (char*)
+    //   * [tos + 18] object to verify (oop)
+    //   * [tos + 19] saved rax - saved by caller and bashed
+    //   * = popped on exit
+
+    __ movq(c_rarg0, Address(rsp, error_msg));   // pass address of error message
+    __ movq(c_rarg1, Address(rsp, return_addr)); // pass return address
+    __ movq(c_rarg2, rsp);                       // pass address of regs on stack
     __ movq(r12, rsp);                           // remember rsp
     __ subq(rsp, frame::arg_reg_save_area_bytes);// windows
     __ andq(rsp, -16);                           // align stack as required by ABI
     BLOCK_COMMENT("call MacroAssembler::debug");
     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug)));
     __ movq(rsp, r12);                           // restore rsp
-    __ reinit_heapbase();                        // r12 is heapbase
-    __ popaq();                                  // pop registers
+    __ popaq();                                  // pop registers (includes r12)
     __ ret(3 * wordSize);                        // pop caller saved stuff
 
     return start;
@@ -1038,7 +1059,7 @@
     assert_different_registers(Rtmp, Rint);
     __ movslq(Rtmp, Rint);
     __ cmpq(Rtmp, Rint);
-    __ jccb(Assembler::equal, L);
+    __ jcc(Assembler::equal, L);
     __ stop("high 32-bits of int value are not 0");
     __ bind(L);
 #endif

--- a/hotspot/src/cpu/x86/vm/x86_64.ad	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/cpu/x86/vm/x86_64.ad	Wed Apr 23 11:20:36 2008 -0700
@@ -6080,7 +6080,8 @@
   predicate(n->in(MemNode::Address)->bottom_type()->is_narrow());
 
   ins_cost(125); // XXX
-  format %{ "movl    $dst, $mem\t# compressed class" %}
+  format %{ "movl    $dst, $mem\t# compressed class\n\t"
+            "decode_heap_oop $dst,$dst" %}
   ins_encode %{
     Address addr = build_address($mem$$base, $mem$$index, $mem$$scale, $mem$$disp);
     Register dst = as_Register($dst$$reg);
@@ -6349,7 +6350,7 @@
 instruct loadConN0(rRegN dst, immN0 src, rFlagsReg cr) %{
   match(Set dst src);
   effect(KILL cr);
-  format %{ "xorq    $dst, $src\t# compressed ptr" %}
+  format %{ "xorq    $dst, $src\t# compressed NULL ptr" %}
   ins_encode %{
     Register dst = $dst$$Register;
     __ xorq(dst, dst);
@@ -6361,7 +6362,8 @@
   match(Set dst src);
 
   ins_cost(125);
-  format %{ "movl    $dst, $src\t# compressed ptr" %}
+  format %{ "movq    $dst, $src\t# compressed ptr\n\t"
+            "encode_heap_oop_not_null $dst,$dst" %}
   ins_encode %{
     address con = (address)$src$$constant;
     Register dst = $dst$$Register;
@@ -6996,6 +6998,7 @@
 
 // Convert oop pointer into compressed form
 instruct encodeHeapOop(rRegN dst, rRegP src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (EncodeP src));
   effect(KILL cr);
   format %{ "encode_heap_oop $dst,$src" %}
@@ -7010,7 +7013,21 @@
   ins_pipe(ialu_reg_long);
 %}
 
+instruct encodeHeapOop_not_null(rRegN dst, rRegP src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_narrowoop()->make_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (EncodeP src));
+  effect(KILL cr);
+  format %{ "encode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ encode_heap_oop_not_null(d, s);
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
 instruct decodeHeapOop(rRegP dst, rRegN src, rFlagsReg cr) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() != TypePtr::NotNull);
   match(Set dst (DecodeN src));
   effect(KILL cr);
   format %{ "decode_heap_oop $dst,$src" %}
@@ -7025,6 +7042,18 @@
   ins_pipe(ialu_reg_long);
 %}
 
+instruct decodeHeapOop_not_null(rRegP dst, rRegN src) %{
+  predicate(n->bottom_type()->is_oopptr()->ptr() == TypePtr::NotNull);
+  match(Set dst (DecodeN src));
+  format %{ "decode_heap_oop_not_null $dst,$src" %}
+  ins_encode %{
+    Register s = $src$$Register;
+    Register d = $dst$$Register;
+    __ decode_heap_oop_not_null(d, s);
+  %}
+  ins_pipe(ialu_reg_long);
+%}
+
 
 //----------Conditional Move---------------------------------------------------
 // Jump

--- a/hotspot/src/share/vm/opto/connode.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/opto/connode.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -563,6 +563,26 @@
   return this;
 }
 
+const Type *DecodeNNode::Value( PhaseTransform *phase ) const {
+  if (phase->type( in(1) ) == TypeNarrowOop::NULL_PTR) {
+    return TypePtr::NULL_PTR;
+  }
+  return bottom_type();
+}
+
+Node* DecodeNNode::decode(PhaseGVN* phase, Node* value) {
+  if (value->Opcode() == Op_EncodeP) {
+    // (DecodeN (EncodeP p)) -> p
+    return value->in(1);
+  }
+  const Type* newtype = value->bottom_type();
+  if (newtype == TypeNarrowOop::NULL_PTR) {
+    return phase->transform(new (phase->C, 1) ConPNode(TypePtr::NULL_PTR));
+  } else {
+    return phase->transform(new (phase->C, 2) DecodeNNode(value, newtype->is_narrowoop()->make_oopptr()));
+  }
+}
+
 Node* EncodePNode::Identity(PhaseTransform* phase) {
   const Type *t = phase->type( in(1) );
   if( t == Type::TOP ) return in(1);
@@ -574,14 +594,26 @@
   return this;
 }
 
+const Type *EncodePNode::Value( PhaseTransform *phase ) const {
+  if (phase->type( in(1) ) == TypePtr::NULL_PTR) {
+    return TypeNarrowOop::NULL_PTR;
+  }
+  return bottom_type();
+}
 
 Node* EncodePNode::encode(PhaseGVN* phase, Node* value) {
+  if (value->Opcode() == Op_DecodeN) {
+    // (EncodeP (DecodeN p)) -> p
+    return value->in(1);
+  }
   const Type* newtype = value->bottom_type();
   if (newtype == TypePtr::NULL_PTR) {
     return phase->transform(new (phase->C, 1) ConNNode(TypeNarrowOop::NULL_PTR));
+  } else if (newtype->isa_oopptr()) {
+    return phase->transform(new (phase->C, 2) EncodePNode(value, newtype->is_oopptr()->make_narrowoop()));
   } else {
-    return phase->transform(new (phase->C, 2) EncodePNode(value,
-                                                          newtype->is_oopptr()->make_narrowoop()));
+    ShouldNotReachHere();
+    return NULL; // to make C++ compiler happy.
   }
 }
 

--- a/hotspot/src/share/vm/opto/connode.hpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/opto/connode.hpp	Wed Apr 23 11:20:36 2008 -0700
@@ -282,6 +282,7 @@
   }
   virtual int Opcode() const;
   virtual Node *Identity( PhaseTransform *phase );
+  virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegN; }
 
   static Node* encode(PhaseGVN* phase, Node* value);
@@ -300,7 +301,10 @@
   }
   virtual int Opcode() const;
   virtual Node *Identity( PhaseTransform *phase );
+  virtual const Type *Value( PhaseTransform *phase ) const;
   virtual uint  ideal_reg() const { return Op_RegP; }
+
+  static Node* decode(PhaseGVN* phase, Node* value);
 };
 
 //------------------------------Conv2BNode-------------------------------------

--- a/hotspot/src/share/vm/opto/escape.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/opto/escape.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -215,6 +215,10 @@
   VectorSet visited(Thread::current()->resource_area());
   GrowableArray<uint>  worklist;
 
+#ifdef ASSERT
+  Node *orig_n = n;
+#endif
+
   n = n->uncast();
   PointsToNode  npt = _nodes->at_grow(n->_idx);
 
@@ -223,8 +227,14 @@
     ptset.set(n->_idx);
     return;
   }
-  assert(npt._node != NULL, "unregistered node");
-
+#ifdef ASSERT
+  if (npt._node == NULL) {
+    if (orig_n != n)
+      orig_n->dump();
+    n->dump();
+    assert(npt._node != NULL, "unregistered node");
+  }
+#endif
   worklist.push(n->_idx);
   while(worklist.length() > 0) {
     int ni = worklist.pop();
@@ -266,7 +276,7 @@
   PointsToNode *ptn = ptnode_adr(ni);
 
   // Mark current edges as visited and move deferred edges to separate array.
-  for (; i < ptn->edge_count(); i++) {
+  while (i < ptn->edge_count()) {
     uint t = ptn->edge_target(i);
 #ifdef ASSERT
     assert(!visited->test_set(t), "expecting no duplications");
@@ -276,6 +286,8 @@
     if (ptn->edge_type(i) == PointsToNode::DeferredEdge) {
       ptn->remove_edge(t, PointsToNode::DeferredEdge);
       deferred_edges->append(t);
+    } else {
+      i++;
     }
   }
   for (int next = 0; next < deferred_edges->length(); ++next) {
@@ -1716,6 +1728,8 @@
     }
     case Op_CastPP:
     case Op_CheckCastPP:
+    case Op_EncodeP:
+    case Op_DecodeN:
     {
       add_node(n, PointsToNode::LocalVar, PointsToNode::UnknownEscape, false);
       int ti = n->in(1)->_idx;
@@ -1743,12 +1757,6 @@
       add_node(n, PointsToNode::JavaObject, es, true);
       break;
     }
-    case Op_CreateEx:
-    {
-      // assume that all exception objects globally escape
-      add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
-      break;
-    }
     case Op_ConN:
     {
       // assume all narrow oop constants globally escape except for null
@@ -1761,6 +1769,12 @@
       add_node(n, PointsToNode::JavaObject, es, true);
       break;
     }
+    case Op_CreateEx:
+    {
+      // assume that all exception objects globally escape
+      add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
+      break;
+    }
     case Op_LoadKlass:
     {
       add_node(n, PointsToNode::JavaObject, PointsToNode::GlobalEscape, true);
@@ -1976,10 +1990,11 @@
       break;
     }
     case Op_LoadP:
+    case Op_LoadN:
     {
       const Type *t = phase->type(n);
 #ifdef ASSERT
-      if (t->isa_ptr() == NULL)
+      if (!t->isa_narrowoop() && t->isa_ptr() == NULL)
         assert(false, "Op_LoadP");
 #endif
 
@@ -2060,11 +2075,16 @@
       break;
     }
     case Op_StoreP:
+    case Op_StoreN:
     case Op_StorePConditional:
     case Op_CompareAndSwapP:
+    case Op_CompareAndSwapN:
     {
       Node *adr = n->in(MemNode::Address);
       const Type *adr_type = phase->type(adr);
+      if (adr_type->isa_narrowoop()) {
+        adr_type = adr_type->is_narrowoop()->make_oopptr();
+      }
 #ifdef ASSERT
       if (!adr_type->isa_oopptr())
         assert(phase->type(adr) == TypeRawPtr::NOTNULL, "Op_StoreP");

--- a/hotspot/src/share/vm/opto/macro.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/opto/macro.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -584,7 +584,7 @@
 
       const Type *field_type;
       // The next code is taken from Parse::do_get_xxx().
-      if (basic_elem_type == T_OBJECT) {
+      if (basic_elem_type == T_OBJECT || basic_elem_type == T_ARRAY) {
         if (!elem_type->is_loaded()) {
           field_type = TypeInstPtr::BOTTOM;
         } else if (field != NULL && field->is_constant()) {
@@ -597,6 +597,10 @@
         } else {
           field_type = TypeOopPtr::make_from_klass(elem_type->as_klass());
         }
+        if (UseCompressedOops) {
+          field_type = field_type->is_oopptr()->make_narrowoop();
+          basic_elem_type = T_NARROWOOP;
+        }
       } else {
         field_type = Type::get_const_basic_type(basic_elem_type);
       }
@@ -659,6 +663,13 @@
 #endif
         return false;
       }
+      if (UseCompressedOops && field_type->isa_narrowoop()) {
+        // Enable "DecodeN(EncodeP(Allocate)) --> Allocate" transformation
+        // to be able scalar replace the allocation.
+        _igvn.set_delay_transform(false);
+        field_val = DecodeNNode::decode(&_igvn, field_val);
+        _igvn.set_delay_transform(true);
+      }
       sfpt->add_req(field_val);
     }
     JVMState *jvms = sfpt->jvms();

--- a/hotspot/src/share/vm/opto/memnode.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/opto/memnode.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -754,13 +754,12 @@
       const TypeNarrowOop* narrowtype;
       if (rt->isa_narrowoop()) {
         narrowtype = rt->is_narrowoop();
-        rt = narrowtype->make_oopptr();
       } else {
         narrowtype = rt->is_oopptr()->make_narrowoop();
       }
       Node* load  = gvn.transform(new (C, 3) LoadNNode(ctl, mem, adr, adr_type, narrowtype));
 
-      return new (C, 2) DecodeNNode(load, rt);
+      return DecodeNNode::decode(&gvn, load);
     } else
 #endif
       {
@@ -1841,15 +1840,7 @@
         (UseCompressedOops && val->bottom_type()->isa_klassptr() &&
          adr->bottom_type()->isa_rawptr())) {
       const TypePtr* type = val->bottom_type()->is_ptr();
-      Node* cp;
-      if (type->isa_oopptr()) {
-        const TypeNarrowOop* etype = type->is_oopptr()->make_narrowoop();
-        cp = gvn.transform(new (C, 2) EncodePNode(val, etype));
-      } else if (type == TypePtr::NULL_PTR) {
-        cp = gvn.transform(new (C, 1) ConNNode(TypeNarrowOop::NULL_PTR));
-      } else {
-        ShouldNotReachHere();
-      }
+      Node* cp = EncodePNode::encode(&gvn, val);
       return new (C, 4) StoreNNode(ctl, mem, adr, adr_type, cp);
     } else
 #endif

--- a/hotspot/src/share/vm/runtime/sharedRuntime.cpp	Thu Apr 17 07:16:03 2008 -0700
+++ b/hotspot/src/share/vm/runtime/sharedRuntime.cpp	Wed Apr 23 11:20:36 2008 -0700
@@ -2176,6 +2176,8 @@
 #ifndef PRODUCT
 bool AdapterHandlerLibrary::contains(CodeBlob* b) {
 
+  if (_handlers == NULL) return false;
+
   for (int i = 0 ; i < _handlers->length() ; i++) {
     AdapterHandlerEntry* a = get_entry(i);
     if ( a != NULL && b == CodeCache::find_blob(a->get_i2c_entry()) ) return true;