--- a/hotspot/src/cpu/sparc/vm/vm_version_sparc.cpp	Thu Apr 22 13:23:15 2010 -0700
+++ b/hotspot/src/cpu/sparc/vm/vm_version_sparc.cpp	Mon May 03 10:24:51 2010 -0700
@@ -104,6 +104,12 @@
     if (FLAG_IS_DEFAULT(OptoLoopAlignment)) {
       FLAG_SET_DEFAULT(OptoLoopAlignment, 4);
     }
+    // When using CMS, we cannot use memset() in BOT updates because
+    // the sun4v/CMT version in libc_psr uses BIS which exposes
+    // "phantom zeros" to concurrent readers. See 6948537.
+    if (FLAG_IS_DEFAULT(UseMemSetInBOT) && UseConcMarkSweepGC) {
+      FLAG_SET_DEFAULT(UseMemSetInBOT, false);
+    }
   }
 
   // Use hardware population count instruction if available.

--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Thu Apr 22 13:23:15 2010 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp	Mon May 03 10:24:51 2010 -0700
@@ -789,6 +789,14 @@
   _gc_counters = new CollectorCounters("CMS", 1);
   _completed_initialization = true;
   _inter_sweep_timer.start();  // start of time
+#ifdef SPARC
+  // Issue a stern warning, but allow use for experimentation and debugging.
+  if (VM_Version::is_sun4v() && UseMemSetInBOT) {
+    assert(!FLAG_IS_DEFAULT(UseMemSetInBOT), "Error");
+    warning("Experimental flag -XX:+UseMemSetInBOT is known to cause instability"
+            " on sun4v; please understand that you are using at your own risk!");
+  }
+#endif
 }
 
 const char* ConcurrentMarkSweepGeneration::name() const {

--- a/hotspot/src/share/vm/memory/blockOffsetTable.hpp	Thu Apr 22 13:23:15 2010 -0700
+++ b/hotspot/src/share/vm/memory/blockOffsetTable.hpp	Mon May 03 10:24:51 2010 -0700
@@ -140,14 +140,38 @@
            "right address out of range");
     assert(left  < right, "Heap addresses out of order");
     size_t num_cards = pointer_delta(right, left) >> LogN_words;
-    memset(&_offset_array[index_for(left)], offset, num_cards);
+
+    // Below, we may use an explicit loop instead of memset()
+    // because on certain platforms memset() can give concurrent
+    // readers "out-of-thin-air," phantom zeros; see 6948537.
+    if (UseMemSetInBOT) {
+      memset(&_offset_array[index_for(left)], offset, num_cards);
+    } else {
+      size_t i = index_for(left);
+      const size_t end = i + num_cards;
+      for (; i < end; i++) {
+        _offset_array[i] = offset;
+      }
+    }
   }
 
   void set_offset_array(size_t left, size_t right, u_char offset) {
     assert(right < _vs.committed_size(), "right address out of range");
     assert(left  <= right, "indexes out of order");
     size_t num_cards = right - left + 1;
-    memset(&_offset_array[left], offset, num_cards);
+
+    // Below, we may use an explicit loop instead of memset
+    // because on certain platforms memset() can give concurrent
+    // readers "out-of-thin-air," phantom zeros; see 6948537.
+    if (UseMemSetInBOT) {
+      memset(&_offset_array[left], offset, num_cards);
+    } else {
+      size_t i = left;
+      const size_t end = i + num_cards;
+      for (; i < end; i++) {
+        _offset_array[i] = offset;
+      }
+    }
   }
 
   void check_offset_array(size_t index, HeapWord* high, HeapWord* low) const {

--- a/hotspot/src/share/vm/runtime/globals.hpp	Thu Apr 22 13:23:15 2010 -0700
+++ b/hotspot/src/share/vm/runtime/globals.hpp	Mon May 03 10:24:51 2010 -0700
@@ -327,6 +327,10 @@
   product(bool, UseMembar, false,                                           \
           "(Unstable) Issues membars on thread state transitions")          \
                                                                             \
+  /* Temporary: See 6948537 */                                             \
+  experimental(bool, UseMemSetInBOT, true,                                  \
+          "(Unstable) uses memset in BOT updates in GC code")               \
+                                                                            \
   diagnostic(bool, UnlockDiagnosticVMOptions, trueInDebug,                  \
           "Enable normal processing of flags relating to field diagnostics")\
                                                                             \