--- a/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/concurrentMark.cpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -1171,12 +1171,12 @@
     if (G1TraceMarkStackOverflow)
       gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
   } else {
+    SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
     // We're done with marking.
     // This is the end of  the marking cycle, we're expected all
     // threads to have SATB queues with active set to true.
-    JavaThread::satb_mark_queue_set().set_active_all_threads(
-                                                  false, /* new active value */
-                                                  true /* expected_active */);
+    satb_mq_set.set_active_all_threads(false, /* new active value */
+                                       true /* expected_active */);
 
     if (VerifyDuringGC) {
       HandleMark hm;  // handle scope

--- a/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.inline.hpp	Wed Jan 19 09:35:17 2011 -0500
@@ -28,7 +28,7 @@
 #include "gc_implementation/g1/concurrentMark.hpp"
 #include "gc_implementation/g1/g1CollectedHeap.hpp"
 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
-#include "gc_implementation/g1/heapRegionSeq.hpp"
+#include "gc_implementation/g1/heapRegionSeq.inline.hpp"
 #include "utilities/taskqueue.hpp"
 
 // Inline functions for G1CollectedHeap

--- a/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1_globals.hpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -99,6 +99,13 @@
   develop(intx, G1SATBProcessCompletedThreshold, 20,                        \
           "Number of completed buffers that triggers log processing.")      \
                                                                             \
+  product(uintx, G1SATBBufferEnqueueingThresholdPercent, 60,                \
+          "Before enqueueing them, each mutator thread tries to do some "   \
+          "filtering on the SATB buffers it generates. If post-filtering "  \
+          "the percentage of retained entries is over this threshold "      \
+          "the buffer will be enqueued for processing. A value of 0 "       \
+          "specifies that mutator threads should not do such filtering.")   \
+                                                                            \
   develop(intx, G1ExtraRegionSurvRate, 33,                                  \
           "If the young survival rate is S, and there's room left in "      \
           "to-space, we will allow regions whose survival rate is up to "   \

--- a/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/heapRegionSeq.hpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -106,7 +106,7 @@
 
   // If "addr" falls within a region in the sequence, return that region,
   // or else NULL.
-  HeapRegion* addr_to_region(const void* addr);
+  inline HeapRegion* addr_to_region(const void* addr);
 
   void print();
 

--- a/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.cpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,8 +38,8 @@
 # include "thread_windows.inline.hpp"
 #endif
 
-PtrQueue::PtrQueue(PtrQueueSet* qset_, bool perm, bool active) :
-  _qset(qset_), _buf(NULL), _index(0), _active(active),
+PtrQueue::PtrQueue(PtrQueueSet* qset, bool perm, bool active) :
+  _qset(qset), _buf(NULL), _index(0), _active(active),
   _perm(perm), _lock(NULL)
 {}
 
@@ -153,10 +153,16 @@
 }
 
 void PtrQueue::handle_zero_index() {
-  assert(0 == _index, "Precondition.");
+  assert(_index == 0, "Precondition.");
+
   // This thread records the full buffer and allocates a new one (while
   // holding the lock if there is one).
   if (_buf != NULL) {
+    if (!should_enqueue_buffer()) {
+      assert(_index > 0, "the buffer can only be re-used if it's not full");
+      return;
+    }
+
     if (_lock) {
       assert(_lock->owned_by_self(), "Required.");
 

--- a/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/ptrQueue.hpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -68,7 +68,7 @@
 public:
   // Initialize this queue to contain a null buffer, and be part of the
   // given PtrQueueSet.
-  PtrQueue(PtrQueueSet*, bool perm = false, bool active = false);
+  PtrQueue(PtrQueueSet* qset, bool perm = false, bool active = false);
   // Release any contained resources.
   void flush();
   // Calls flush() when destroyed.
@@ -85,6 +85,14 @@
     else enqueue_known_active(ptr);
   }
 
+  // This method is called when we're doing the zero index handling
+  // and gives a chance to the queues to do any pre-enqueueing
+  // processing they might want to do on the buffer. It should return
+  // true if the buffer should be enqueued, or false if enough
+  // entries were cleared from it so that it can be re-used. It should
+  // not return false if the buffer is still full (otherwise we can
+  // get into an infinite loop).
+  virtual bool should_enqueue_buffer() { return true; }
   void handle_zero_index();
   void locking_enqueue_completed_buffer(void** buf);
 

--- a/hotspot/src/share/vm/gc_implementation/g1/satbQueue.cpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/satbQueue.cpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,12 +23,98 @@
  */
 
 #include "precompiled.hpp"
+#include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
 #include "gc_implementation/g1/satbQueue.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/sharedHeap.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/thread.hpp"
 
+// This method removes entries from an SATB buffer that will not be
+// useful to the concurrent marking threads. An entry is removed if it
+// satisfies one of the following conditions:
+//
+// * it points to an object outside the G1 heap (G1's concurrent
+//     marking only visits objects inside the G1 heap),
+// * it points to an object that has been allocated since marking
+//     started (according to SATB those objects do not need to be
+//     visited during marking), or
+// * it points to an object that has already been marked (no need to
+//     process it again).
+//
+// The rest of the entries will be retained and are compacted towards
+// the top of the buffer. If with this filtering we clear a large
+// enough chunk of the buffer we can re-use it (instead of enqueueing
+// it) and we can just allow the mutator to carry on executing.
+
+bool ObjPtrQueue::should_enqueue_buffer() {
+  assert(_lock == NULL || _lock->owned_by_self(),
+         "we should have taken the lock before calling this");
+
+  // A value of 0 means "don't filter SATB buffers".
+  if (G1SATBBufferEnqueueingThresholdPercent == 0) {
+    return true;
+  }
+
+  G1CollectedHeap* g1h = G1CollectedHeap::heap();
+
+  // This method should only be called if there is a non-NULL buffer
+  // that is full.
+  assert(_index == 0, "pre-condition");
+  assert(_buf != NULL, "pre-condition");
+
+  void** buf = _buf;
+  size_t sz = _sz;
+
+  // Used for sanity checking at the end of the loop.
+  debug_only(size_t entries = 0; size_t retained = 0;)
+
+  size_t i = sz;
+  size_t new_index = sz;
+
+  // Given that we are expecting _index == 0, we could have changed
+  // the loop condition to (i > 0). But we are using _index for
+  // generality.
+  while (i > _index) {
+    assert(i > 0, "we should have at least one more entry to process");
+    i -= oopSize;
+    debug_only(entries += 1;)
+    oop* p = (oop*) &buf[byte_index_to_index((int) i)];
+    oop obj = *p;
+    // NULL the entry so that unused parts of the buffer contain NULLs
+    // at the end. If we are going to retain it we will copy it to its
+    // final place. If we have retained all entries we have visited so
+    // far, we'll just end up copying it to the same place.
+    *p = NULL;
+
+    bool retain = g1h->is_obj_ill(obj);
+    if (retain) {
+      assert(new_index > 0, "we should not have already filled up the buffer");
+      new_index -= oopSize;
+      assert(new_index >= i,
+             "new_index should never be below i, as we alwaysr compact 'up'");
+      oop* new_p = (oop*) &buf[byte_index_to_index((int) new_index)];
+      assert(new_p >= p, "the destination location should never be below "
+             "the source as we always compact 'up'");
+      assert(*new_p == NULL,
+             "we should have already cleared the destination location");
+      *new_p = obj;
+      debug_only(retained += 1;)
+    }
+  }
+  size_t entries_calc = (sz - _index) / oopSize;
+  assert(entries == entries_calc, "the number of entries we counted "
+         "should match the number of entries we calculated");
+  size_t retained_calc = (sz - new_index) / oopSize;
+  assert(retained == retained_calc, "the number of retained entries we counted "
+         "should match the number of retained entries we calculated");
+  size_t perc = retained_calc * 100 / entries_calc;
+  bool should_enqueue = perc > (size_t) G1SATBBufferEnqueueingThresholdPercent;
+  _index = new_index;
+
+  return should_enqueue;
+}
+
 void ObjPtrQueue::apply_closure(ObjectClosure* cl) {
   if (_buf != NULL) {
     apply_closure_to_buffer(cl, _buf, _index, _sz);

--- a/hotspot/src/share/vm/gc_implementation/g1/satbQueue.hpp	Thu Jan 13 17:19:21 2011 -0800
+++ b/hotspot/src/share/vm/gc_implementation/g1/satbQueue.hpp	Wed Jan 19 09:35:17 2011 -0500
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,13 +33,18 @@
 // A ptrQueue whose elements are "oops", pointers to object heads.
 class ObjPtrQueue: public PtrQueue {
 public:
-  ObjPtrQueue(PtrQueueSet* qset_, bool perm = false) :
+  ObjPtrQueue(PtrQueueSet* qset, bool perm = false) :
     // SATB queues are only active during marking cycles. We create
     // them with their active field set to false. If a thread is
     // created during a cycle and its SATB queue needs to be activated
     // before the thread starts running, we'll need to set its active
     // field to true. This is done in JavaThread::initialize_queues().
-    PtrQueue(qset_, perm, false /* active */) { }
+    PtrQueue(qset, perm, false /* active */) { }
+
+  // Overrides PtrQueue::should_enqueue_buffer(). See the method's
+  // definition for more information.
+  virtual bool should_enqueue_buffer();
+
   // Apply the closure to all elements, and reset the index to make the
   // buffer empty.
   void apply_closure(ObjectClosure* cl);