--- a/hotspot/src/share/vm/gc/cms/parNewGeneration.cpp Tue Feb 23 09:52:46 2016 +0100
+++ b/hotspot/src/share/vm/gc/cms/parNewGeneration.cpp Tue Feb 23 14:14:29 2016 +0100
@@ -39,6 +39,7 @@
#include "gc/shared/genOopClosures.inline.hpp"
#include "gc/shared/generation.hpp"
#include "gc/shared/plab.inline.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/space.hpp"
#include "gc/shared/spaceDecorator.hpp"
@@ -64,6 +65,7 @@
int thread_num_,
ObjToScanQueueSet* work_queue_set_,
Stack<oop, mtGC>* overflow_stacks_,
+ PreservedMarks* preserved_marks_,
size_t desired_plab_sz_,
ParallelTaskTerminator& term_) :
_to_space(to_space_),
@@ -73,6 +75,7 @@
_work_queue(work_queue_set_->queue(thread_num_)),
_to_space_full(false),
_overflow_stack(overflow_stacks_ ? overflow_stacks_ + thread_num_ : NULL),
+ _preserved_marks(preserved_marks_),
_ageTable(false), // false ==> not the global age table, no perf data.
_to_space_alloc_buffer(desired_plab_sz_),
_to_space_closure(young_gen_, this),
@@ -286,6 +289,7 @@
Generation& old_gen,
ObjToScanQueueSet& queue_set,
Stack<oop, mtGC>* overflow_stacks_,
+ PreservedMarksSet& preserved_marks_set,
size_t desired_plab_sz,
ParallelTaskTerminator& term);
@@ -322,6 +326,7 @@
Generation& old_gen,
ObjToScanQueueSet& queue_set,
Stack<oop, mtGC>* overflow_stacks,
+ PreservedMarksSet& preserved_marks_set,
size_t desired_plab_sz,
ParallelTaskTerminator& term)
: ResourceArray(sizeof(ParScanThreadState), num_threads),
@@ -336,7 +341,8 @@
for (int i = 0; i < num_threads; ++i) {
new ((ParScanThreadState*)_data + i)
ParScanThreadState(&to_space, &young_gen, &old_gen, i, &queue_set,
- overflow_stacks, desired_plab_sz, term);
+ overflow_stacks, preserved_marks_set.get(i),
+ desired_plab_sz, term);
}
}
@@ -905,12 +911,16 @@
// Set the correct parallelism (number of queues) in the reference processor
ref_processor()->set_active_mt_degree(active_workers);
+ // Need to initialize the preserved marks before the ThreadStateSet c'tor.
+ _preserved_marks_set.init(active_workers);
+
// Always set the terminator for the active number of workers
// because only those workers go through the termination protocol.
ParallelTaskTerminator _term(active_workers, task_queues());
ParScanThreadStateSet thread_state_set(active_workers,
*to(), *this, *_old_gen, *task_queues(),
- _overflow_stacks, desired_plab_sz(), _term);
+ _overflow_stacks, _preserved_marks_set,
+ desired_plab_sz(), _term);
thread_state_set.reset(active_workers, promotion_failed());
@@ -993,6 +1003,7 @@
} else {
handle_promotion_failed(gch, thread_state_set);
}
+ _preserved_marks_set.reclaim();
// set new iteration safe limit for the survivor spaces
from()->set_concurrent_iteration_safe_limit(from()->top());
to()->set_concurrent_iteration_safe_limit(to()->top());
@@ -1070,15 +1081,6 @@
return forward_ptr;
}
-void ParNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) {
- if (m->must_be_preserved_for_promotion_failure(obj)) {
- // We should really have separate per-worker stacks, rather
- // than use locking of a common pair of stacks.
- MutexLocker ml(ParGCRareEvent_lock);
- preserve_mark(obj, m);
- }
-}
-
// Multiple GC threads may try to promote an object. If the object
// is successfully promoted, a forwarding pointer will be installed in
// the object in the young generation. This method claims the right
@@ -1136,7 +1138,7 @@
_promotion_failed = true;
new_obj = old;
- preserve_mark_if_necessary(old, m);
+ par_scan_state->preserved_marks()->push_if_necessary(old, m);
par_scan_state->register_promotion_failure(sz);
}
--- a/hotspot/src/share/vm/gc/cms/parNewGeneration.hpp Tue Feb 23 09:52:46 2016 +0100
+++ b/hotspot/src/share/vm/gc/cms/parNewGeneration.hpp Tue Feb 23 14:14:29 2016 +0100
@@ -30,6 +30,7 @@
#include "gc/shared/copyFailedInfo.hpp"
#include "gc/shared/gcTrace.hpp"
#include "gc/shared/plab.hpp"
+#include "gc/shared/preservedMarks.hpp"
#include "gc/shared/taskqueue.hpp"
#include "memory/padded.hpp"
@@ -65,6 +66,7 @@
private:
ObjToScanQueue *_work_queue;
Stack<oop, mtGC>* const _overflow_stack;
+ PreservedMarks* const _preserved_marks;
PLAB _to_space_alloc_buffer;
@@ -128,6 +130,7 @@
Generation* old_gen_, int thread_num_,
ObjToScanQueueSet* work_queue_set_,
Stack<oop, mtGC>* overflow_stacks_,
+ PreservedMarks* preserved_marks_,
size_t desired_plab_sz_,
ParallelTaskTerminator& term_);
@@ -136,6 +139,8 @@
ObjToScanQueue* work_queue() { return _work_queue; }
+ PreservedMarks* preserved_marks() const { return _preserved_marks; }
+
PLAB* to_space_alloc_buffer() {
return &_to_space_alloc_buffer;
}
@@ -331,10 +336,6 @@
static oop real_forwardee_slow(oop obj);
static void waste_some_time();
- // Preserve the mark of "obj", if necessary, in preparation for its mark
- // word being overwritten with a self-forwarding-pointer.
- void preserve_mark_if_necessary(oop obj, markOop m);
-
void handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set);
protected:
--- a/hotspot/src/share/vm/gc/g1/g1EvacFailure.hpp Tue Feb 23 09:52:46 2016 +0100
+++ b/hotspot/src/share/vm/gc/g1/g1EvacFailure.hpp Tue Feb 23 14:14:29 2016 +0100
@@ -27,25 +27,12 @@
#include "gc/g1/g1OopClosures.hpp"
#include "gc/g1/heapRegionManager.hpp"
+ #include "gc/shared/preservedMarks.hpp"
#include "gc/shared/workgroup.hpp"
#include "utilities/globalDefinitions.hpp"
class G1CollectedHeap;
-class OopAndMarkOop {
- oop _o;
- markOop _m;
- public:
- OopAndMarkOop(oop obj, markOop m) : _o(obj), _m(m) {
- }
-
- void set_mark() {
- _o->set_mark(_m);
- }
-};
-
-typedef Stack<OopAndMarkOop,mtGC> OopAndMarkOopStack;
-
// Task to fixup self-forwarding pointers
// installed as a result of an evacuation failure.
class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask {
--- a/hotspot/src/share/vm/gc/serial/defNewGeneration.cpp Tue Feb 23 09:52:46 2016 +0100
+++ b/hotspot/src/share/vm/gc/serial/defNewGeneration.cpp Tue Feb 23 14:14:29 2016 +0100
@@ -36,6 +36,7 @@
#include "gc/shared/genCollectedHeap.hpp"
#include "gc/shared/genOopClosures.inline.hpp"
#include "gc/shared/generationSpec.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
#include "gc/shared/referencePolicy.hpp"
#include "gc/shared/space.inline.hpp"
#include "gc/shared/spaceDecorator.hpp"
@@ -184,6 +185,7 @@
size_t initial_size,
const char* policy)
: Generation(rs, initial_size),
+ _preserved_marks_set(false /* in_c_heap */),
_promo_failure_drain_in_progress(false),
_should_allocate_from_space(false)
{
@@ -602,6 +604,8 @@
age_table()->clear();
to()->clear(SpaceDecorator::Mangle);
+ // The preserved marks should be empty at the start of the GC.
+ _preserved_marks_set.init(1);
gch->rem_set()->prepare_for_younger_refs_iterate(false);
@@ -704,6 +708,8 @@
// Reset the PromotionFailureALot counters.
NOT_PRODUCT(gch->reset_promotion_should_fail();)
}
+ // We should have processed and cleared all the preserved marks.
+ _preserved_marks_set.reclaim();
// set new iteration safe limit for the survivor spaces
from()->set_concurrent_iteration_safe_limit(from()->top());
to()->set_concurrent_iteration_safe_limit(to()->top());
@@ -721,13 +727,6 @@
gc_tracer.report_gc_end(_gc_timer->gc_end(), _gc_timer->time_partitions());
}
-class RemoveForwardPointerClosure: public ObjectClosure {
-public:
- void do_object(oop obj) {
- obj->init_mark();
- }
-};
-
void DefNewGeneration::init_assuming_no_promotion_failure() {
_promotion_failed = false;
_promotion_failed_info.reset();
@@ -735,33 +734,12 @@
}
void DefNewGeneration::remove_forwarding_pointers() {
- RemoveForwardPointerClosure rspc;
+ RemoveForwardedPointerClosure rspc;
eden()->object_iterate(&rspc);
from()->object_iterate(&rspc);
// Now restore saved marks, if any.
- assert(_objs_with_preserved_marks.size() == _preserved_marks_of_objs.size(),
- "should be the same");
- while (!_objs_with_preserved_marks.is_empty()) {
- oop obj = _objs_with_preserved_marks.pop();
- markOop m = _preserved_marks_of_objs.pop();
- obj->set_mark(m);
- }
- _objs_with_preserved_marks.clear(true);
- _preserved_marks_of_objs.clear(true);
-}
-
-void DefNewGeneration::preserve_mark(oop obj, markOop m) {
- assert(_promotion_failed && m->must_be_preserved_for_promotion_failure(obj),
- "Oversaving!");
- _objs_with_preserved_marks.push(obj);
- _preserved_marks_of_objs.push(m);
-}
-
-void DefNewGeneration::preserve_mark_if_necessary(oop obj, markOop m) {
- if (m->must_be_preserved_for_promotion_failure(obj)) {
- preserve_mark(obj, m);
- }
+ _preserved_marks_set.restore();
}
void DefNewGeneration::handle_promotion_failure(oop old) {
@@ -769,7 +747,7 @@
_promotion_failed = true;
_promotion_failed_info.register_copy_failure(old->size());
- preserve_mark_if_necessary(old, old->mark());
+ _preserved_marks_set.get()->push_if_necessary(old, old->mark());
// forward to self
old->forward_to(old);
--- a/hotspot/src/share/vm/gc/serial/defNewGeneration.hpp Tue Feb 23 09:52:46 2016 +0100
+++ b/hotspot/src/share/vm/gc/serial/defNewGeneration.hpp Tue Feb 23 14:14:29 2016 +0100
@@ -30,6 +30,7 @@
#include "gc/shared/copyFailedInfo.hpp"
#include "gc/shared/generation.hpp"
#include "gc/shared/generationCounters.hpp"
+#include "gc/shared/preservedMarks.hpp"
#include "utilities/stack.hpp"
class ContiguousSpace;
@@ -87,15 +88,8 @@
// therefore we must remove their forwarding pointers.
void remove_forwarding_pointers();
- // Preserve the mark of "obj", if necessary, in preparation for its mark
- // word being overwritten with a self-forwarding-pointer.
- void preserve_mark_if_necessary(oop obj, markOop m);
- void preserve_mark(oop obj, markOop m); // work routine used by the above
-
- // Together, these keep <object with a preserved mark, mark value> pairs.
- // They should always contain the same number of elements.
- Stack<oop, mtGC> _objs_with_preserved_marks;
- Stack<markOop, mtGC> _preserved_marks_of_objs;
+ // Preserved marks
+ PreservedMarksSet _preserved_marks_set;
// Promotion failure handling
ExtendedOopClosure *_promo_failure_scan_stack_closure;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc/shared/preservedMarks.cpp Tue Feb 23 14:14:29 2016 +0100
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/preservedMarks.inline.hpp"
+#include "memory/allocation.inline.hpp"
+#include "oops/oop.inline.hpp"
+
+void PreservedMarks::restore() {
+ // First, iterate over the stack and restore all marks.
+ StackIterator<OopAndMarkOop, mtGC> iter(_stack);
+ while (!iter.is_empty()) {
+ OopAndMarkOop elem = iter.next();
+ elem.set_mark();
+ }
+
+ // Second, reclaim all the stack memory
+ _stack.clear(true /* clear_cache */);
+}
+
+void RemoveForwardedPointerClosure::do_object(oop obj) {
+ if (obj->is_forwarded()) {
+ obj->init_mark();
+ }
+}
+
+void PreservedMarksSet::init(uint num) {
+ assert(_stacks == NULL && _num == 0, "do not re-initialize");
+ assert(num > 0, "pre-condition");
+ if (_in_c_heap) {
+ _stacks = NEW_C_HEAP_ARRAY(Padded<PreservedMarks>, num, mtGC);
+ } else {
+ _stacks = NEW_RESOURCE_ARRAY(Padded<PreservedMarks>, num);
+ }
+ for (uint i = 0; i < num; i += 1) {
+ ::new (_stacks + i) PreservedMarks();
+ }
+ _num = num;
+
+ assert_empty();
+}
+
+void PreservedMarksSet::restore() {
+ for (uint i = 0; i < _num; i += 1) {
+ get(i)->restore();
+ }
+}
+
+void PreservedMarksSet::reclaim() {
+ assert_empty();
+
+ for (uint i = 0; i < _num; i += 1) {
+ _stacks[i].~Padded<PreservedMarks>();
+ }
+
+ if (_in_c_heap) {
+ FREE_C_HEAP_ARRAY(Padded<PreservedMarks>, _stacks);
+ } else {
+ // the array was resource-allocated, so nothing to do
+ }
+ _stacks = NULL;
+ _num = 0;
+}
+
+#ifndef PRODUCT
+void PreservedMarksSet::assert_empty() {
+ assert(_stacks != NULL && _num > 0, "should have been initialized");
+ for (uint i = 0; i < _num; i += 1) {
+ assert(get(i)->is_empty(), "stack should be empty");
+ }
+}
+#endif // ndef PRODUCT
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc/shared/preservedMarks.hpp Tue Feb 23 14:14:29 2016 +0100
@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
+#define SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/padded.hpp"
+#include "oops/oop.hpp"
+#include "utilities/stack.hpp"
+
+class OopAndMarkOop {
+private:
+ oop _o;
+ markOop _m;
+
+public:
+ OopAndMarkOop(oop obj, markOop m) : _o(obj), _m(m) { }
+
+ void set_mark() const {
+ _o->set_mark(_m);
+ }
+};
+typedef Stack<OopAndMarkOop, mtGC> OopAndMarkOopStack;
+
+class PreservedMarks VALUE_OBJ_CLASS_SPEC {
+private:
+ OopAndMarkOopStack _stack;
+
+ inline bool should_preserve_mark(oop obj, markOop m) const;
+ inline void push(oop obj, markOop m);
+
+public:
+ bool is_empty() const { return _stack.is_empty(); }
+ inline void push_if_necessary(oop obj, markOop m);
+ // Iterate over the stack, restore the preserved marks, then reclaim
+ // the memory taken up by stack chunks.
+ void restore();
+ ~PreservedMarks() { assert(is_empty(), "should have been cleared"); }
+};
+
+class RemoveForwardedPointerClosure: public ObjectClosure {
+public:
+ virtual void do_object(oop obj);
+};
+
+class PreservedMarksSet VALUE_OBJ_CLASS_SPEC {
+private:
+ // true -> _stacks will be allocated in the C heap
+ // false -> _stacks will be allocated in the resource arena
+ const bool _in_c_heap;
+
+ // Number of stacks we have allocated (typically, one stack per GC worker).
+ // This should be >= 1 if the stacks have been initialized,
+ // or == 0 if they have not.
+ uint _num;
+
+ // Stack array (typically, one stack per GC worker) of length _num.
+ // This should be != NULL if the stacks have been initialized,
+ // or == NULL if they have not.
+ Padded<PreservedMarks>* _stacks;
+
+public:
+ // Return the i'th stack.
+ PreservedMarks* get(uint i = 0) const {
+ assert(_num > 0 && _stacks != NULL, "stacks should have been initialized");
+ assert(i < _num, "pre-condition");
+ return (_stacks + i);
+ }
+
+ // Allocate stack array.
+ void init(uint num);
+ // Iterate over all stacks, restore all preserved marks, then
+ // reclaim the memory taken up by stack chunks.
+ void restore();
+ // Reclaim stack array.
+ void reclaim();
+
+ // Assert all the stacks are empty.
+ void assert_empty() PRODUCT_RETURN;
+
+ PreservedMarksSet(bool in_c_heap)
+ : _in_c_heap(in_c_heap), _num(0), _stacks(NULL) { }
+
+ ~PreservedMarksSet() {
+ assert(_stacks == NULL && _num == 0, "stacks should have been reclaimed");
+ }
+};
+
+#endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_HPP
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc/shared/preservedMarks.inline.hpp Tue Feb 23 14:14:29 2016 +0100
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "gc/shared/preservedMarks.hpp"
+#include "oops/markOop.inline.hpp"
+#include "utilities/stack.inline.hpp"
+
+#ifndef SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
+#define SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP
+
+inline bool PreservedMarks::should_preserve_mark(oop obj, markOop m) const {
+ return m->must_be_preserved_for_promotion_failure(obj);
+}
+
+inline void PreservedMarks::push(oop obj, markOop m) {
+ assert(should_preserve_mark(obj, m), "pre-condition");
+ OopAndMarkOop elem(obj, m);
+ _stack.push(elem);
+}
+
+inline void PreservedMarks::push_if_necessary(oop obj, markOop m) {
+ if (should_preserve_mark(obj, m)) {
+ push(obj, m);
+ }
+}
+
+#endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP