8227226: ZGC: Segmented array clearing
Reviewed-by: eosterlund
Contributed-by: stefan.karlsson@oracle.com, erik.osterlund@oracle.com, per.liden@oracle.com, sci@amazon.com
--- a/src/hotspot/share/gc/shared/memAllocator.cpp Thu Aug 22 14:09:36 2019 -0700
+++ b/src/hotspot/share/gc/shared/memAllocator.cpp Fri Aug 23 08:48:18 2019 +0200
@@ -226,10 +226,10 @@
size_t size_in_bytes = _allocator._word_size * HeapWordSize;
if (_allocated_outside_tlab) {
- AllocTracer::send_allocation_outside_tlab(_allocator._klass, mem, size_in_bytes, _thread);
+ AllocTracer::send_allocation_outside_tlab(obj()->klass(), mem, size_in_bytes, _thread);
} else if (_allocated_tlab_size != 0) {
// TLAB was refilled
- AllocTracer::send_allocation_in_new_tlab(_allocator._klass, mem, _allocated_tlab_size * HeapWordSize,
+ AllocTracer::send_allocation_in_new_tlab(obj()->klass(), mem, _allocated_tlab_size * HeapWordSize,
size_in_bytes, _thread);
}
}
@@ -237,7 +237,7 @@
void MemAllocator::Allocation::notify_allocation_dtrace_sampler() {
if (DTraceAllocProbes) {
// support for Dtrace object alloc event (no-op most of the time)
- Klass* klass = _allocator._klass;
+ Klass* klass = obj()->klass();
size_t word_size = _allocator._word_size;
if (klass != NULL && klass->name() != NULL) {
SharedRuntime::dtrace_object_alloc(obj(), (int)word_size);
--- a/src/hotspot/share/gc/shared/memAllocator.hpp Thu Aug 22 14:09:36 2019 -0700
+++ b/src/hotspot/share/gc/shared/memAllocator.hpp Fri Aug 23 08:48:18 2019 +0200
@@ -59,7 +59,7 @@
// This finish constructing an oop by installing the mark word and the Klass* pointer
// last. At the point when the Klass pointer is initialized, this is a constructed object
// that must be parseable as an oop by concurrent collectors.
- oop finish(HeapWord* mem) const;
+ virtual oop finish(HeapWord* mem) const;
// Raw memory allocation. This may or may not use TLAB allocations to satisfy the
// allocation. A GC implementation may override this function to satisfy the allocation
--- a/src/hotspot/share/gc/z/zCollectedHeap.cpp Thu Aug 22 14:09:36 2019 -0700
+++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp Fri Aug 23 08:48:18 2019 +0200
@@ -28,6 +28,7 @@
#include "gc/z/zGlobals.hpp"
#include "gc/z/zHeap.inline.hpp"
#include "gc/z/zNMethod.hpp"
+#include "gc/z/zObjArrayAllocator.hpp"
#include "gc/z/zServiceability.hpp"
#include "gc/z/zStat.hpp"
#include "gc/z/zUtils.inline.hpp"
@@ -127,6 +128,15 @@
return (HeapWord*)addr;
}
+oop ZCollectedHeap::array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS) {
+ if (!do_zero) {
+ return CollectedHeap::array_allocate(klass, size, length, false /* do_zero */, THREAD);
+ }
+
+ ZObjArrayAllocator allocator(klass, size, length, THREAD);
+ return allocator.allocate();
+}
+
HeapWord* ZCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) {
const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(size));
return (HeapWord*)_heap.alloc_object(size_in_bytes);
--- a/src/hotspot/share/gc/z/zCollectedHeap.hpp Thu Aug 22 14:09:36 2019 -0700
+++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp Fri Aug 23 08:48:18 2019 +0200
@@ -75,6 +75,7 @@
virtual uint32_t hash_oop(oop obj) const;
+ virtual oop array_allocate(Klass* klass, int size, int length, bool do_zero, TRAPS);
virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded);
virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
size_t size,
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp Fri Aug 23 08:48:18 2019 +0200
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zObjArrayAllocator.hpp"
+#include "gc/z/zUtils.inline.hpp"
+#include "memory/universe.hpp"
+#include "oops/arrayKlass.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/handles.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// To avoid delaying safepoints, clearing of arrays is split up in segments
+// with safepoint polling inbetween. However, we can't have a not-yet-cleared
+// array of oops on the heap when we safepoint since the GC will then stumble
+// across uninitialized oops. To avoid this we let an array of oops be an
+// array of a primitive type of the same size until the clearing has completed.
+// A max segment size of 64K was chosen because benchmarking suggests that is
+// offers a good trade-off between allocation time and time-to-safepoint.
+
+static Klass* substitute_object_array_klass(Klass* klass) {
+ if (!klass->is_objArray_klass()) {
+ return klass;
+ }
+
+ Klass* const substitute_klass = Universe::longArrayKlassObj();
+ const BasicType type = ArrayKlass::cast(klass)->element_type();
+ const BasicType substitute_type = ArrayKlass::cast(substitute_klass)->element_type();
+ assert(type2aelembytes(type) == type2aelembytes(substitute_type), "Element size mismatch");
+ return substitute_klass;
+}
+
+ZObjArrayAllocator::ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread) :
+ ObjArrayAllocator(substitute_object_array_klass(klass), word_size, length, false /* do_zero */, thread),
+ _final_klass(klass) {}
+
+oop ZObjArrayAllocator::finish(HeapWord* mem) const {
+ HandleMark hm;
+ Handle array(_thread, ObjArrayAllocator::finish(mem));
+
+ const size_t segment_max = ZUtils::bytes_to_words(64 * K);
+ const size_t skip = arrayOopDesc::header_size(ArrayKlass::cast(_klass)->element_type());
+ size_t remaining = _word_size - skip;
+
+ while (remaining > 0) {
+ // Clear segment
+ const size_t segment = MIN2(remaining, segment_max);
+ Copy::zero_to_words((HeapWord*)array() + (_word_size - remaining), segment);
+ remaining -= segment;
+
+ if (remaining > 0) {
+ // Safepoint
+ ThreadBlockInVM tbivm((JavaThread*)_thread);
+ }
+ }
+
+ if (_klass != _final_klass) {
+ // Set final klass
+ oopDesc::release_set_klass((HeapWord*)array(), _final_klass);
+ }
+
+ return array();
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zObjArrayAllocator.hpp Fri Aug 23 08:48:18 2019 +0200
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP
+#define SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP
+
+#include "gc/shared/memAllocator.hpp"
+
+class ZObjArrayAllocator : public ObjArrayAllocator {
+private:
+ Klass* const _final_klass;
+
+public:
+ ZObjArrayAllocator(Klass* klass, size_t word_size, int length, Thread* thread);
+
+ virtual oop finish(HeapWord* mem) const;
+};
+
+#endif // SHARE_GC_Z_ZOBJARRAYALLOCATOR_HPP