8031323: Optionally align objects copied to survivor spaces
Reviewed-by: brutisso, tschatzl
--- a/hotspot/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/g1/g1ParScanThreadState.cpp Fri Aug 01 15:40:12 2014 -0700
@@ -288,7 +288,12 @@
}
HeapWord* G1ParScanThreadState::allocate(GCAllocPurpose purpose, size_t word_sz) {
- HeapWord* obj = alloc_buffer(purpose)->allocate(word_sz);
+ HeapWord* obj = NULL;
+ if (purpose == GCAllocForSurvived) {
+ obj = alloc_buffer(GCAllocForSurvived)->allocate_aligned(word_sz, SurvivorAlignmentInBytes);
+ } else {
+ obj = alloc_buffer(GCAllocForTenured)->allocate(word_sz);
+ }
if (obj != NULL) {
return obj;
}
--- a/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.cpp Fri Aug 01 15:40:12 2014 -0700
@@ -28,12 +28,12 @@
#include "gc_implementation/parNew/parOopClosures.inline.hpp"
#include "gc_implementation/shared/adaptiveSizePolicy.hpp"
#include "gc_implementation/shared/ageTable.hpp"
-#include "gc_implementation/shared/parGCAllocBuffer.hpp"
+#include "gc_implementation/shared/copyFailedInfo.hpp"
#include "gc_implementation/shared/gcHeapSummary.hpp"
#include "gc_implementation/shared/gcTimer.hpp"
#include "gc_implementation/shared/gcTrace.hpp"
#include "gc_implementation/shared/gcTraceTime.hpp"
-#include "gc_implementation/shared/copyFailedInfo.hpp"
+#include "gc_implementation/shared/parGCAllocBuffer.inline.hpp"
#include "gc_implementation/shared/spaceDecorator.hpp"
#include "memory/defNewGeneration.inline.hpp"
#include "memory/genCollectedHeap.hpp"
@@ -252,7 +252,7 @@
plab->set_word_size(buf_size);
plab->set_buf(buf_space);
record_survivor_plab(buf_space, buf_size);
- obj = plab->allocate(word_sz);
+ obj = plab->allocate_aligned(word_sz, SurvivorAlignmentInBytes);
// Note that we cannot compare buf_size < word_sz below
// because of AlignmentReserve (see ParGCAllocBuffer::allocate()).
assert(obj != NULL || plab->words_remaining() < word_sz,
--- a/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/parNew/parNewGeneration.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -168,7 +168,7 @@
HeapWord* alloc_in_to_space_slow(size_t word_sz);
HeapWord* alloc_in_to_space(size_t word_sz) {
- HeapWord* obj = to_space_alloc_buffer()->allocate(word_sz);
+ HeapWord* obj = to_space_alloc_buffer()->allocate_aligned(word_sz, SurvivorAlignmentInBytes);
if (obj != NULL) return obj;
else return alloc_in_to_space_slow(word_sz);
}
--- a/hotspot/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -26,6 +26,7 @@
#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_HPP
#include "gc_implementation/parallelScavenge/objectStartArray.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
#include "memory/allocation.hpp"
//
@@ -94,23 +95,9 @@
PSYoungPromotionLAB() { }
// Not MT safe
- HeapWord* allocate(size_t size) {
- // Can't assert this, when young fills, we keep the LAB around, but flushed.
- // assert(_state != flushed, "Sanity");
- HeapWord* obj = top();
- HeapWord* new_top = obj + size;
- // The 'new_top>obj' check is needed to detect overflow of obj+size.
- if (new_top > obj && new_top <= end()) {
- set_top(new_top);
- assert(is_object_aligned((intptr_t)obj) && is_object_aligned((intptr_t)new_top),
- "checking alignment");
- return obj;
- }
+ inline HeapWord* allocate(size_t size);
- return NULL;
- }
-
- debug_only(virtual bool lab_is_valid(MemRegion lab));
+ debug_only(virtual bool lab_is_valid(MemRegion lab);)
};
class PSOldPromotionLAB : public PSPromotionLAB {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP
+#define SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP
+
+#include "gc_implementation/parallelScavenge/psPromotionLAB.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
+
+HeapWord* PSYoungPromotionLAB::allocate(size_t size) {
+ // Can't assert this, when young fills, we keep the LAB around, but flushed.
+ // assert(_state != flushed, "Sanity");
+ HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end(), SurvivorAlignmentInBytes);
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ HeapWord* new_top = obj + size;
+ // The 'new_top>obj' check is needed to detect overflow of obj+size.
+ if (new_top > obj && new_top <= end()) {
+ set_top(new_top);
+ assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_object_aligned((intptr_t)new_top),
+ "checking alignment");
+ return obj;
+ } else {
+ set_top(obj);
+ return NULL;
+ }
+}
+
+#endif // SHARE_VM_GC_IMPLEMENTATION_PARALLELSCAVENGE_PSPROMOTIONLAB_INLINE_HPP
--- a/hotspot/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/parallelScavenge/psPromotionManager.inline.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -27,6 +27,7 @@
#include "gc_implementation/parallelScavenge/psOldGen.hpp"
#include "gc_implementation/parallelScavenge/psPromotionManager.hpp"
+#include "gc_implementation/parallelScavenge/psPromotionLAB.inline.hpp"
#include "gc_implementation/parallelScavenge/psScavenge.hpp"
#include "oops/oop.psgc.inline.hpp"
--- a/hotspot/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_implementation/shared/parGCAllocBuffer.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -24,7 +24,7 @@
#ifndef SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP
#define SHARE_VM_GC_IMPLEMENTATION_PARNEW_PARGCALLOCBUFFER_HPP
-
+#include "gc_interface/collectedHeap.hpp"
#include "memory/allocation.hpp"
#include "memory/blockOffsetTable.hpp"
#include "memory/threadLocalAllocBuffer.hpp"
@@ -84,6 +84,9 @@
}
}
+ // Allocate the object aligned to "alignment_in_bytes".
+ HeapWord* allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes);
+
// Undo the last allocation in the buffer, which is required to be of the
// "obj" of the given "word_sz".
void undo_allocation(HeapWord* obj, size_t word_sz) {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc_implementation/shared/parGCAllocBuffer.inline.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP
+#define SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP
+
+#include "gc_implementation/shared/parGCAllocBuffer.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
+
+HeapWord* ParGCAllocBuffer::allocate_aligned(size_t word_sz, unsigned short alignment_in_bytes) {
+
+ HeapWord* res = CollectedHeap::align_allocation_or_fail(_top, _end, alignment_in_bytes);
+ if (res == NULL) {
+ return NULL;
+ }
+
+ // Set _top so that allocate(), which expects _top to be correctly set,
+ // can be used below.
+ _top = res;
+ return allocate(word_sz);
+}
+
+#endif // SHARE_VM_GC_IMPLEMENTATION_SHARED_PARGCALLOCBUFFER_INLINE_HPP
--- a/hotspot/src/share/vm/gc_interface/collectedHeap.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_interface/collectedHeap.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -351,6 +351,12 @@
fill_with_object(start, pointer_delta(end, start), zap);
}
+ // Return the address "addr" aligned by "alignment_in_bytes" if such
+ // an address is below "end". Return NULL otherwise.
+ inline static HeapWord* align_allocation_or_fail(HeapWord* addr,
+ HeapWord* end,
+ unsigned short alignment_in_bytes);
+
// Some heaps may offer a contiguous region for shared non-blocking
// allocation, via inlined code (by exporting the address of the top and
// end fields defining the extent of the contiguous allocation region.)
--- a/hotspot/src/share/vm/gc_interface/collectedHeap.inline.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/gc_interface/collectedHeap.inline.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -241,6 +241,44 @@
oop_iterate(&no_header_cl);
}
+
+inline HeapWord* CollectedHeap::align_allocation_or_fail(HeapWord* addr,
+ HeapWord* end,
+ unsigned short alignment_in_bytes) {
+ if (alignment_in_bytes <= ObjectAlignmentInBytes) {
+ return addr;
+ }
+
+ assert(is_ptr_aligned(addr, HeapWordSize),
+ err_msg("Address " PTR_FORMAT " is not properly aligned.", p2i(addr)));
+ assert(is_size_aligned(alignment_in_bytes, HeapWordSize),
+ err_msg("Alignment size %u is incorrect.", alignment_in_bytes));
+
+ HeapWord* new_addr = (HeapWord*) align_pointer_up(addr, alignment_in_bytes);
+ size_t padding = pointer_delta(new_addr, addr);
+
+ if (padding == 0) {
+ return addr;
+ }
+
+ if (padding < CollectedHeap::min_fill_size()) {
+ padding += alignment_in_bytes / HeapWordSize;
+ assert(padding >= CollectedHeap::min_fill_size(),
+ err_msg("alignment_in_bytes %u is expect to be larger "
+ "than the minimum object size", alignment_in_bytes));
+ new_addr = addr + padding;
+ }
+
+ assert(new_addr > addr, err_msg("Unexpected arithmetic overflow "
+ PTR_FORMAT " not greater than " PTR_FORMAT, p2i(new_addr), p2i(addr)));
+ if(new_addr < end) {
+ CollectedHeap::fill_with_object(addr, padding);
+ return new_addr;
+ } else {
+ return NULL;
+ }
+}
+
#ifndef PRODUCT
inline bool
--- a/hotspot/src/share/vm/memory/defNewGeneration.cpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/memory/defNewGeneration.cpp Fri Aug 01 15:40:12 2014 -0700
@@ -790,7 +790,7 @@
// Try allocating obj in to-space (unless too old)
if (old->age() < tenuring_threshold()) {
- obj = (oop) to()->allocate(s);
+ obj = (oop) to()->allocate_aligned(s);
}
// Otherwise try allocating obj tenured
--- a/hotspot/src/share/vm/memory/space.cpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/memory/space.cpp Fri Aug 01 15:40:12 2014 -0700
@@ -28,6 +28,7 @@
#include "gc_implementation/shared/liveRange.hpp"
#include "gc_implementation/shared/markSweep.hpp"
#include "gc_implementation/shared/spaceDecorator.hpp"
+#include "gc_interface/collectedHeap.inline.hpp"
#include "memory/blockOffsetTable.inline.hpp"
#include "memory/defNewGeneration.hpp"
#include "memory/genCollectedHeap.hpp"
@@ -720,6 +721,27 @@
} while (true);
}
+HeapWord* ContiguousSpace::allocate_aligned(size_t size) {
+ assert(Heap_lock->owned_by_self() || (SafepointSynchronize::is_at_safepoint() && Thread::current()->is_VM_thread()), "not locked");
+ HeapWord* end_value = end();
+
+ HeapWord* obj = CollectedHeap::align_allocation_or_fail(top(), end_value, SurvivorAlignmentInBytes);
+ if (obj == NULL) {
+ return NULL;
+ }
+
+ if (pointer_delta(end_value, obj) >= size) {
+ HeapWord* new_top = obj + size;
+ set_top(new_top);
+ assert(is_ptr_aligned(obj, SurvivorAlignmentInBytes) && is_aligned(new_top),
+ "checking alignment");
+ return obj;
+ } else {
+ set_top(obj);
+ return NULL;
+ }
+}
+
// Requires locking.
HeapWord* ContiguousSpace::allocate(size_t size) {
return allocate_impl(size, end());
--- a/hotspot/src/share/vm/memory/space.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/memory/space.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -526,6 +526,7 @@
// Allocation (return NULL if full)
virtual HeapWord* allocate(size_t word_size);
virtual HeapWord* par_allocate(size_t word_size);
+ HeapWord* allocate_aligned(size_t word_size);
// Iteration
void oop_iterate(ExtendedOopClosure* cl);
--- a/hotspot/src/share/vm/oops/oop.pcgc.inline.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/oops/oop.pcgc.inline.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -55,8 +55,6 @@
klass()->oop_follow_contents(cm, this);
}
-// Used by parallel old GC.
-
inline oop oopDesc::forward_to_atomic(oop p) {
assert(ParNewGeneration::is_legal_forward_ptr(p),
"illegal forwarding pointer value.");
--- a/hotspot/src/share/vm/runtime/arguments.cpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/runtime/arguments.cpp Fri Aug 01 15:40:12 2014 -0700
@@ -1431,6 +1431,22 @@
(int)ObjectAlignmentInBytes, os::vm_page_size());
return false;
}
+ if(SurvivorAlignmentInBytes == 0) {
+ SurvivorAlignmentInBytes = ObjectAlignmentInBytes;
+ } else {
+ if (!is_power_of_2(SurvivorAlignmentInBytes)) {
+ jio_fprintf(defaultStream::error_stream(),
+ "error: SurvivorAlignmentInBytes=%d must be power of 2\n",
+ (int)SurvivorAlignmentInBytes);
+ return false;
+ }
+ if (SurvivorAlignmentInBytes < ObjectAlignmentInBytes) {
+ jio_fprintf(defaultStream::error_stream(),
+ "error: SurvivorAlignmentInBytes=%d must be greater than ObjectAlignmentInBytes=%d \n",
+ (int)SurvivorAlignmentInBytes, (int)ObjectAlignmentInBytes);
+ return false;
+ }
+ }
return true;
}
--- a/hotspot/src/share/vm/runtime/globals.hpp Thu Jul 31 09:23:24 2014 +0200
+++ b/hotspot/src/share/vm/runtime/globals.hpp Fri Aug 01 15:40:12 2014 -0700
@@ -3871,6 +3871,9 @@
product(bool, PrintGCCause, true, \
"Include GC cause in GC logging") \
\
+ experimental(intx, SurvivorAlignmentInBytes, 0, \
+ "Default survivor space alignment in bytes") \
+ \
product(bool , AllowNonVirtualCalls, false, \
"Obey the ACC_SUPER flag and allow invokenonvirtual calls") \
\