6687581: Make CMS work with compressed oops
Summary: Make FreeChunk read markword instead of LSB in _klass pointer to indicate that it's a FreeChunk for compressed oops.
Reviewed-by: ysr, jmasa
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/memory/CompactibleFreeListSpace.java Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/memory/CompactibleFreeListSpace.java Mon Jun 09 11:51:19 2008 -0400
@@ -121,7 +121,7 @@
cur = cur.addOffsetTo(adjustObjectSizeInBytes(size));
}
- if (FreeChunk.secondWordIndicatesFreeChunk(dbg.getAddressValue(klassOop))) {
+ if (FreeChunk.indicatesFreeChunk(cur)) {
if (! cur.equals(regionStart)) {
res.add(new MemRegion(regionStart, cur));
}
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/memory/FreeChunk.java Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/memory/FreeChunk.java Mon Jun 09 11:51:19 2008 -0400
@@ -28,6 +28,7 @@
import sun.jvm.hotspot.debugger.*;
import sun.jvm.hotspot.types.*;
import sun.jvm.hotspot.runtime.*;
+import sun.jvm.hotspot.oops.*;
public class FreeChunk extends VMObject {
static {
@@ -42,13 +43,13 @@
Type type = db.lookupType("FreeChunk");
nextField = type.getAddressField("_next");
prevField = type.getAddressField("_prev");
- sizeField = type.getCIntegerField("_size");
+ sizeField = type.getAddressField("_size");
}
// Fields
private static AddressField nextField;
private static AddressField prevField;
- private static CIntegerField sizeField;
+ private static AddressField sizeField;
// Accessors
public FreeChunk next() {
@@ -61,20 +62,34 @@
}
public long size() {
- return sizeField.getValue(addr);
+ if (VM.getVM().isCompressedOopsEnabled()) {
+ Mark mark = new Mark(sizeField.getValue(addr));
+ return mark.getSize();
+ } else {
+ Address size = sizeField.getValue(addr);
+ Debugger dbg = VM.getVM().getDebugger();
+ return dbg.getAddressValue(size);
+ }
}
public FreeChunk(Address addr) {
super(addr);
}
- public static boolean secondWordIndicatesFreeChunk(long word) {
- return (word & 0x1L) == 0x1L;
+ public static boolean indicatesFreeChunk(Address cur) {
+ FreeChunk f = new FreeChunk(cur);
+ return f.isFree();
}
public boolean isFree() {
- Debugger dbg = VM.getVM().getDebugger();
- Address prev = prevField.getValue(addr);
- return secondWordIndicatesFreeChunk(dbg.getAddressValue(prev));
+ if (VM.getVM().isCompressedOopsEnabled()) {
+ Mark mark = new Mark(sizeField.getValue(addr));
+ return mark.isCmsFreeChunk();
+ } else {
+ Address prev = prevField.getValue(addr);
+ Debugger dbg = VM.getVM().getDebugger();
+ long word = dbg.getAddressValue(prev);
+ return (word & 0x1L) == 0x1L;
+ }
}
}
--- a/hotspot/agent/src/share/classes/sun/jvm/hotspot/oops/Mark.java Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/agent/src/share/classes/sun/jvm/hotspot/oops/Mark.java Mon Jun 09 11:51:19 2008 -0400
@@ -79,6 +79,11 @@
noHashInPlace = db.lookupLongConstant("markOopDesc::no_hash_in_place").longValue();
noLockInPlace = db.lookupLongConstant("markOopDesc::no_lock_in_place").longValue();
maxAge = db.lookupLongConstant("markOopDesc::max_age").longValue();
+
+ /* Constants in markOop used by CMS. */
+ cmsShift = db.lookupLongConstant("markOopDesc::cms_shift").longValue();
+ cmsMask = db.lookupLongConstant("markOopDesc::cms_mask").longValue();
+ sizeShift = db.lookupLongConstant("markOopDesc::size_shift").longValue();
}
// Field accessors
@@ -120,6 +125,11 @@
private static long maxAge;
+ /* Constants in markOop used by CMS. */
+ private static long cmsShift;
+ private static long cmsMask;
+ private static long sizeShift;
+
public Mark(Address addr) {
super(addr);
}
@@ -290,4 +300,11 @@
//
// // Recover address of oop from encoded form used in mark
// inline void* decode_pointer() { return clear_lock_bits(); }
+
+ // Copy markOop methods for CMS here.
+ public boolean isCmsFreeChunk() {
+ return isUnlocked() &&
+ (Bits.maskBitsLong(value() >> cmsShift, cmsMask) & 0x1L) == 0x1L;
+ }
+ public long getSize() { return (long)(value() >> sizeShift); }
}
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Mon Jun 09 11:51:19 2008 -0400
@@ -805,28 +805,30 @@
// This must be volatile, or else there is a danger that the compiler
// will compile the code below into a sometimes-infinite loop, by keeping
// the value read the first time in a register.
- oop o = (oop)p;
- volatile oop* second_word_addr = o->klass_addr();
while (true) {
- klassOop k = (klassOop)(*second_word_addr);
// We must do this until we get a consistent view of the object.
- if (FreeChunk::secondWordIndicatesFreeChunk((intptr_t)k)) {
- FreeChunk* fc = (FreeChunk*)p;
- volatile size_t* sz_addr = (volatile size_t*)(fc->size_addr());
- size_t res = (*sz_addr);
- klassOop k2 = (klassOop)(*second_word_addr); // Read to confirm.
- if (k == k2) {
+ if (FreeChunk::indicatesFreeChunk(p)) {
+ volatile FreeChunk* fc = (volatile FreeChunk*)p;
+ size_t res = fc->size();
+ // If the object is still a free chunk, return the size, else it
+ // has been allocated so try again.
+ if (FreeChunk::indicatesFreeChunk(p)) {
assert(res != 0, "Block size should not be 0");
return res;
}
- } else if (k != NULL) {
- assert(k->is_oop(true /* ignore mark word */), "Should really be klass oop.");
- assert(o->is_parsable(), "Should be parsable");
- assert(o->is_oop(true /* ignore mark word */), "Should be an oop.");
- size_t res = o->size_given_klass(k->klass_part());
- res = adjustObjectSize(res);
- assert(res != 0, "Block size should not be 0");
- return res;
+ } else {
+ // must read from what 'p' points to in each loop.
+ klassOop k = ((volatile oopDesc*)p)->klass_or_null();
+ if (k != NULL) {
+ assert(k->is_oop(true /* ignore mark word */), "Should really be klass oop.");
+ oop o = (oop)p;
+ assert(o->is_parsable(), "Should be parsable");
+ assert(o->is_oop(true /* ignore mark word */), "Should be an oop.");
+ size_t res = o->size_given_klass(k->klass_part());
+ res = adjustObjectSize(res);
+ assert(res != 0, "Block size should not be 0");
+ return res;
+ }
}
}
}
@@ -845,31 +847,31 @@
// This must be volatile, or else there is a danger that the compiler
// will compile the code below into a sometimes-infinite loop, by keeping
// the value read the first time in a register.
- oop o = (oop)p;
- volatile oop* second_word_addr = o->klass_addr();
DEBUG_ONLY(uint loops = 0;)
while (true) {
- klassOop k = (klassOop)(*second_word_addr);
// We must do this until we get a consistent view of the object.
- if (FreeChunk::secondWordIndicatesFreeChunk((intptr_t)k)) {
- FreeChunk* fc = (FreeChunk*)p;
- volatile size_t* sz_addr = (volatile size_t*)(fc->size_addr());
- size_t res = (*sz_addr);
- klassOop k2 = (klassOop)(*second_word_addr); // Read to confirm.
- if (k == k2) {
+ if (FreeChunk::indicatesFreeChunk(p)) {
+ volatile FreeChunk* fc = (volatile FreeChunk*)p;
+ size_t res = fc->size();
+ if (FreeChunk::indicatesFreeChunk(p)) {
assert(res != 0, "Block size should not be 0");
assert(loops == 0, "Should be 0");
return res;
}
- } else if (k != NULL && o->is_parsable()) {
- assert(k->is_oop(), "Should really be klass oop.");
- assert(o->is_oop(), "Should be an oop");
- size_t res = o->size_given_klass(k->klass_part());
- res = adjustObjectSize(res);
- assert(res != 0, "Block size should not be 0");
- return res;
} else {
- return c->block_size_if_printezis_bits(p);
+ // must read from what 'p' points to in each loop.
+ klassOop k = ((volatile oopDesc*)p)->klass_or_null();
+ if (k != NULL && ((oopDesc*)p)->is_parsable()) {
+ assert(k->is_oop(), "Should really be klass oop.");
+ oop o = (oop)p;
+ assert(o->is_oop(), "Should be an oop");
+ size_t res = o->size_given_klass(k->klass_part());
+ res = adjustObjectSize(res);
+ assert(res != 0, "Block size should not be 0");
+ return res;
+ } else {
+ return c->block_size_if_printezis_bits(p);
+ }
}
assert(loops == 0, "Can loop at most once");
DEBUG_ONLY(loops++;)
@@ -907,9 +909,8 @@
// and those objects (if garbage) may have been modified to hold
// live range information.
// assert(ParallelGCThreads > 0 || _bt.block_start(p) == p, "Should be a block boundary");
- klassOop k = oop(p)->klass();
- intptr_t ki = (intptr_t)k;
- if (FreeChunk::secondWordIndicatesFreeChunk(ki)) return false;
+ if (FreeChunk::indicatesFreeChunk(p)) return false;
+ klassOop k = oop(p)->klass_or_null();
if (k != NULL) {
// Ignore mark word because it may have been used to
// chain together promoted objects (the last one
@@ -1027,7 +1028,7 @@
FreeChunk* fc = (FreeChunk*)res;
fc->markNotFree();
assert(!fc->isFree(), "shouldn't be marked free");
- assert(oop(fc)->klass() == NULL, "should look uninitialized");
+ assert(oop(fc)->klass_or_null() == NULL, "should look uninitialized");
// Verify that the block offset table shows this to
// be a single block, but not one which is unallocated.
_bt.verify_single_block(res, size);
@@ -2593,7 +2594,7 @@
}
res->markNotFree();
assert(!res->isFree(), "shouldn't be marked free");
- assert(oop(res)->klass() == NULL, "should look uninitialized");
+ assert(oop(res)->klass_or_null() == NULL, "should look uninitialized");
// mangle a just allocated object with a distinct pattern.
debug_only(res->mangleAllocated(word_sz));
return (HeapWord*)res;
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Mon Jun 09 11:51:19 2008 -0400
@@ -190,7 +190,8 @@
// depends on this property.
debug_only(
FreeChunk* junk = NULL;
- assert(junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
+ assert(UseCompressedOops ||
+ junk->prev_addr() == (void*)(oop(junk)->klass_addr()),
"Offset of FreeChunk::_prev within FreeChunk must match"
" that of OopDesc::_klass within OopDesc");
)
@@ -1039,7 +1040,7 @@
// mark end of object
}
// check that oop looks uninitialized
- assert(oop(start)->klass() == NULL, "_klass should be NULL");
+ assert(oop(start)->klass_or_null() == NULL, "_klass should be NULL");
}
void CMSCollector::promoted(bool par, HeapWord* start,
@@ -1309,17 +1310,25 @@
}
}
oop obj = oop(obj_ptr);
- assert(obj->klass() == NULL, "Object should be uninitialized here.");
+ assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
// Otherwise, copy the object. Here we must be careful to insert the
// klass pointer last, since this marks the block as an allocated object.
+ // Except with compressed oops it's the mark word.
HeapWord* old_ptr = (HeapWord*)old;
if (word_sz > (size_t)oopDesc::header_size()) {
Copy::aligned_disjoint_words(old_ptr + oopDesc::header_size(),
obj_ptr + oopDesc::header_size(),
word_sz - oopDesc::header_size());
}
+
+ if (UseCompressedOops) {
+ // Copy gap missed by (aligned) header size calculation above
+ obj->set_klass_gap(old->klass_gap());
+ }
+
// Restore the mark word copied above.
obj->set_mark(m);
+
// Now we can track the promoted object, if necessary. We take care
// To delay the transition from uninitialized to full object
// (i.e., insertion of klass pointer) until after, so that it
@@ -1327,7 +1336,8 @@
if (promoInfo->tracking()) {
promoInfo->track((PromotedObject*)obj, old->klass());
}
- // Finally, install the klass pointer.
+
+ // Finally, install the klass pointer (this should be volatile).
obj->set_klass(old->klass());
assert(old->is_oop(), "Will dereference klass ptr below");
@@ -6165,7 +6175,7 @@
HeapWord* CMSCollector::next_card_start_after_block(HeapWord* addr) const {
size_t sz = 0;
oop p = (oop)addr;
- if (p->klass() != NULL && p->is_parsable()) {
+ if (p->klass_or_null() != NULL && p->is_parsable()) {
sz = CompactibleFreeListSpace::adjustObjectSize(p->size());
} else {
sz = block_size_using_printezis_bits(addr);
@@ -6602,7 +6612,7 @@
}
if (_bitMap->isMarked(addr)) {
// it's marked; is it potentially uninitialized?
- if (p->klass() != NULL) {
+ if (p->klass_or_null() != NULL) {
if (CMSPermGenPrecleaningEnabled && !p->is_parsable()) {
// Signal precleaning to redirty the card since
// the klass pointer is already installed.
@@ -6615,11 +6625,8 @@
if (p->is_objArray()) {
// objArrays are precisely marked; restrict scanning
// to dirty cards only.
- size = p->oop_iterate(_scanningClosure, mr);
- assert(size == CompactibleFreeListSpace::adjustObjectSize(size),
- "adjustObjectSize should be the identity for array sizes, "
- "which are necessarily larger than minimum object size of "
- "two heap words");
+ size = CompactibleFreeListSpace::adjustObjectSize(
+ p->oop_iterate(_scanningClosure, mr));
} else {
// A non-array may have been imprecisely marked; we need
// to scan object in its entirety.
@@ -6653,7 +6660,7 @@
}
} else {
// Either a not yet marked object or an uninitialized object
- if (p->klass() == NULL || !p->is_parsable()) {
+ if (p->klass_or_null() == NULL || !p->is_parsable()) {
// An uninitialized object, skip to the next card, since
// we may not be able to read its P-bits yet.
assert(size == 0, "Initial value");
@@ -6710,7 +6717,7 @@
HeapWord* addr = (HeapWord*)p;
DEBUG_ONLY(_collector->verify_work_stacks_empty();)
assert(!_span.contains(addr), "we are scanning the survivor spaces");
- assert(p->klass() != NULL, "object should be initializd");
+ assert(p->klass_or_null() != NULL, "object should be initializd");
assert(p->is_parsable(), "must be parsable.");
// an initialized object; ignore mark word in verification below
// since we are running concurrent with mutators
@@ -6868,7 +6875,7 @@
assert(_skipBits == 0, "tautology");
_skipBits = 2; // skip next two marked bits ("Printezis-marks")
oop p = oop(addr);
- if (p->klass() == NULL || !p->is_parsable()) {
+ if (p->klass_or_null() == NULL || !p->is_parsable()) {
DEBUG_ONLY(if (!_verifying) {)
// We re-dirty the cards on which this object lies and increase
// the _threshold so that we'll come back to scan this object
@@ -6890,7 +6897,7 @@
if (_threshold < end_card_addr) {
_threshold = end_card_addr;
}
- if (p->klass() != NULL) {
+ if (p->klass_or_null() != NULL) {
// Redirty the range of cards...
_mut->mark_range(redirty_range);
} // ...else the setting of klass will dirty the card anyway.
@@ -7048,7 +7055,7 @@
assert(_skip_bits == 0, "tautology");
_skip_bits = 2; // skip next two marked bits ("Printezis-marks")
oop p = oop(addr);
- if (p->klass() == NULL || !p->is_parsable()) {
+ if (p->klass_or_null() == NULL || !p->is_parsable()) {
// in the case of Clean-on-Enter optimization, redirty card
// and avoid clearing card by increasing the threshold.
return;
@@ -8023,7 +8030,7 @@
"alignment problem");
#ifdef DEBUG
- if (oop(addr)->klass() != NULL &&
+ if (oop(addr)->klass_or_null() != NULL &&
( !_collector->should_unload_classes()
|| oop(addr)->is_parsable())) {
// Ignore mark word because we are running concurrent with mutators
@@ -8036,7 +8043,7 @@
} else {
// This should be an initialized object that's alive.
- assert(oop(addr)->klass() != NULL &&
+ assert(oop(addr)->klass_or_null() != NULL &&
(!_collector->should_unload_classes()
|| oop(addr)->is_parsable()),
"Should be an initialized object");
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeBlockDictionary.hpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeBlockDictionary.hpp Mon Jun 09 11:51:19 2008 -0400
@@ -22,88 +22,6 @@
*
*/
-//
-// Free block maintenance for Concurrent Mark Sweep Generation
-//
-// The main data structure for free blocks are
-// . an indexed array of small free blocks, and
-// . a dictionary of large free blocks
-//
-
-// No virtuals in FreeChunk (don't want any vtables).
-
-// A FreeChunk is merely a chunk that can be in a doubly linked list
-// and has a size field. NOTE: FreeChunks are distinguished from allocated
-// objects in two ways (by the sweeper). The second word (prev) has the
-// LSB set to indicate a free chunk; allocated objects' klass() pointers
-// don't have their LSB set. The corresponding bit in the CMSBitMap is
-// set when the chunk is allocated. There are also blocks that "look free"
-// but are not part of the free list and should not be coalesced into larger
-// free blocks. These free blocks have their two LSB's set.
-
-class FreeChunk VALUE_OBJ_CLASS_SPEC {
- friend class VMStructs;
- FreeChunk* _next;
- FreeChunk* _prev;
- size_t _size;
-
- public:
- NOT_PRODUCT(static const size_t header_size();)
- // Returns "true" if the "wrd", which is required to be the second word
- // of a block, indicates that the block represents a free chunk.
- static bool secondWordIndicatesFreeChunk(intptr_t wrd) {
- return (wrd & 0x1) == 0x1;
- }
- bool isFree() const {
- return secondWordIndicatesFreeChunk((intptr_t)_prev);
- }
- bool cantCoalesce() const { return (((intptr_t)_prev) & 0x3) == 0x3; }
- FreeChunk* next() const { return _next; }
- FreeChunk* prev() const { return (FreeChunk*)(((intptr_t)_prev) & ~(0x3)); }
- debug_only(void* prev_addr() const { return (void*)&_prev; })
-
- void linkAfter(FreeChunk* ptr) {
- linkNext(ptr);
- if (ptr != NULL) ptr->linkPrev(this);
- }
- void linkAfterNonNull(FreeChunk* ptr) {
- assert(ptr != NULL, "precondition violation");
- linkNext(ptr);
- ptr->linkPrev(this);
- }
- void linkNext(FreeChunk* ptr) { _next = ptr; }
- void linkPrev(FreeChunk* ptr) { _prev = (FreeChunk*)((intptr_t)ptr | 0x1); }
- void clearPrev() { _prev = NULL; }
- void clearNext() { _next = NULL; }
- void dontCoalesce() {
- // the block should be free
- assert(isFree(), "Should look like a free block");
- _prev = (FreeChunk*)(((intptr_t)_prev) | 0x2);
- }
- void markFree() { _prev = (FreeChunk*)((intptr_t)_prev | 0x1); }
- void markNotFree() { _prev = NULL; }
-
- size_t size() const { return _size; }
- void setSize(size_t size) { _size = size; }
-
- // For volatile reads:
- size_t* size_addr() { return &_size; }
-
- // Return the address past the end of this chunk
- HeapWord* end() const { return ((HeapWord*) this) + _size; }
-
- // debugging
- void verify() const PRODUCT_RETURN;
- void verifyList() const PRODUCT_RETURN;
- void mangleAllocated(size_t size) PRODUCT_RETURN;
- void mangleFreed(size_t size) PRODUCT_RETURN;
-};
-
-// Alignment helpers etc.
-#define numQuanta(x,y) ((x+y-1)/y)
-enum AlignmentConstants {
- MinChunkSize = numQuanta(sizeof(FreeChunk), MinObjAlignmentInBytes) * MinObjAlignment
-};
// A FreeBlockDictionary is an abstract superclass that will allow
// a number of alternative implementations in the future.
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.cpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.cpp Mon Jun 09 11:51:19 2008 -0400
@@ -47,15 +47,15 @@
Copy::fill_to_words(addr + hdr, size - hdr, baadbabeHeapWord);
}
-void FreeChunk::mangleFreed(size_t size) {
+void FreeChunk::mangleFreed(size_t sz) {
assert(baadbabeHeapWord != deadbeefHeapWord, "Need distinct patterns");
// mangle all but the header of a just-freed block of storage
// just prior to passing it to the storage dictionary
- assert(size >= MinChunkSize, "smallest size of object");
- assert(size == _size, "just checking");
+ assert(sz >= MinChunkSize, "smallest size of object");
+ assert(sz == size(), "just checking");
HeapWord* addr = (HeapWord*)this;
size_t hdr = header_size();
- Copy::fill_to_words(addr + hdr, size - hdr, deadbeefHeapWord);
+ Copy::fill_to_words(addr + hdr, sz - hdr, deadbeefHeapWord);
}
void FreeChunk::verifyList() const {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp Mon Jun 09 11:51:19 2008 -0400
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2001-2005 Sun Microsystems, Inc. All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ *
+ */
+
+//
+// Free block maintenance for Concurrent Mark Sweep Generation
+//
+// The main data structure for free blocks are
+// . an indexed array of small free blocks, and
+// . a dictionary of large free blocks
+//
+
+// No virtuals in FreeChunk (don't want any vtables).
+
+// A FreeChunk is merely a chunk that can be in a doubly linked list
+// and has a size field. NOTE: FreeChunks are distinguished from allocated
+// objects in two ways (by the sweeper), depending on whether the VM is 32 or
+// 64 bits.
+// In 32 bits or 64 bits without CompressedOops, the second word (prev) has the
+// LSB set to indicate a free chunk; allocated objects' klass() pointers
+// don't have their LSB set. The corresponding bit in the CMSBitMap is
+// set when the chunk is allocated. There are also blocks that "look free"
+// but are not part of the free list and should not be coalesced into larger
+// free blocks. These free blocks have their two LSB's set.
+
+class FreeChunk VALUE_OBJ_CLASS_SPEC {
+ friend class VMStructs;
+ // For 64 bit compressed oops, the markOop encodes both the size and the
+ // indication that this is a FreeChunk and not an object.
+ volatile size_t _size;
+ FreeChunk* _prev;
+ FreeChunk* _next;
+
+ markOop mark() const volatile { return (markOop)_size; }
+ void set_mark(markOop m) { _size = (size_t)m; }
+
+ public:
+ NOT_PRODUCT(static const size_t header_size();)
+
+ // Returns "true" if the address indicates that the block represents
+ // a free chunk.
+ static bool indicatesFreeChunk(const HeapWord* addr) {
+ // Force volatile read from addr because value might change between
+ // calls. We really want the read of _mark and _prev from this pointer
+ // to be volatile but making the fields volatile causes all sorts of
+ // compilation errors.
+ return ((volatile FreeChunk*)addr)->isFree();
+ }
+
+ bool isFree() const volatile {
+ LP64_ONLY(if (UseCompressedOops) return mark()->is_cms_free_chunk(); else)
+ return (((intptr_t)_prev) & 0x1) == 0x1;
+ }
+ bool cantCoalesce() const {
+ assert(isFree(), "can't get coalesce bit on not free");
+ return (((intptr_t)_prev) & 0x2) == 0x2;
+ }
+ void dontCoalesce() {
+ // the block should be free
+ assert(isFree(), "Should look like a free block");
+ _prev = (FreeChunk*)(((intptr_t)_prev) | 0x2);
+ }
+ FreeChunk* prev() const {
+ return (FreeChunk*)(((intptr_t)_prev) & ~(0x3));
+ }
+
+ debug_only(void* prev_addr() const { return (void*)&_prev; })
+
+ size_t size() const volatile {
+ LP64_ONLY(if (UseCompressedOops) return mark()->get_size(); else )
+ return _size;
+ }
+ void setSize(size_t sz) {
+ LP64_ONLY(if (UseCompressedOops) set_mark(markOopDesc::set_size_and_free(sz)); else )
+ _size = sz;
+ }
+
+ FreeChunk* next() const { return _next; }
+
+ void linkAfter(FreeChunk* ptr) {
+ linkNext(ptr);
+ if (ptr != NULL) ptr->linkPrev(this);
+ }
+ void linkAfterNonNull(FreeChunk* ptr) {
+ assert(ptr != NULL, "precondition violation");
+ linkNext(ptr);
+ ptr->linkPrev(this);
+ }
+ void linkNext(FreeChunk* ptr) { _next = ptr; }
+ void linkPrev(FreeChunk* ptr) {
+ LP64_ONLY(if (UseCompressedOops) _prev = ptr; else)
+ _prev = (FreeChunk*)((intptr_t)ptr | 0x1);
+ }
+ void clearPrev() { _prev = NULL; }
+ void clearNext() { _next = NULL; }
+ void markNotFree() {
+ LP64_ONLY(if (UseCompressedOops) set_mark(markOopDesc::prototype());)
+ // Also set _prev to null
+ _prev = NULL;
+ }
+
+ // Return the address past the end of this chunk
+ HeapWord* end() const { return ((HeapWord*) this) + size(); }
+
+ // debugging
+ void verify() const PRODUCT_RETURN;
+ void verifyList() const PRODUCT_RETURN;
+ void mangleAllocated(size_t size) PRODUCT_RETURN;
+ void mangleFreed(size_t size) PRODUCT_RETURN;
+};
+
+// Alignment helpers etc.
+#define numQuanta(x,y) ((x+y-1)/y)
+enum AlignmentConstants {
+ MinChunkSize = numQuanta(sizeof(FreeChunk), MinObjAlignmentInBytes) * MinObjAlignment
+};
+
--- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp Mon Jun 09 11:51:19 2008 -0400
@@ -23,6 +23,7 @@
*/
#define VM_STRUCTS_CMS(nonstatic_field, \
+ volatile_nonstatic_field, \
static_field) \
nonstatic_field(CompactibleFreeListSpace, _collector, CMSCollector*) \
nonstatic_field(CompactibleFreeListSpace, _bt, BlockOffsetArrayNonContigSpace) \
@@ -36,9 +37,9 @@
nonstatic_field(CMSCollector, _markBitMap, CMSBitMap) \
nonstatic_field(ConcurrentMarkSweepGeneration, _cmsSpace, CompactibleFreeListSpace*) \
static_field(ConcurrentMarkSweepThread, _collector, CMSCollector*) \
+ volatile_nonstatic_field(FreeChunk, _size, size_t) \
nonstatic_field(FreeChunk, _next, FreeChunk*) \
- nonstatic_field(FreeChunk, _prev, FreeChunk*) \
- nonstatic_field(FreeChunk, _size, size_t)
+ nonstatic_field(FreeChunk, _prev, FreeChunk*)
#define VM_TYPES_CMS(declare_type, \
declare_toplevel_type) \
--- a/hotspot/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/gc_implementation/includeDB_gc_concurrentMarkSweep Mon Jun 09 11:51:19 2008 -0400
@@ -206,6 +206,7 @@
freeBlockDictionary.hpp allocation.hpp
freeBlockDictionary.hpp debug.hpp
+freeBlockDictionary.hpp freeChunk.hpp
freeBlockDictionary.hpp globalDefinitions.hpp
freeBlockDictionary.hpp memRegion.hpp
freeBlockDictionary.hpp mutex.hpp
@@ -214,6 +215,14 @@
freeChunk.cpp copy.hpp
freeChunk.cpp freeBlockDictionary.hpp
+freeChunk.hpp allocation.hpp
+freeChunk.hpp debug.hpp
+freeChunk.hpp globalDefinitions.hpp
+freeChunk.hpp markOop.hpp
+freeChunk.hpp memRegion.hpp
+freeChunk.hpp mutex.hpp
+freeChunk.hpp ostream.hpp
+
freeList.cpp freeBlockDictionary.hpp
freeList.cpp freeList.hpp
freeList.cpp globals.hpp
--- a/hotspot/src/share/vm/oops/markOop.hpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/oops/markOop.hpp Mon Jun 09 11:51:19 2008 -0400
@@ -29,8 +29,10 @@
//
// Bit-format of an object header (most significant first):
//
-//
-// unused:0/25 hash:25/31 age:4 biased_lock:1 lock:2 = 32/64 bits
+// 32 bits: unused:0 hash:25 age:4 biased_lock:1 lock:2
+// 64 bits: unused:24 hash:31 cms:2 age:4 biased_lock:1 lock:2
+// unused:20 size:35 cms:2 age:4 biased_lock:1 lock:2 (if cms
+// free chunk)
//
// - hash contains the identity hash value: largest value is
// 31 bits, see os::random(). Also, 64-bit vm's require
@@ -91,6 +93,7 @@
biased_lock_bits = 1,
max_hash_bits = BitsPerWord - age_bits - lock_bits - biased_lock_bits,
hash_bits = max_hash_bits > 31 ? 31 : max_hash_bits,
+ cms_bits = LP64_ONLY(1) NOT_LP64(0),
epoch_bits = 2
};
@@ -106,7 +109,8 @@
enum { lock_shift = 0,
biased_lock_shift = lock_bits,
age_shift = lock_bits + biased_lock_bits,
- hash_shift = lock_bits + biased_lock_bits + age_bits,
+ cms_shift = age_shift + age_bits,
+ hash_shift = cms_shift + cms_bits,
epoch_shift = hash_shift
};
@@ -118,7 +122,9 @@
age_mask = right_n_bits(age_bits),
age_mask_in_place = age_mask << age_shift,
epoch_mask = right_n_bits(epoch_bits),
- epoch_mask_in_place = epoch_mask << epoch_shift
+ epoch_mask_in_place = epoch_mask << epoch_shift,
+ cms_mask = right_n_bits(cms_bits),
+ cms_mask_in_place = cms_mask << cms_shift
#ifndef _WIN64
,hash_mask = right_n_bits(hash_bits),
hash_mask_in_place = (address_word)hash_mask << hash_shift
@@ -360,4 +366,40 @@
// see the definition in markOop.cpp for the gory details
bool should_not_be_cached() const;
+
+ // These markOops indicate cms free chunk blocks and not objects.
+ // In 64 bit, the markOop is set to distinguish them from oops.
+ // These are defined in 32 bit mode for vmStructs.
+ const static uintptr_t cms_free_chunk_pattern = 0x1;
+
+ // Constants for the size field.
+ enum { size_shift = cms_shift + cms_bits,
+ size_bits = 35 // need for compressed oops 32G
+ };
+ // These values are too big for Win64
+ const static uintptr_t size_mask = LP64_ONLY(right_n_bits(size_bits))
+ NOT_LP64(0);
+ const static uintptr_t size_mask_in_place =
+ (address_word)size_mask << size_shift;
+
+#ifdef _LP64
+ static markOop cms_free_prototype() {
+ return markOop(((intptr_t)prototype() & ~cms_mask_in_place) |
+ ((cms_free_chunk_pattern & cms_mask) << cms_shift));
+ }
+ uintptr_t cms_encoding() const {
+ return mask_bits(value() >> cms_shift, cms_mask);
+ }
+ bool is_cms_free_chunk() const {
+ return is_neutral() &&
+ (cms_encoding() & cms_free_chunk_pattern) == cms_free_chunk_pattern;
+ }
+
+ size_t get_size() const { return (size_t)(value() >> size_shift); }
+ static markOop set_size_and_free(size_t size) {
+ assert((size & ~size_mask) == 0, "shouldn't overflow size field");
+ return markOop(((intptr_t)cms_free_prototype() & ~size_mask_in_place) |
+ (((intptr_t)size & size_mask) << size_shift));
+ }
+#endif // _LP64
};
--- a/hotspot/src/share/vm/runtime/arguments.cpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/runtime/arguments.cpp Mon Jun 09 11:51:19 2008 -0400
@@ -1174,7 +1174,7 @@
// field offset to determine free list chunk markers.
// Check that UseCompressedOops can be set with the max heap size allocated
// by ergonomics.
- if (!UseConcMarkSweepGC && MaxHeapSize <= max_heap_for_compressed_oops()) {
+ if (MaxHeapSize <= max_heap_for_compressed_oops()) {
if (FLAG_IS_DEFAULT(UseCompressedOops)) {
// Leave compressed oops off by default. Uncomment
// the following line to return it to default status.
--- a/hotspot/src/share/vm/runtime/vmStructs.cpp Fri Jun 06 11:47:26 2008 -0700
+++ b/hotspot/src/share/vm/runtime/vmStructs.cpp Mon Jun 09 11:51:19 2008 -0400
@@ -1694,7 +1694,12 @@
declare_constant(markOopDesc::no_hash) \
declare_constant(markOopDesc::no_hash_in_place) \
declare_constant(markOopDesc::no_lock_in_place) \
- declare_constant(markOopDesc::max_age)
+ declare_constant(markOopDesc::max_age) \
+ \
+ /* Constants in markOop used by CMS. */ \
+ declare_constant(markOopDesc::cms_shift) \
+ declare_constant(markOopDesc::cms_mask) \
+ declare_constant(markOopDesc::size_shift) \
/* NOTE that we do not use the last_entry() macro here; it is used */
/* in vmStructs_<os>_<cpu>.hpp's VM_LONG_CONSTANTS_OS_CPU macro (and */
@@ -1958,6 +1963,7 @@
GENERATE_STATIC_VM_STRUCT_ENTRY)
VM_STRUCTS_CMS(GENERATE_NONSTATIC_VM_STRUCT_ENTRY, \
+ GENERATE_NONSTATIC_VM_STRUCT_ENTRY, \
GENERATE_STATIC_VM_STRUCT_ENTRY)
#endif // SERIALGC
@@ -2099,6 +2105,7 @@
CHECK_STATIC_VM_STRUCT_ENTRY);
VM_STRUCTS_CMS(CHECK_NONSTATIC_VM_STRUCT_ENTRY,
+ CHECK_VOLATILE_NONSTATIC_VM_STRUCT_ENTRY,
CHECK_STATIC_VM_STRUCT_ENTRY);
#endif // SERIALGC
@@ -2203,6 +2210,7 @@
debug_only(VM_STRUCTS_PARALLELGC(ENSURE_FIELD_TYPE_PRESENT, \
ENSURE_FIELD_TYPE_PRESENT));
debug_only(VM_STRUCTS_CMS(ENSURE_FIELD_TYPE_PRESENT, \
+ ENSURE_FIELD_TYPE_PRESENT, \
ENSURE_FIELD_TYPE_PRESENT));
#endif // SERIALGC
debug_only(VM_STRUCTS_CPU(ENSURE_FIELD_TYPE_PRESENT, \