# HG changeset patch # User jmasa # Date 1348036542 25200 # Node ID 944e56f74fbac6338aeee869ddd2b6a99b7cc510 # Parent f5e31fb617387a345251700e2ca6033d529ab4f0 7045397: NPG: Add freelists to class loader arenas. Reviewed-by: coleenp, stefank, jprovino, ohair diff -r f5e31fb61738 -r 944e56f74fba hotspot/make/excludeSrc.make --- a/hotspot/make/excludeSrc.make Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/make/excludeSrc.make Tue Sep 18 23:35:42 2012 -0700 @@ -79,10 +79,10 @@ CXXFLAGS += -DSERIALGC CFLAGS += -DSERIALGC Src_Files_EXCLUDE += \ - binaryTreeDictionary.cpp cmsAdaptiveSizePolicy.cpp cmsCollectorPolicy.cpp \ + cmsAdaptiveSizePolicy.cpp cmsCollectorPolicy.cpp \ cmsGCAdaptivePolicyCounters.cpp cmsLockVerifier.cpp cmsPermGen.cpp compactibleFreeListSpace.cpp \ - concurrentMarkSweepGeneration.cpp concurrentMarkSweepThread.cpp freeBlockDictionary.cpp \ - freeChunk.cpp freeList.cpp promotionInfo.cpp vmCMSOperations.cpp collectionSetChooser.cpp \ + concurrentMarkSweepGeneration.cpp concurrentMarkSweepThread.cpp \ + freeChunk.cpp adaptiveFreeList.cpp promotionInfo.cpp vmCMSOperations.cpp collectionSetChooser.cpp \ concurrentG1Refine.cpp concurrentG1RefineThread.cpp concurrentMark.cpp concurrentMarkThread.cpp \ dirtyCardQueue.cpp g1AllocRegion.cpp g1BlockOffsetTable.cpp g1CollectedHeap.cpp g1GCPhaseTimes.cpp \ g1CollectorPolicy.cpp g1ErgoVerbose.cpp g1_globals.cpp g1HRPrinter.cpp g1MarkSweep.cpp \ diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" +#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" +#include "memory/freeBlockDictionary.hpp" +#include "memory/sharedHeap.hpp" +#include "runtime/globals.hpp" +#include "runtime/mutex.hpp" +#include "runtime/vmThread.hpp" + +template <> +void AdaptiveFreeList::print_on(outputStream* st, const char* c) const { + if (c != NULL) { + st->print("%16s", c); + } else { + st->print(SIZE_FORMAT_W(16), size()); + } + st->print("\t" + SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" + SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\n", + bfr_surp(), surplus(), desired(), prev_sweep(), before_sweep(), + count(), coal_births(), coal_deaths(), split_births(), split_deaths()); +} + +template +AdaptiveFreeList::AdaptiveFreeList() : FreeList(), _hint(0) { + init_statistics(); +} + +template +AdaptiveFreeList::AdaptiveFreeList(Chunk* fc) : FreeList(fc), _hint(0) { + init_statistics(); +#ifndef PRODUCT + _allocation_stats.set_returned_bytes(size() * HeapWordSize); +#endif +} + +template +void AdaptiveFreeList::initialize() { + FreeList::initialize(); + set_hint(0); + init_statistics(true /* split_birth */); +} + +template +void AdaptiveFreeList::reset(size_t hint) { + FreeList::reset(); + set_hint(hint); +} + +#ifndef PRODUCT +template +void AdaptiveFreeList::assert_proper_lock_protection_work() const { + assert(protecting_lock() != NULL, "Don't call this directly"); + assert(ParallelGCThreads > 0, "Don't call this directly"); + Thread* thr = Thread::current(); + if (thr->is_VM_thread() || thr->is_ConcurrentGC_thread()) { + // assert that we are holding the freelist lock + } else if (thr->is_GC_task_thread()) { + assert(protecting_lock()->owned_by_self(), "FreeList RACE DETECTED"); + } else if (thr->is_Java_thread()) { + assert(!SafepointSynchronize::is_at_safepoint(), "Should not be executing"); + } else { + ShouldNotReachHere(); // unaccounted thread type? + } +} +#endif +template +void AdaptiveFreeList::init_statistics(bool split_birth) { + _allocation_stats.initialize(split_birth); +} + +template +size_t AdaptiveFreeList::get_better_size() { + + // A candidate chunk has been found. If it is already under + // populated and there is a hinT, REturn the hint(). Else + // return the size of this chunk. + if (surplus() <= 0) { + if (hint() != 0) { + return hint(); + } else { + return size(); + } + } else { + // This list has a surplus so use it. + return size(); + } +} + + +template +void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk) { + assert_proper_lock_protection(); + return_chunk_at_head(chunk, true); +} + +template +void AdaptiveFreeList::return_chunk_at_head(Chunk* chunk, bool record_return) { + FreeList::return_chunk_at_head(chunk, record_return); +#ifdef ASSERT + if (record_return) { + increment_returned_bytes_by(size()*HeapWordSize); + } +#endif +} + +template +void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk) { + return_chunk_at_tail(chunk, true); +} + +template +void AdaptiveFreeList::return_chunk_at_tail(Chunk* chunk, bool record_return) { + FreeList::return_chunk_at_tail(chunk, record_return); +#ifdef ASSERT + if (record_return) { + increment_returned_bytes_by(size()*HeapWordSize); + } +#endif +} + +#ifndef PRODUCT +template +void AdaptiveFreeList::verify_stats() const { + // The +1 of the LH comparand is to allow some "looseness" in + // checking: we usually call this interface when adding a block + // and we'll subsequently update the stats; we cannot update the + // stats beforehand because in the case of the large-block BT + // dictionary for example, this might be the first block and + // in that case there would be no place that we could record + // the stats (which are kept in the block itself). + assert((_allocation_stats.prev_sweep() + _allocation_stats.split_births() + + _allocation_stats.coal_births() + 1) // Total Production Stock + 1 + >= (_allocation_stats.split_deaths() + _allocation_stats.coal_deaths() + + (ssize_t)count()), // Total Current Stock + depletion + err_msg("FreeList " PTR_FORMAT " of size " SIZE_FORMAT + " violates Conservation Principle: " + "prev_sweep(" SIZE_FORMAT ")" + " + split_births(" SIZE_FORMAT ")" + " + coal_births(" SIZE_FORMAT ") + 1 >= " + " split_deaths(" SIZE_FORMAT ")" + " coal_deaths(" SIZE_FORMAT ")" + " + count(" SSIZE_FORMAT ")", + this, size(), _allocation_stats.prev_sweep(), _allocation_stats.split_births(), + _allocation_stats.split_births(), _allocation_stats.split_deaths(), + _allocation_stats.coal_deaths(), count())); +} +#endif + +// Needs to be after the definitions have been seen. +template class AdaptiveFreeList; diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_MEMORY_ADAPTIVEFREELIST_HPP +#define SHARE_VM_MEMORY_ADAPTIVEFREELIST_HPP + +#include "memory/freeList.hpp" +#include "gc_implementation/shared/allocationStats.hpp" + +class CompactibleFreeListSpace; + +// A class for maintaining a free list of Chunk's. The FreeList +// maintains a the structure of the list (head, tail, etc.) plus +// statistics for allocations from the list. The links between items +// are not part of FreeList. The statistics are +// used to make decisions about coalescing Chunk's when they +// are swept during collection. +// +// See the corresponding .cpp file for a description of the specifics +// for that implementation. + +class Mutex; + +template +class AdaptiveFreeList : public FreeList { + friend class CompactibleFreeListSpace; + friend class VMStructs; + // friend class PrintTreeCensusClosure; + + size_t _hint; // next larger size list with a positive surplus + + AllocationStats _allocation_stats; // allocation-related statistics + + public: + + AdaptiveFreeList(); + AdaptiveFreeList(Chunk* fc); + + using FreeList::assert_proper_lock_protection; +#ifdef ASSERT + using FreeList::protecting_lock; +#endif + using FreeList::count; + using FreeList::size; + using FreeList::verify_chunk_in_free_list; + using FreeList::getFirstNChunksFromList; + using FreeList::print_on; + void return_chunk_at_head(Chunk* fc, bool record_return); + void return_chunk_at_head(Chunk* fc); + void return_chunk_at_tail(Chunk* fc, bool record_return); + void return_chunk_at_tail(Chunk* fc); + using FreeList::return_chunk_at_tail; + using FreeList::remove_chunk; + using FreeList::prepend; + using FreeList::print_labels_on; + using FreeList::get_chunk_at_head; + + // Initialize. + void initialize(); + + // Reset the head, tail, hint, and count of a free list. + void reset(size_t hint); + + void assert_proper_lock_protection_work() const PRODUCT_RETURN; + + void print_on(outputStream* st, const char* c = NULL) const; + + size_t hint() const { + return _hint; + } + void set_hint(size_t v) { + assert_proper_lock_protection(); + assert(v == 0 || size() < v, "Bad hint"); + _hint = v; + } + + size_t get_better_size(); + + // Accessors for statistics + void init_statistics(bool split_birth = false); + + AllocationStats* allocation_stats() { + assert_proper_lock_protection(); + return &_allocation_stats; + } + + ssize_t desired() const { + return _allocation_stats.desired(); + } + void set_desired(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_desired(v); + } + void compute_desired(float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate) { + assert_proper_lock_protection(); + _allocation_stats.compute_desired(count(), + inter_sweep_current, + inter_sweep_estimate, + intra_sweep_estimate); + } + ssize_t coal_desired() const { + return _allocation_stats.coal_desired(); + } + void set_coal_desired(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_desired(v); + } + + ssize_t surplus() const { + return _allocation_stats.surplus(); + } + void set_surplus(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_surplus(v); + } + void increment_surplus() { + assert_proper_lock_protection(); + _allocation_stats.increment_surplus(); + } + void decrement_surplus() { + assert_proper_lock_protection(); + _allocation_stats.decrement_surplus(); + } + + ssize_t bfr_surp() const { + return _allocation_stats.bfr_surp(); + } + void set_bfr_surp(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_bfr_surp(v); + } + ssize_t prev_sweep() const { + return _allocation_stats.prev_sweep(); + } + void set_prev_sweep(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_prev_sweep(v); + } + ssize_t before_sweep() const { + return _allocation_stats.before_sweep(); + } + void set_before_sweep(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_before_sweep(v); + } + + ssize_t coal_births() const { + return _allocation_stats.coal_births(); + } + void set_coal_births(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_births(v); + } + void increment_coal_births() { + assert_proper_lock_protection(); + _allocation_stats.increment_coal_births(); + } + + ssize_t coal_deaths() const { + return _allocation_stats.coal_deaths(); + } + void set_coal_deaths(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_coal_deaths(v); + } + void increment_coal_deaths() { + assert_proper_lock_protection(); + _allocation_stats.increment_coal_deaths(); + } + + ssize_t split_births() const { + return _allocation_stats.split_births(); + } + void set_split_births(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_split_births(v); + } + void increment_split_births() { + assert_proper_lock_protection(); + _allocation_stats.increment_split_births(); + } + + ssize_t split_deaths() const { + return _allocation_stats.split_deaths(); + } + void set_split_deaths(ssize_t v) { + assert_proper_lock_protection(); + _allocation_stats.set_split_deaths(v); + } + void increment_split_deaths() { + assert_proper_lock_protection(); + _allocation_stats.increment_split_deaths(); + } + +#ifndef PRODUCT + // For debugging. The "_returned_bytes" in all the lists are summed + // and compared with the total number of bytes swept during a + // collection. + size_t returned_bytes() const { return _allocation_stats.returned_bytes(); } + void set_returned_bytes(size_t v) { _allocation_stats.set_returned_bytes(v); } + void increment_returned_bytes_by(size_t v) { + _allocation_stats.set_returned_bytes(_allocation_stats.returned_bytes() + v); + } + // Stats verification + void verify_stats() const; +#endif // NOT PRODUCT +}; + +#endif // SHARE_VM_MEMORY_ADAPTIVEFREELIST_HPP diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -91,7 +91,7 @@ _collector(NULL) { assert(sizeof(FreeChunk) / BytesPerWord <= MinChunkSize, - "FreeChunk is larger than expected"); + "FreeChunk is larger than expected"); _bt.set_space(this); initialize(mr, SpaceDecorator::Clear, SpaceDecorator::Mangle); // We have all of "mr", all of which we place in the dictionary @@ -101,14 +101,14 @@ // implementation, namely, the simple binary tree (splaying // temporarily disabled). switch (dictionaryChoice) { + case FreeBlockDictionary::dictionaryBinaryTree: + _dictionary = new BinaryTreeDictionary(mr); + break; case FreeBlockDictionary::dictionarySplayTree: case FreeBlockDictionary::dictionarySkipList: default: warning("dictionaryChoice: selected option not understood; using" " default BinaryTreeDictionary implementation instead."); - case FreeBlockDictionary::dictionaryBinaryTree: - _dictionary = new BinaryTreeDictionary(mr, use_adaptive_freelists); - break; } assert(_dictionary != NULL, "CMS dictionary initialization"); // The indexed free lists are initially all empty and are lazily @@ -453,7 +453,7 @@ reportIndexedFreeListStatistics(); gclog_or_tty->print_cr("Layout of Indexed Freelists"); gclog_or_tty->print_cr("---------------------------"); - FreeList::print_labels_on(st, "size"); + AdaptiveFreeList::print_labels_on(st, "size"); for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { _indexedFreeList[i].print_on(gclog_or_tty); for (FreeChunk* fc = _indexedFreeList[i].head(); fc != NULL; @@ -1319,7 +1319,7 @@ size_t currSize = numWords + MinChunkSize; assert(currSize % MinObjAlignment == 0, "currSize should be aligned"); for (i = currSize; i < IndexSetSize; i += IndexSetStride) { - FreeList* fl = &_indexedFreeList[i]; + AdaptiveFreeList* fl = &_indexedFreeList[i]; if (fl->head()) { ret = getFromListGreater(fl, numWords); assert(ret == NULL || ret->is_free(), "Should be returning a free chunk"); @@ -1702,7 +1702,9 @@ _dictionary->return_chunk(chunk); #ifndef PRODUCT if (CMSCollector::abstract_state() != CMSCollector::Sweeping) { - TreeChunk::as_TreeChunk(chunk)->list()->verify_stats(); + TreeChunk* tc = TreeChunk::as_TreeChunk(chunk); + TreeList* tl = tc->list(); + tl->verify_stats(); } #endif // PRODUCT } @@ -1745,7 +1747,7 @@ { MutexLockerEx x(lock, Mutex::_no_safepoint_check_flag); ec = dictionary()->find_largest_dict(); // get largest block - if (ec != NULL && ec->end() == chunk) { + if (ec != NULL && ec->end() == (uintptr_t*) chunk) { // It's a coterminal block - we can coalesce. size_t old_size = ec->size(); coalDeath(old_size); @@ -1850,11 +1852,11 @@ the excess is >= MIN_CHUNK. */ size_t start = align_object_size(numWords + MinChunkSize); if (start < IndexSetSize) { - FreeList* it = _indexedFreeList; + AdaptiveFreeList* it = _indexedFreeList; size_t hint = _indexedFreeList[start].hint(); while (hint < IndexSetSize) { assert(hint % MinObjAlignment == 0, "hint should be aligned"); - FreeList *fl = &_indexedFreeList[hint]; + AdaptiveFreeList *fl = &_indexedFreeList[hint]; if (fl->surplus() > 0 && fl->head() != NULL) { // Found a list with surplus, reset original hint // and split out a free chunk which is returned. @@ -1873,7 +1875,7 @@ } /* Requires fl->size >= numWords + MinChunkSize */ -FreeChunk* CompactibleFreeListSpace::getFromListGreater(FreeList* fl, +FreeChunk* CompactibleFreeListSpace::getFromListGreater(AdaptiveFreeList* fl, size_t numWords) { FreeChunk *curr = fl->head(); size_t oldNumWords = curr->size(); @@ -2155,7 +2157,7 @@ assert_locked(); size_t i; for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - FreeList* fl = &_indexedFreeList[i]; + AdaptiveFreeList* fl = &_indexedFreeList[i]; if (PrintFLSStatistics > 1) { gclog_or_tty->print("size[%d] : ", i); } @@ -2174,7 +2176,7 @@ assert_locked(); size_t i; for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - FreeList *fl = &_indexedFreeList[i]; + AdaptiveFreeList *fl = &_indexedFreeList[i]; fl->set_surplus(fl->count() - (ssize_t)((double)fl->desired() * CMSSmallSplitSurplusPercent)); } @@ -2185,7 +2187,7 @@ size_t i; size_t h = IndexSetSize; for (i = IndexSetSize - 1; i != 0; i -= IndexSetStride) { - FreeList *fl = &_indexedFreeList[i]; + AdaptiveFreeList *fl = &_indexedFreeList[i]; fl->set_hint(h); if (fl->surplus() > 0) { h = i; @@ -2197,7 +2199,7 @@ assert_locked(); size_t i; for (i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - FreeList *fl = &_indexedFreeList[i]; + AdaptiveFreeList *fl = &_indexedFreeList[i]; fl->set_prev_sweep(fl->count()); fl->set_coal_births(0); fl->set_coal_deaths(0); @@ -2224,7 +2226,7 @@ bool CompactibleFreeListSpace::coalOverPopulated(size_t size) { if (size < SmallForDictionary) { - FreeList *fl = &_indexedFreeList[size]; + AdaptiveFreeList *fl = &_indexedFreeList[size]; return (fl->coal_desired() < 0) || ((int)fl->count() > fl->coal_desired()); } else { @@ -2234,14 +2236,14 @@ void CompactibleFreeListSpace::smallCoalBirth(size_t size) { assert(size < SmallForDictionary, "Size too large for indexed list"); - FreeList *fl = &_indexedFreeList[size]; + AdaptiveFreeList *fl = &_indexedFreeList[size]; fl->increment_coal_births(); fl->increment_surplus(); } void CompactibleFreeListSpace::smallCoalDeath(size_t size) { assert(size < SmallForDictionary, "Size too large for indexed list"); - FreeList *fl = &_indexedFreeList[size]; + AdaptiveFreeList *fl = &_indexedFreeList[size]; fl->increment_coal_deaths(); fl->decrement_surplus(); } @@ -2250,7 +2252,7 @@ if (size < SmallForDictionary) { smallCoalBirth(size); } else { - dictionary()->dict_census_udpate(size, + dictionary()->dict_census_update(size, false /* split */, true /* birth */); } @@ -2260,7 +2262,7 @@ if(size < SmallForDictionary) { smallCoalDeath(size); } else { - dictionary()->dict_census_udpate(size, + dictionary()->dict_census_update(size, false /* split */, false /* birth */); } @@ -2268,14 +2270,14 @@ void CompactibleFreeListSpace::smallSplitBirth(size_t size) { assert(size < SmallForDictionary, "Size too large for indexed list"); - FreeList *fl = &_indexedFreeList[size]; + AdaptiveFreeList *fl = &_indexedFreeList[size]; fl->increment_split_births(); fl->increment_surplus(); } void CompactibleFreeListSpace::smallSplitDeath(size_t size) { assert(size < SmallForDictionary, "Size too large for indexed list"); - FreeList *fl = &_indexedFreeList[size]; + AdaptiveFreeList *fl = &_indexedFreeList[size]; fl->increment_split_deaths(); fl->decrement_surplus(); } @@ -2284,7 +2286,7 @@ if (size < SmallForDictionary) { smallSplitBirth(size); } else { - dictionary()->dict_census_udpate(size, + dictionary()->dict_census_update(size, true /* split */, true /* birth */); } @@ -2294,7 +2296,7 @@ if (size < SmallForDictionary) { smallSplitDeath(size); } else { - dictionary()->dict_census_udpate(size, + dictionary()->dict_census_update(size, true /* split */, false /* birth */); } @@ -2517,10 +2519,10 @@ #ifndef PRODUCT void CompactibleFreeListSpace::check_free_list_consistency() const { - assert(_dictionary->min_size() <= IndexSetSize, + assert((TreeChunk::min_size() <= IndexSetSize), "Some sizes can't be allocated without recourse to" " linear allocation buffers"); - assert(BinaryTreeDictionary::min_tree_chunk_size*HeapWordSize == sizeof(TreeChunk), + assert((TreeChunk::min_size()*HeapWordSize == sizeof(TreeChunk)), "else MIN_TREE_CHUNK_SIZE is wrong"); assert(IndexSetStart != 0, "IndexSetStart not initialized"); assert(IndexSetStride != 0, "IndexSetStride not initialized"); @@ -2529,15 +2531,15 @@ void CompactibleFreeListSpace::printFLCensus(size_t sweep_count) const { assert_lock_strong(&_freelistLock); - FreeList total; + AdaptiveFreeList total; gclog_or_tty->print("end sweep# " SIZE_FORMAT "\n", sweep_count); - FreeList::print_labels_on(gclog_or_tty, "size"); + AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); size_t total_free = 0; for (size_t i = IndexSetStart; i < IndexSetSize; i += IndexSetStride) { - const FreeList *fl = &_indexedFreeList[i]; + const AdaptiveFreeList *fl = &_indexedFreeList[i]; total_free += fl->count() * fl->size(); if (i % (40*IndexSetStride) == 0) { - FreeList::print_labels_on(gclog_or_tty, "size"); + AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); } fl->print_on(gclog_or_tty); total.set_bfr_surp( total.bfr_surp() + fl->bfr_surp() ); @@ -2620,7 +2622,7 @@ res = _cfls->getChunkFromDictionaryExact(word_sz); if (res == NULL) return NULL; } else { - FreeList* fl = &_indexedFreeList[word_sz]; + AdaptiveFreeList* fl = &_indexedFreeList[word_sz]; if (fl->count() == 0) { // Attempt to refill this local free list. get_from_global_pool(word_sz, fl); @@ -2640,7 +2642,7 @@ // Get a chunk of blocks of the right size and update related // book-keeping stats -void CFLS_LAB::get_from_global_pool(size_t word_sz, FreeList* fl) { +void CFLS_LAB::get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl) { // Get the #blocks we want to claim size_t n_blks = (size_t)_blocks_to_claim[word_sz].average(); assert(n_blks > 0, "Error"); @@ -2722,7 +2724,7 @@ if (num_retire > 0) { _cfls->_indexedFreeList[i].prepend(&_indexedFreeList[i]); // Reset this list. - _indexedFreeList[i] = FreeList(); + _indexedFreeList[i] = AdaptiveFreeList(); _indexedFreeList[i].set_size(i); } } @@ -2736,7 +2738,7 @@ } } -void CompactibleFreeListSpace:: par_get_chunk_of_blocks(size_t word_sz, size_t n, FreeList* fl) { +void CompactibleFreeListSpace:: par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl) { assert(fl->count() == 0, "Precondition."); assert(word_sz < CompactibleFreeListSpace::IndexSetSize, "Precondition"); @@ -2752,12 +2754,12 @@ (cur_sz < CompactibleFreeListSpace::IndexSetSize) && (CMSSplitIndexedFreeListBlocks || k <= 1); k++, cur_sz = k * word_sz) { - FreeList fl_for_cur_sz; // Empty. + AdaptiveFreeList fl_for_cur_sz; // Empty. fl_for_cur_sz.set_size(cur_sz); { MutexLockerEx x(_indexedFreeListParLocks[cur_sz], Mutex::_no_safepoint_check_flag); - FreeList* gfl = &_indexedFreeList[cur_sz]; + AdaptiveFreeList* gfl = &_indexedFreeList[cur_sz]; if (gfl->count() != 0) { // nn is the number of chunks of size cur_sz that // we'd need to split k-ways each, in order to create @@ -2832,12 +2834,11 @@ MutexLockerEx x(parDictionaryAllocLock(), Mutex::_no_safepoint_check_flag); while (n > 0) { - fc = dictionary()->get_chunk(MAX2(n * word_sz, - _dictionary->min_size()), + fc = dictionary()->get_chunk(MAX2(n * word_sz, _dictionary->min_size()), FreeBlockDictionary::atLeast); if (fc != NULL) { _bt.allocated((HeapWord*)fc, fc->size(), true /* reducing */); // update _unallocated_blk - dictionary()->dict_census_udpate(fc->size(), + dictionary()->dict_census_update(fc->size(), true /*split*/, false /*birth*/); break; @@ -2890,7 +2891,7 @@ fc->set_size(prefix_size); if (rem >= IndexSetSize) { returnChunkToDictionary(rem_fc); - dictionary()->dict_census_udpate(rem, true /*split*/, true /*birth*/); + dictionary()->dict_census_update(rem, true /*split*/, true /*birth*/); rem_fc = NULL; } // Otherwise, return it to the small list below. diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/compactibleFreeListSpace.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -25,6 +25,7 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_COMPACTIBLEFREELISTSPACE_HPP #define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_COMPACTIBLEFREELISTSPACE_HPP +#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" #include "gc_implementation/concurrentMarkSweep/promotionInfo.hpp" #include "memory/binaryTreeDictionary.hpp" #include "memory/blockOffsetTable.inline.hpp" @@ -38,6 +39,7 @@ class CompactibleFreeListSpace; class BlkClosure; class BlkClosureCareful; +class FreeChunk; class UpwardsObjectClosure; class ObjectClosureCareful; class Klass; @@ -131,7 +133,7 @@ FreeBlockDictionary::DictionaryChoice _dictionaryChoice; FreeBlockDictionary* _dictionary; // ptr to dictionary for large size blocks - FreeList _indexedFreeList[IndexSetSize]; + AdaptiveFreeList _indexedFreeList[IndexSetSize]; // indexed array for small size blocks // allocation stategy bool _fitStrategy; // Use best fit strategy. @@ -168,7 +170,7 @@ // If the count of "fl" is negative, it's absolute value indicates a // number of free chunks that had been previously "borrowed" from global // list of size "word_sz", and must now be decremented. - void par_get_chunk_of_blocks(size_t word_sz, size_t n, FreeList* fl); + void par_get_chunk_of_blocks(size_t word_sz, size_t n, AdaptiveFreeList* fl); // Allocation helper functions // Allocate using a strategy that takes from the indexed free lists @@ -214,7 +216,7 @@ // and return it. The split off remainder is returned to // the free lists. The old name for getFromListGreater // was lookInListGreater. - FreeChunk* getFromListGreater(FreeList* fl, size_t numWords); + FreeChunk* getFromListGreater(AdaptiveFreeList* fl, size_t numWords); // Get a chunk in the indexed free list or dictionary, // by considering a larger chunk and splitting it. FreeChunk* getChunkFromGreater(size_t numWords); @@ -621,7 +623,7 @@ CompactibleFreeListSpace* _cfls; // Our local free lists. - FreeList _indexedFreeList[CompactibleFreeListSpace::IndexSetSize]; + AdaptiveFreeList _indexedFreeList[CompactibleFreeListSpace::IndexSetSize]; // Initialized from a command-line arg. @@ -634,7 +636,7 @@ size_t _num_blocks [CompactibleFreeListSpace::IndexSetSize]; // Internal work method - void get_from_global_pool(size_t word_sz, FreeList* fl); + void get_from_global_pool(size_t word_sz, AdaptiveFreeList* fl); public: CFLS_LAB(CompactibleFreeListSpace* cfls); diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/concurrentMarkSweepGeneration.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -9143,7 +9143,7 @@ size_t shrinkable_size_in_bytes = chunk_at_end->size(); size_t aligned_shrinkable_size_in_bytes = align_size_down(shrinkable_size_in_bytes, os::vm_page_size()); - assert(unallocated_start <= chunk_at_end->end(), + assert(unallocated_start <= (HeapWord*) chunk_at_end->end(), "Inconsistent chunk at end of space"); size_t bytes = MIN2(desired_bytes, aligned_shrinkable_size_in_bytes); size_t word_size_before = heap_word_size(_virtual_space.committed_size()); @@ -9210,7 +9210,7 @@ assert(_cmsSpace->unallocated_block() <= _cmsSpace->end(), "Inconsistency at end of space"); - assert(chunk_at_end->end() == _cmsSpace->end(), + assert(chunk_at_end->end() == (uintptr_t*) _cmsSpace->end(), "Shrinking is inconsistent"); return; } diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/freeChunk.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -133,7 +133,7 @@ } // Return the address past the end of this chunk - HeapWord* end() const { return ((HeapWord*) this) + size(); } + uintptr_t* end() const { return ((uintptr_t*) this) + size(); } // debugging void verify() const PRODUCT_RETURN; diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp --- a/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/concurrentMarkSweep/vmStructs_cms.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -25,6 +25,8 @@ #ifndef SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMSTRUCTS_CMS_HPP #define SHARE_VM_GC_IMPLEMENTATION_CONCURRENTMARKSWEEP_VMSTRUCTS_CMS_HPP +typedef BinaryTreeDictionary AFLBinaryTreeDictionary; + #define VM_STRUCTS_CMS(nonstatic_field, \ volatile_nonstatic_field, \ static_field) \ @@ -38,14 +40,8 @@ nonstatic_field(CMSCollector, _markBitMap, CMSBitMap) \ nonstatic_field(ConcurrentMarkSweepGeneration, _cmsSpace, CompactibleFreeListSpace*) \ static_field(ConcurrentMarkSweepThread, _collector, CMSCollector*) \ - volatile_nonstatic_field(FreeChunk, _size, size_t) \ - nonstatic_field(FreeChunk, _next, FreeChunk*) \ - nonstatic_field(FreeChunk, _prev, FreeChunk*) \ nonstatic_field(LinearAllocBlock, _word_size, size_t) \ - nonstatic_field(FreeList, _size, size_t) \ - nonstatic_field(FreeList, _count, ssize_t) \ - nonstatic_field(BinaryTreeDictionary,_total_size, size_t) \ - nonstatic_field(CompactibleFreeListSpace, _dictionary, FreeBlockDictionary*) \ + nonstatic_field(AFLBinaryTreeDictionary, _total_size, size_t) \ nonstatic_field(CompactibleFreeListSpace, _indexedFreeList[0], FreeList) \ nonstatic_field(CompactibleFreeListSpace, _smallLinearAllocBlock, LinearAllocBlock) @@ -60,19 +56,17 @@ declare_toplevel_type(CMSCollector) \ declare_toplevel_type(CMSBitMap) \ declare_toplevel_type(FreeChunk) \ + declare_toplevel_type(Metablock) \ declare_toplevel_type(ConcurrentMarkSweepThread*) \ declare_toplevel_type(ConcurrentMarkSweepGeneration*) \ declare_toplevel_type(SurrogateLockerThread*) \ declare_toplevel_type(CompactibleFreeListSpace*) \ declare_toplevel_type(CMSCollector*) \ - declare_toplevel_type(FreeChunk*) \ - declare_toplevel_type(BinaryTreeDictionary*) \ - declare_toplevel_type(FreeBlockDictionary*) \ - declare_toplevel_type(FreeList*) \ - declare_toplevel_type(FreeList) \ + declare_toplevel_type(AFLBinaryTreeDictionary*) \ declare_toplevel_type(LinearAllocBlock) \ declare_toplevel_type(FreeBlockDictionary) \ - declare_type(BinaryTreeDictionary, FreeBlockDictionary) + declare_type(AFLBinaryTreeDictionary, FreeBlockDictionary) \ + declare_type(AFLBinaryTreeDictionary, FreeBlockDictionary) \ #define VM_INT_CONSTANTS_CMS(declare_constant) \ declare_constant(Generation::ConcurrentMarkSweep) \ diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp --- a/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/gc_implementation/shared/vmGCOperations.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -191,7 +191,7 @@ class VM_CollectForMetadataAllocation: public VM_GC_Operation { private: MetaWord* _result; - size_t _size; // size of object to be allocated + size_t _size; // size of object to be allocated Metaspace::MetadataType _mdtype; ClassLoaderData* _loader_data; public: diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/binaryTreeDictionary.cpp --- a/hotspot/src/share/vm/memory/binaryTreeDictionary.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/binaryTreeDictionary.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -25,9 +25,15 @@ #include "precompiled.hpp" #include "gc_implementation/shared/allocationStats.hpp" #include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.hpp" +#include "memory/freeBlockDictionary.hpp" +#include "memory/metablock.hpp" +#include "memory/metachunk.hpp" #include "runtime/globals.hpp" #include "utilities/ostream.hpp" #ifndef SERIALGC +#include "gc_implementation/concurrentMarkSweep/adaptiveFreeList.hpp" +#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" #include "gc_implementation/shared/spaceDecorator.hpp" #include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" #endif // SERIALGC @@ -37,15 +43,18 @@ // This is currently used in the Concurrent Mark&Sweep implementation. //////////////////////////////////////////////////////////////////////////////// -template -TreeChunk* TreeChunk::as_TreeChunk(Chunk* fc) { +template class FreeList_t> +size_t TreeChunk::_min_tree_chunk_size = sizeof(TreeChunk)/HeapWordSize; + +template class FreeList_t> +TreeChunk* TreeChunk::as_TreeChunk(Chunk_t* fc) { // Do some assertion checking here. - return (TreeChunk*) fc; + return (TreeChunk*) fc; } -template -void TreeChunk::verify_tree_chunk_list() const { - TreeChunk* nextTC = (TreeChunk*)next(); +template class FreeList_t> +void TreeChunk::verify_tree_chunk_list() const { + TreeChunk* nextTC = (TreeChunk*)next(); if (prev() != NULL) { // interior list node shouldn'r have tree fields guarantee(embedded_list()->parent() == NULL && embedded_list()->left() == NULL && embedded_list()->right() == NULL, "should be clear"); @@ -57,53 +66,113 @@ } } +template class FreeList_t> +TreeList::TreeList() {} -template -TreeList* TreeList::as_TreeList(TreeChunk* tc) { +template class FreeList_t> +TreeList* +TreeList::as_TreeList(TreeChunk* tc) { // This first free chunk in the list will be the tree list. - assert(tc->size() >= BinaryTreeDictionary::min_tree_chunk_size, "Chunk is too small for a TreeChunk"); - TreeList* tl = tc->embedded_list(); + assert((tc->size() >= (TreeChunk::min_size())), + "Chunk is too small for a TreeChunk"); + TreeList* tl = tc->embedded_list(); + tl->initialize(); tc->set_list(tl); -#ifdef ASSERT - tl->set_protecting_lock(NULL); -#endif - tl->set_hint(0); tl->set_size(tc->size()); tl->link_head(tc); tl->link_tail(tc); tl->set_count(1); - tl->init_statistics(true /* split_birth */); - tl->set_parent(NULL); - tl->set_left(NULL); - tl->set_right(NULL); + + return tl; +} + + +template class FreeList_t> +TreeList* +get_chunk(size_t size, enum FreeBlockDictionary::Dither dither) { + FreeBlockDictionary::verify_par_locked(); + Chunk_t* res = get_chunk_from_tree(size, dither); + assert(res == NULL || res->is_free(), + "Should be returning a free chunk"); + assert(dither != FreeBlockDictionary::exactly || + res->size() == size, "Not correct size"); + return res; +} + +template class FreeList_t> +TreeList* +TreeList::as_TreeList(HeapWord* addr, size_t size) { + TreeChunk* tc = (TreeChunk*) addr; + assert((size >= TreeChunk::min_size()), + "Chunk is too small for a TreeChunk"); + // The space will have been mangled initially but + // is not remangled when a Chunk_t is returned to the free list + // (since it is used to maintain the chunk on the free list). + tc->assert_is_mangled(); + tc->set_size(size); + tc->link_prev(NULL); + tc->link_next(NULL); + TreeList* tl = TreeList::as_TreeList(tc); return tl; } -template -TreeList* TreeList::as_TreeList(HeapWord* addr, size_t size) { - TreeChunk* tc = (TreeChunk*) addr; - assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "Chunk is too small for a TreeChunk"); - // The space in the heap will have been mangled initially but - // is not remangled when a free chunk is returned to the free list - // (since it is used to maintain the chunk on the free list). - assert((ZapUnusedHeapArea && - SpaceMangler::is_mangled((HeapWord*) tc->size_addr()) && - SpaceMangler::is_mangled((HeapWord*) tc->prev_addr()) && - SpaceMangler::is_mangled((HeapWord*) tc->next_addr())) || - (tc->size() == 0 && tc->prev() == NULL && tc->next() == NULL), - "Space should be clear or mangled"); - tc->set_size(size); - tc->link_prev(NULL); - tc->link_next(NULL); - TreeList* tl = TreeList::as_TreeList(tc); - return tl; + +#ifndef SERIALGC +// Specialize for AdaptiveFreeList which tries to avoid +// splitting a chunk of a size that is under populated in favor of +// an over populated size. The general get_better_list() just returns +// the current list. +template <> +TreeList* +TreeList::get_better_list( + BinaryTreeDictionary* dictionary) { + // A candidate chunk has been found. If it is already under + // populated, get a chunk associated with the hint for this + // chunk. + + TreeList* curTL = this; + if (surplus() <= 0) { + /* Use the hint to find a size with a surplus, and reset the hint. */ + TreeList* hintTL = this; + while (hintTL->hint() != 0) { + assert(hintTL->hint() > hintTL->size(), + "hint points in the wrong direction"); + hintTL = dictionary->find_list(hintTL->hint()); + assert(curTL != hintTL, "Infinite loop"); + if (hintTL == NULL || + hintTL == curTL /* Should not happen but protect against it */ ) { + // No useful hint. Set the hint to NULL and go on. + curTL->set_hint(0); + break; + } + assert(hintTL->size() > curTL->size(), "hint is inconsistent"); + if (hintTL->surplus() > 0) { + // The hint led to a list that has a surplus. Use it. + // Set the hint for the candidate to an overpopulated + // size. + curTL->set_hint(hintTL->size()); + // Change the candidate. + curTL = hintTL; + break; + } + } + } + return curTL; +} +#endif // SERIALGC + +template class FreeList_t> +TreeList* +TreeList::get_better_list( + BinaryTreeDictionary* dictionary) { + return this; } -template -TreeList* TreeList::remove_chunk_replace_if_needed(TreeChunk* tc) { +template class FreeList_t> +TreeList* TreeList::remove_chunk_replace_if_needed(TreeChunk* tc) { - TreeList* retTL = this; - Chunk* list = head(); + TreeList* retTL = this; + Chunk_t* list = head(); assert(!list || list != list->next(), "Chunk on list twice"); assert(tc != NULL, "Chunk being removed is NULL"); assert(parent() == NULL || this == parent()->left() || @@ -112,13 +181,13 @@ assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); - Chunk* prevFC = tc->prev(); - TreeChunk* nextTC = TreeChunk::as_TreeChunk(tc->next()); + Chunk_t* prevFC = tc->prev(); + TreeChunk* nextTC = TreeChunk::as_TreeChunk(tc->next()); assert(list != NULL, "should have at least the target chunk"); // Is this the first item on the list? if (tc == list) { - // The "getChunk..." functions for a TreeList will not return the + // The "getChunk..." functions for a TreeList will not return the // first chunk in the list unless it is the last chunk in the list // because the first chunk is also acting as the tree node. // When coalescing happens, however, the first chunk in the a tree @@ -127,8 +196,8 @@ // allocated when the sweeper yields (giving up the free list lock) // to allow mutator activity. If this chunk is the first in the // list and is not the last in the list, do the work to copy the - // TreeList from the first chunk to the next chunk and update all - // the TreeList pointers in the chunks in the list. + // TreeList from the first chunk to the next chunk and update all + // the TreeList pointers in the chunks in the list. if (nextTC == NULL) { assert(prevFC == NULL, "Not last chunk in the list"); set_tail(NULL); @@ -141,11 +210,11 @@ // This can be slow for a long list. Consider having // an option that does not allow the first chunk on the // list to be coalesced. - for (TreeChunk* curTC = nextTC; curTC != NULL; - curTC = TreeChunk::as_TreeChunk(curTC->next())) { + for (TreeChunk* curTC = nextTC; curTC != NULL; + curTC = TreeChunk::as_TreeChunk(curTC->next())) { curTC->set_list(retTL); } - // Fix the parent to point to the new TreeList. + // Fix the parent to point to the new TreeList. if (retTL->parent() != NULL) { if (this == retTL->parent()->left()) { retTL->parent()->set_left(retTL); @@ -176,9 +245,9 @@ prevFC->link_after(nextTC); } - // Below this point the embeded TreeList being used for the + // Below this point the embeded TreeList being used for the // tree node may have changed. Don't use "this" - // TreeList*. + // TreeList*. // chunk should still be a free chunk (bit set in _prev) assert(!retTL->head() || retTL->size() == retTL->head()->size(), "Wrong sized chunk in list"); @@ -188,7 +257,7 @@ tc->set_list(NULL); bool prev_found = false; bool next_found = false; - for (Chunk* curFC = retTL->head(); + for (Chunk_t* curFC = retTL->head(); curFC != NULL; curFC = curFC->next()) { assert(curFC != tc, "Chunk is still in list"); if (curFC == prevFC) { @@ -215,8 +284,8 @@ return retTL; } -template -void TreeList::return_chunk_at_tail(TreeChunk* chunk) { +template class FreeList_t> +void TreeList::return_chunk_at_tail(TreeChunk* chunk) { assert(chunk != NULL, "returning NULL chunk"); assert(chunk->list() == this, "list should be set for chunk"); assert(tail() != NULL, "The tree list is embedded in the first chunk"); @@ -225,12 +294,12 @@ assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); - Chunk* fc = tail(); + Chunk_t* fc = tail(); fc->link_after(chunk); link_tail(chunk); assert(!tail() || size() == tail()->size(), "Wrong sized chunk in list"); - increment_count(); + FreeList_t::increment_count(); debug_only(increment_returned_bytes_by(chunk->size()*sizeof(HeapWord));) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); @@ -238,10 +307,10 @@ // Add this chunk at the head of the list. "At the head of the list" // is defined to be after the chunk pointer to by head(). This is -// because the TreeList is embedded in the first TreeChunk in the -// list. See the definition of TreeChunk. -template -void TreeList::return_chunk_at_head(TreeChunk* chunk) { +// because the TreeList is embedded in the first TreeChunk in the +// list. See the definition of TreeChunk. +template class FreeList_t> +void TreeList::return_chunk_at_head(TreeChunk* chunk) { assert(chunk->list() == this, "list should be set for chunk"); assert(head() != NULL, "The tree list is embedded in the first chunk"); assert(chunk != NULL, "returning NULL chunk"); @@ -249,7 +318,7 @@ assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); - Chunk* fc = head()->next(); + Chunk_t* fc = head()->next(); if (fc != NULL) { chunk->link_after(fc); } else { @@ -258,28 +327,38 @@ } head()->link_after(chunk); assert(!head() || size() == head()->size(), "Wrong sized chunk in list"); - increment_count(); + FreeList_t::increment_count(); debug_only(increment_returned_bytes_by(chunk->size()*sizeof(HeapWord));) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); } -template -TreeChunk* TreeList::head_as_TreeChunk() { - assert(head() == NULL || TreeChunk::as_TreeChunk(head())->list() == this, - "Wrong type of chunk?"); - return TreeChunk::as_TreeChunk(head()); +template class FreeList_t> +void TreeChunk::assert_is_mangled() const { + assert((ZapUnusedHeapArea && + SpaceMangler::is_mangled((HeapWord*) Chunk_t::size_addr()) && + SpaceMangler::is_mangled((HeapWord*) Chunk_t::prev_addr()) && + SpaceMangler::is_mangled((HeapWord*) Chunk_t::next_addr())) || + (size() == 0 && prev() == NULL && next() == NULL), + "Space should be clear or mangled"); } -template -TreeChunk* TreeList::first_available() { +template class FreeList_t> +TreeChunk* TreeList::head_as_TreeChunk() { + assert(head() == NULL || (TreeChunk::as_TreeChunk(head())->list() == this), + "Wrong type of chunk?"); + return TreeChunk::as_TreeChunk(head()); +} + +template class FreeList_t> +TreeChunk* TreeList::first_available() { assert(head() != NULL, "The head of the list cannot be NULL"); - Chunk* fc = head()->next(); - TreeChunk* retTC; + Chunk_t* fc = head()->next(); + TreeChunk* retTC; if (fc == NULL) { retTC = head_as_TreeChunk(); } else { - retTC = TreeChunk::as_TreeChunk(fc); + retTC = TreeChunk::as_TreeChunk(fc); } assert(retTC->list() == this, "Wrong type of chunk."); return retTC; @@ -288,41 +367,32 @@ // Returns the block with the largest heap address amongst // those in the list for this size; potentially slow and expensive, // use with caution! -template -TreeChunk* TreeList::largest_address() { +template class FreeList_t> +TreeChunk* TreeList::largest_address() { assert(head() != NULL, "The head of the list cannot be NULL"); - Chunk* fc = head()->next(); - TreeChunk* retTC; + Chunk_t* fc = head()->next(); + TreeChunk* retTC; if (fc == NULL) { retTC = head_as_TreeChunk(); } else { // walk down the list and return the one with the highest // heap address among chunks of this size. - Chunk* last = fc; + Chunk_t* last = fc; while (fc->next() != NULL) { if ((HeapWord*)last < (HeapWord*)fc) { last = fc; } fc = fc->next(); } - retTC = TreeChunk::as_TreeChunk(last); + retTC = TreeChunk::as_TreeChunk(last); } assert(retTC->list() == this, "Wrong type of chunk."); return retTC; } -template -BinaryTreeDictionary::BinaryTreeDictionary(bool adaptive_freelists, bool splay) : - _splay(splay), _adaptive_freelists(adaptive_freelists), - _total_size(0), _total_free_blocks(0), _root(0) {} - -template -BinaryTreeDictionary::BinaryTreeDictionary(MemRegion mr, - bool adaptive_freelists, - bool splay): - _adaptive_freelists(adaptive_freelists), _splay(splay) -{ - assert(mr.word_size() >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); +template class FreeList_t> +BinaryTreeDictionary::BinaryTreeDictionary(MemRegion mr) { + assert((mr.byte_size() > min_size()), "minimum chunk size"); reset(mr); assert(root()->left() == NULL, "reset check failed"); @@ -333,52 +403,48 @@ assert(total_free_blocks() == 1, "reset check failed"); } -template -void BinaryTreeDictionary::inc_total_size(size_t inc) { +template class FreeList_t> +void BinaryTreeDictionary::inc_total_size(size_t inc) { _total_size = _total_size + inc; } -template -void BinaryTreeDictionary::dec_total_size(size_t dec) { +template class FreeList_t> +void BinaryTreeDictionary::dec_total_size(size_t dec) { _total_size = _total_size - dec; } -template -void BinaryTreeDictionary::reset(MemRegion mr) { - assert(mr.word_size() >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); - set_root(TreeList::as_TreeList(mr.start(), mr.word_size())); +template class FreeList_t> +void BinaryTreeDictionary::reset(MemRegion mr) { + assert((mr.byte_size() > min_size()), "minimum chunk size"); + set_root(TreeList::as_TreeList(mr.start(), mr.word_size())); set_total_size(mr.word_size()); set_total_free_blocks(1); } -template -void BinaryTreeDictionary::reset(HeapWord* addr, size_t byte_size) { +template class FreeList_t> +void BinaryTreeDictionary::reset(HeapWord* addr, size_t byte_size) { MemRegion mr(addr, heap_word_size(byte_size)); reset(mr); } -template -void BinaryTreeDictionary::reset() { +template class FreeList_t> +void BinaryTreeDictionary::reset() { set_root(NULL); set_total_size(0); set_total_free_blocks(0); } // Get a free block of size at least size from tree, or NULL. -// If a splay step is requested, the removal algorithm (only) incorporates -// a splay step as follows: -// . the search proceeds down the tree looking for a possible -// match. At the (closest) matching location, an appropriate splay step is applied -// (zig, zig-zig or zig-zag). A chunk of the appropriate size is then returned -// if available, and if it's the last chunk, the node is deleted. A deteleted -// node is replaced in place by its tree successor. -template -TreeChunk* -BinaryTreeDictionary::get_chunk_from_tree(size_t size, enum FreeBlockDictionary::Dither dither, bool splay) +template class FreeList_t> +TreeChunk* +BinaryTreeDictionary::get_chunk_from_tree( + size_t size, + enum FreeBlockDictionary::Dither dither) { - TreeList *curTL, *prevTL; - TreeChunk* retTC = NULL; - assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); + TreeList *curTL, *prevTL; + TreeChunk* retTC = NULL; + + assert((size >= min_size()), "minimum chunk size"); if (FLSVerifyDictionary) { verify_tree(); } @@ -398,7 +464,7 @@ } if (curTL == NULL) { // couldn't find exact match - if (dither == FreeBlockDictionary::exactly) return NULL; + if (dither == FreeBlockDictionary::exactly) return NULL; // try and find the next larger size by walking back up the search path for (curTL = prevTL; curTL != NULL;) { @@ -410,46 +476,9 @@ } if (curTL != NULL) { assert(curTL->size() >= size, "size inconsistency"); - if (adaptive_freelists()) { - // A candidate chunk has been found. If it is already under - // populated, get a chunk associated with the hint for this - // chunk. - if (curTL->surplus() <= 0) { - /* Use the hint to find a size with a surplus, and reset the hint. */ - TreeList* hintTL = curTL; - while (hintTL->hint() != 0) { - assert(hintTL->hint() == 0 || hintTL->hint() > hintTL->size(), - "hint points in the wrong direction"); - hintTL = find_list(hintTL->hint()); - assert(curTL != hintTL, "Infinite loop"); - if (hintTL == NULL || - hintTL == curTL /* Should not happen but protect against it */ ) { - // No useful hint. Set the hint to NULL and go on. - curTL->set_hint(0); - break; - } - assert(hintTL->size() > size, "hint is inconsistent"); - if (hintTL->surplus() > 0) { - // The hint led to a list that has a surplus. Use it. - // Set the hint for the candidate to an overpopulated - // size. - curTL->set_hint(hintTL->size()); - // Change the candidate. - curTL = hintTL; - break; - } - // The evm code reset the hint of the candidate as - // at an interim point. Why? Seems like this leaves - // the hint pointing to a list that didn't work. - // curTL->set_hint(hintTL->size()); - } - } - } - // don't waste time splaying if chunk's singleton - if (splay && curTL->head()->next() != NULL) { - semi_splay_step(curTL); - } + curTL = curTL->get_better_list(this); + retTC = curTL->first_available(); assert((retTC != NULL) && (curTL->count() > 0), "A list in the binary tree should not be NULL"); @@ -465,9 +494,9 @@ return retTC; } -template -TreeList* BinaryTreeDictionary::find_list(size_t size) const { - TreeList* curTL; +template class FreeList_t> +TreeList* BinaryTreeDictionary::find_list(size_t size) const { + TreeList* curTL; for (curTL = root(); curTL != NULL;) { if (curTL->size() == size) { // exact match break; @@ -484,10 +513,10 @@ } -template -bool BinaryTreeDictionary::verify_chunk_in_free_list(Chunk* tc) const { +template class FreeList_t> +bool BinaryTreeDictionary::verify_chunk_in_free_list(Chunk_t* tc) const { size_t size = tc->size(); - TreeList* tl = find_list(size); + TreeList* tl = find_list(size); if (tl == NULL) { return false; } else { @@ -495,9 +524,9 @@ } } -template -Chunk* BinaryTreeDictionary::find_largest_dict() const { - TreeList *curTL = root(); +template class FreeList_t> +Chunk_t* BinaryTreeDictionary::find_largest_dict() const { + TreeList *curTL = root(); if (curTL != NULL) { while(curTL->right() != NULL) curTL = curTL->right(); return curTL->largest_address(); @@ -510,15 +539,15 @@ // chunk in a list on a tree node, just unlink it. // If it is the last chunk in the list (the next link is NULL), // remove the node and repair the tree. -template -TreeChunk* -BinaryTreeDictionary::remove_chunk_from_tree(TreeChunk* tc) { +template class FreeList_t> +TreeChunk* +BinaryTreeDictionary::remove_chunk_from_tree(TreeChunk* tc) { assert(tc != NULL, "Should not call with a NULL chunk"); assert(tc->is_free(), "Header is not marked correctly"); - TreeList *newTL, *parentTL; - TreeChunk* retTC; - TreeList* tl = tc->list(); + TreeList *newTL, *parentTL; + TreeChunk* retTC; + TreeList* tl = tc->list(); debug_only( bool removing_only_chunk = false; if (tl == _root) { @@ -538,8 +567,8 @@ retTC = tc; // Removing this chunk can have the side effect of changing the node - // (TreeList*) in the tree. If the node is the root, update it. - TreeList* replacementTL = tl->remove_chunk_replace_if_needed(tc); + // (TreeList*) in the tree. If the node is the root, update it. + TreeList* replacementTL = tl->remove_chunk_replace_if_needed(tc); assert(tc->is_free(), "Chunk should still be free"); assert(replacementTL->parent() == NULL || replacementTL == replacementTL->parent()->left() || @@ -549,17 +578,18 @@ assert(replacementTL->parent() == NULL, "Incorrectly replacing root"); set_root(replacementTL); } - debug_only( +#ifdef ASSERT if (tl != replacementTL) { assert(replacementTL->head() != NULL, "If the tree list was replaced, it should not be a NULL list"); - TreeList* rhl = replacementTL->head_as_TreeChunk()->list(); - TreeList* rtl = TreeChunk::as_TreeChunk(replacementTL->tail())->list(); + TreeList* rhl = replacementTL->head_as_TreeChunk()->list(); + TreeList* rtl = + TreeChunk::as_TreeChunk(replacementTL->tail())->list(); assert(rhl == replacementTL, "Broken head"); assert(rtl == replacementTL, "Broken tail"); assert(replacementTL->size() == tc->size(), "Broken size"); } - ) +#endif // Does the tree need to be repaired? if (replacementTL->count() == 0) { @@ -574,7 +604,7 @@ } else if (replacementTL->right() == NULL) { // right is NULL newTL = replacementTL->left(); - debug_only(replacementTL->clearLeft();) + debug_only(replacementTL->clear_left();) } else { // we have both children, so, by patriarchal convention, // my replacement is least node in right sub-tree complicated_splice = true; @@ -623,7 +653,7 @@ newTL->set_right(replacementTL->right()); debug_only( replacementTL->clear_right(); - replacementTL->clearLeft(); + replacementTL->clear_left(); ) } assert(replacementTL->right() == NULL && @@ -644,21 +674,21 @@ verify_tree(); } assert(!removing_only_chunk || _root == NULL, "root should be NULL"); - return TreeChunk::as_TreeChunk(retTC); + return TreeChunk::as_TreeChunk(retTC); } // Remove the leftmost node (lm) in the tree and return it. // If lm has a right child, link it to the left node of // the parent of lm. -template -TreeList* BinaryTreeDictionary::remove_tree_minimum(TreeList* tl) { +template class FreeList_t> +TreeList* BinaryTreeDictionary::remove_tree_minimum(TreeList* tl) { assert(tl != NULL && tl->parent() != NULL, "really need a proper sub-tree"); // locate the subtree minimum by walking down left branches - TreeList* curTL = tl; + TreeList* curTL = tl; for (; curTL->left() != NULL; curTL = curTL->left()); // obviously curTL now has at most one child, a right child if (curTL != root()) { // Should this test just be removed? - TreeList* parentTL = curTL->parent(); + TreeList* parentTL = curTL->parent(); if (parentTL->left() == curTL) { // curTL is a left child parentTL->set_left(curTL->right()); } else { @@ -685,31 +715,14 @@ return curTL; } -// Based on a simplification of the algorithm by Sleator and Tarjan (JACM 1985). -// The simplifications are the following: -// . we splay only when we delete (not when we insert) -// . we apply a single spay step per deletion/access -// By doing such partial splaying, we reduce the amount of restructuring, -// while getting a reasonably efficient search tree (we think). -// [Measurements will be needed to (in)validate this expectation.] - -template -void BinaryTreeDictionary::semi_splay_step(TreeList* tc) { - // apply a semi-splay step at the given node: - // . if root, norting needs to be done - // . if child of root, splay once - // . else zig-zig or sig-zag depending on path from grandparent - if (root() == tc) return; - warning("*** Splaying not yet implemented; " - "tree operations may be inefficient ***"); -} - -template -void BinaryTreeDictionary::insert_chunk_in_tree(Chunk* fc) { - TreeList *curTL, *prevTL; +template class FreeList_t> +void BinaryTreeDictionary::insert_chunk_in_tree(Chunk_t* fc) { + TreeList *curTL, *prevTL; size_t size = fc->size(); - assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "too small to be a TreeList"); + assert((size >= min_size()), + err_msg(SIZE_FORMAT " is too small to be a TreeChunk " SIZE_FORMAT, + size, min_size())); if (FLSVerifyDictionary) { verify_tree(); } @@ -729,9 +742,9 @@ curTL = curTL->right(); } } - TreeChunk* tc = TreeChunk::as_TreeChunk(fc); + TreeChunk* tc = TreeChunk::as_TreeChunk(fc); // This chunk is being returned to the binary tree. Its embedded - // TreeList should be unused at this point. + // TreeList should be unused at this point. tc->initialize(); if (curTL != NULL) { // exact match tc->set_list(curTL); @@ -739,8 +752,8 @@ } else { // need a new node in tree tc->clear_next(); tc->link_prev(NULL); - TreeList* newTL = TreeList::as_TreeList(tc); - assert(((TreeChunk*)tc)->list() == newTL, + TreeList* newTL = TreeList::as_TreeList(tc); + assert(((TreeChunk*)tc)->list() == newTL, "List was not initialized correctly"); if (prevTL == NULL) { // we are the only tree node assert(root() == NULL, "control point invariant"); @@ -768,30 +781,30 @@ } } -template -size_t BinaryTreeDictionary::max_chunk_size() const { - FreeBlockDictionary::verify_par_locked(); - TreeList* tc = root(); +template class FreeList_t> +size_t BinaryTreeDictionary::max_chunk_size() const { + FreeBlockDictionary::verify_par_locked(); + TreeList* tc = root(); if (tc == NULL) return 0; for (; tc->right() != NULL; tc = tc->right()); return tc->size(); } -template -size_t BinaryTreeDictionary::total_list_length(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::total_list_length(TreeList* tl) const { size_t res; res = tl->count(); #ifdef ASSERT size_t cnt; - Chunk* tc = tl->head(); + Chunk_t* tc = tl->head(); for (cnt = 0; tc != NULL; tc = tc->next(), cnt++); assert(res == cnt, "The count is not being maintained correctly"); #endif return res; } -template -size_t BinaryTreeDictionary::total_size_in_tree(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::total_size_in_tree(TreeList* tl) const { if (tl == NULL) return 0; return (tl->size() * total_list_length(tl)) + @@ -799,8 +812,8 @@ total_size_in_tree(tl->right()); } -template -double BinaryTreeDictionary::sum_of_squared_block_sizes(TreeList* const tl) const { +template class FreeList_t> +double BinaryTreeDictionary::sum_of_squared_block_sizes(TreeList* const tl) const { if (tl == NULL) { return 0.0; } @@ -811,8 +824,8 @@ return curr; } -template -size_t BinaryTreeDictionary::total_free_blocks_in_tree(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::total_free_blocks_in_tree(TreeList* tl) const { if (tl == NULL) return 0; return total_list_length(tl) + @@ -820,28 +833,28 @@ total_free_blocks_in_tree(tl->right()); } -template -size_t BinaryTreeDictionary::num_free_blocks() const { +template class FreeList_t> +size_t BinaryTreeDictionary::num_free_blocks() const { assert(total_free_blocks_in_tree(root()) == total_free_blocks(), "_total_free_blocks inconsistency"); return total_free_blocks(); } -template -size_t BinaryTreeDictionary::tree_height_helper(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::tree_height_helper(TreeList* tl) const { if (tl == NULL) return 0; return 1 + MAX2(tree_height_helper(tl->left()), tree_height_helper(tl->right())); } -template -size_t BinaryTreeDictionary::treeHeight() const { +template class FreeList_t> +size_t BinaryTreeDictionary::tree_height() const { return tree_height_helper(root()); } -template -size_t BinaryTreeDictionary::total_nodes_helper(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::total_nodes_helper(TreeList* tl) const { if (tl == NULL) { return 0; } @@ -849,14 +862,18 @@ total_nodes_helper(tl->right()); } -template -size_t BinaryTreeDictionary::total_nodes_in_tree(TreeList* tl) const { +template class FreeList_t> +size_t BinaryTreeDictionary::total_nodes_in_tree(TreeList* tl) const { return total_nodes_helper(root()); } -template -void BinaryTreeDictionary::dict_census_udpate(size_t size, bool split, bool birth){ - TreeList* nd = find_list(size); +template class FreeList_t> +void BinaryTreeDictionary::dict_census_update(size_t size, bool split, bool birth){} + +#ifndef SERIALGC +template <> +void BinaryTreeDictionary::dict_census_update(size_t size, bool split, bool birth){ + TreeList* nd = find_list(size); if (nd) { if (split) { if (birth) { @@ -882,16 +899,26 @@ // This is a birth associated with a LinAB. The chunk // for the LinAB is not in the dictionary. } +#endif // SERIALGC -template -bool BinaryTreeDictionary::coal_dict_over_populated(size_t size) { +template class FreeList_t> +bool BinaryTreeDictionary::coal_dict_over_populated(size_t size) { + // For the general type of freelists, encourage coalescing by + // returning true. + return true; +} + +#ifndef SERIALGC +template <> +bool BinaryTreeDictionary::coal_dict_over_populated(size_t size) { if (FLSAlwaysCoalesceLarge) return true; - TreeList* list_of_size = find_list(size); + TreeList* list_of_size = find_list(size); // None of requested size implies overpopulated. return list_of_size == NULL || list_of_size->coal_desired() <= 0 || list_of_size->count() > list_of_size->coal_desired(); } +#endif // SERIALGC // Closures for walking the binary tree. // do_list() walks the free list in a node applying the closure @@ -899,19 +926,18 @@ // do_tree() walks the nodes in the binary tree applying do_list() // to each list at each node. -template +template class FreeList_t> class TreeCensusClosure : public StackObj { protected: - virtual void do_list(FreeList* fl) = 0; + virtual void do_list(FreeList_t* fl) = 0; public: - virtual void do_tree(TreeList* tl) = 0; + virtual void do_tree(TreeList* tl) = 0; }; -template -class AscendTreeCensusClosure : public TreeCensusClosure { - using TreeCensusClosure::do_list; +template class FreeList_t> +class AscendTreeCensusClosure : public TreeCensusClosure { public: - void do_tree(TreeList* tl) { + void do_tree(TreeList* tl) { if (tl != NULL) { do_tree(tl->left()); do_list(tl); @@ -920,11 +946,10 @@ } }; -template -class DescendTreeCensusClosure : public TreeCensusClosure { - using TreeCensusClosure::do_list; +template class FreeList_t> +class DescendTreeCensusClosure : public TreeCensusClosure { public: - void do_tree(TreeList* tl) { + void do_tree(TreeList* tl) { if (tl != NULL) { do_tree(tl->right()); do_list(tl); @@ -935,8 +960,8 @@ // For each list in the tree, calculate the desired, desired // coalesce, count before sweep, and surplus before sweep. -template -class BeginSweepClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class BeginSweepClosure : public AscendTreeCensusClosure { double _percentage; float _inter_sweep_current; float _inter_sweep_estimate; @@ -951,32 +976,36 @@ _inter_sweep_estimate(inter_sweep_estimate), _intra_sweep_estimate(intra_sweep_estimate) { } - void do_list(FreeList* fl) { + void do_list(FreeList* fl) {} + +#ifndef SERIALGC + void do_list(AdaptiveFreeList* fl) { double coalSurplusPercent = _percentage; fl->compute_desired(_inter_sweep_current, _inter_sweep_estimate, _intra_sweep_estimate); fl->set_coal_desired((ssize_t)((double)fl->desired() * coalSurplusPercent)); fl->set_before_sweep(fl->count()); fl->set_bfr_surp(fl->surplus()); } +#endif // SERIALGC }; // Used to search the tree until a condition is met. // Similar to TreeCensusClosure but searches the // tree and returns promptly when found. -template +template class FreeList_t> class TreeSearchClosure : public StackObj { protected: - virtual bool do_list(FreeList* fl) = 0; + virtual bool do_list(FreeList_t* fl) = 0; public: - virtual bool do_tree(TreeList* tl) = 0; + virtual bool do_tree(TreeList* tl) = 0; }; #if 0 // Don't need this yet but here for symmetry. -template -class AscendTreeSearchClosure : public TreeSearchClosure { +template class FreeList_t> +class AscendTreeSearchClosure : public TreeSearchClosure { public: - bool do_tree(TreeList* tl) { + bool do_tree(TreeList* tl) { if (tl != NULL) { if (do_tree(tl->left())) return true; if (do_list(tl)) return true; @@ -987,11 +1016,10 @@ }; #endif -template -class DescendTreeSearchClosure : public TreeSearchClosure { - using TreeSearchClosure::do_list; +template class FreeList_t> +class DescendTreeSearchClosure : public TreeSearchClosure { public: - bool do_tree(TreeList* tl) { + bool do_tree(TreeList* tl) { if (tl != NULL) { if (do_tree(tl->right())) return true; if (do_list(tl)) return true; @@ -1003,17 +1031,17 @@ // Searches the tree for a chunk that ends at the // specified address. -template -class EndTreeSearchClosure : public DescendTreeSearchClosure { +template class FreeList_t> +class EndTreeSearchClosure : public DescendTreeSearchClosure { HeapWord* _target; - Chunk* _found; + Chunk_t* _found; public: EndTreeSearchClosure(HeapWord* target) : _target(target), _found(NULL) {} - bool do_list(FreeList* fl) { - Chunk* item = fl->head(); + bool do_list(FreeList_t* fl) { + Chunk_t* item = fl->head(); while (item != NULL) { - if (item->end() == _target) { + if (item->end() == (uintptr_t*) _target) { _found = item; return true; } @@ -1021,22 +1049,22 @@ } return false; } - Chunk* found() { return _found; } + Chunk_t* found() { return _found; } }; -template -Chunk* BinaryTreeDictionary::find_chunk_ends_at(HeapWord* target) const { - EndTreeSearchClosure etsc(target); +template class FreeList_t> +Chunk_t* BinaryTreeDictionary::find_chunk_ends_at(HeapWord* target) const { + EndTreeSearchClosure etsc(target); bool found_target = etsc.do_tree(root()); assert(found_target || etsc.found() == NULL, "Consistency check"); assert(!found_target || etsc.found() != NULL, "Consistency check"); return etsc.found(); } -template -void BinaryTreeDictionary::begin_sweep_dict_census(double coalSurplusPercent, +template class FreeList_t> +void BinaryTreeDictionary::begin_sweep_dict_census(double coalSurplusPercent, float inter_sweep_current, float inter_sweep_estimate, float intra_sweep_estimate) { - BeginSweepClosure bsc(coalSurplusPercent, inter_sweep_current, + BeginSweepClosure bsc(coalSurplusPercent, inter_sweep_current, inter_sweep_estimate, intra_sweep_estimate); bsc.do_tree(root()); @@ -1045,84 +1073,91 @@ // Closures and methods for calculating total bytes returned to the // free lists in the tree. #ifndef PRODUCT -template -class InitializeDictReturnedBytesClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class InitializeDictReturnedBytesClosure : public AscendTreeCensusClosure { public: - void do_list(FreeList* fl) { + void do_list(FreeList_t* fl) { fl->set_returned_bytes(0); } }; -template -void BinaryTreeDictionary::initialize_dict_returned_bytes() { - InitializeDictReturnedBytesClosure idrb; +template class FreeList_t> +void BinaryTreeDictionary::initialize_dict_returned_bytes() { + InitializeDictReturnedBytesClosure idrb; idrb.do_tree(root()); } -template -class ReturnedBytesClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class ReturnedBytesClosure : public AscendTreeCensusClosure { size_t _dict_returned_bytes; public: ReturnedBytesClosure() { _dict_returned_bytes = 0; } - void do_list(FreeList* fl) { + void do_list(FreeList_t* fl) { _dict_returned_bytes += fl->returned_bytes(); } size_t dict_returned_bytes() { return _dict_returned_bytes; } }; -template -size_t BinaryTreeDictionary::sum_dict_returned_bytes() { - ReturnedBytesClosure rbc; +template class FreeList_t> +size_t BinaryTreeDictionary::sum_dict_returned_bytes() { + ReturnedBytesClosure rbc; rbc.do_tree(root()); return rbc.dict_returned_bytes(); } // Count the number of entries in the tree. -template -class treeCountClosure : public DescendTreeCensusClosure { +template class FreeList_t> +class treeCountClosure : public DescendTreeCensusClosure { public: uint count; treeCountClosure(uint c) { count = c; } - void do_list(FreeList* fl) { + void do_list(FreeList_t* fl) { count++; } }; -template -size_t BinaryTreeDictionary::total_count() { - treeCountClosure ctc(0); +template class FreeList_t> +size_t BinaryTreeDictionary::total_count() { + treeCountClosure ctc(0); ctc.do_tree(root()); return ctc.count; } #endif // PRODUCT // Calculate surpluses for the lists in the tree. -template -class setTreeSurplusClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class setTreeSurplusClosure : public AscendTreeCensusClosure { double percentage; public: setTreeSurplusClosure(double v) { percentage = v; } - void do_list(FreeList* fl) { + void do_list(FreeList* fl) {} + +#ifndef SERIALGC + void do_list(AdaptiveFreeList* fl) { double splitSurplusPercent = percentage; fl->set_surplus(fl->count() - (ssize_t)((double)fl->desired() * splitSurplusPercent)); } +#endif // SERIALGC }; -template -void BinaryTreeDictionary::set_tree_surplus(double splitSurplusPercent) { - setTreeSurplusClosure sts(splitSurplusPercent); +template class FreeList_t> +void BinaryTreeDictionary::set_tree_surplus(double splitSurplusPercent) { + setTreeSurplusClosure sts(splitSurplusPercent); sts.do_tree(root()); } // Set hints for the lists in the tree. -template -class setTreeHintsClosure : public DescendTreeCensusClosure { +template class FreeList_t> +class setTreeHintsClosure : public DescendTreeCensusClosure { size_t hint; public: setTreeHintsClosure(size_t v) { hint = v; } - void do_list(FreeList* fl) { + void do_list(FreeList* fl) {} + +#ifndef SERIALGC + void do_list(AdaptiveFreeList* fl) { fl->set_hint(hint); assert(fl->hint() == 0 || fl->hint() > fl->size(), "Current hint is inconsistent"); @@ -1130,35 +1165,40 @@ hint = fl->size(); } } +#endif // SERIALGC }; -template -void BinaryTreeDictionary::set_tree_hints(void) { - setTreeHintsClosure sth(0); +template class FreeList_t> +void BinaryTreeDictionary::set_tree_hints(void) { + setTreeHintsClosure sth(0); sth.do_tree(root()); } // Save count before previous sweep and splits and coalesces. -template -class clearTreeCensusClosure : public AscendTreeCensusClosure { - void do_list(FreeList* fl) { +template class FreeList_t> +class clearTreeCensusClosure : public AscendTreeCensusClosure { + void do_list(FreeList* fl) {} + +#ifndef SERIALGC + void do_list(AdaptiveFreeList* fl) { fl->set_prev_sweep(fl->count()); fl->set_coal_births(0); fl->set_coal_deaths(0); fl->set_split_births(0); fl->set_split_deaths(0); } +#endif // SERIALGC }; -template -void BinaryTreeDictionary::clear_tree_census(void) { - clearTreeCensusClosure ctc; +template class FreeList_t> +void BinaryTreeDictionary::clear_tree_census(void) { + clearTreeCensusClosure ctc; ctc.do_tree(root()); } // Do reporting and post sweep clean up. -template -void BinaryTreeDictionary::end_sweep_dict_census(double splitSurplusPercent) { +template class FreeList_t> +void BinaryTreeDictionary::end_sweep_dict_census(double splitSurplusPercent) { // Does walking the tree 3 times hurt? set_tree_surplus(splitSurplusPercent); set_tree_hints(); @@ -1169,9 +1209,9 @@ } // Print summary statistics -template -void BinaryTreeDictionary::report_statistics() const { - FreeBlockDictionary::verify_par_locked(); +template class FreeList_t> +void BinaryTreeDictionary::report_statistics() const { + FreeBlockDictionary::verify_par_locked(); gclog_or_tty->print("Statistics for BinaryTreeDictionary:\n" "------------------------------------\n"); size_t total_size = total_chunk_size(debug_only(NULL)); @@ -1182,36 +1222,47 @@ if (free_blocks > 0) { gclog_or_tty->print("Av. Block Size: %d\n", total_size/free_blocks); } - gclog_or_tty->print("Tree Height: %d\n", treeHeight()); + gclog_or_tty->print("Tree Height: %d\n", tree_height()); } // Print census information - counts, births, deaths, etc. // for each list in the tree. Also print some summary // information. -template -class PrintTreeCensusClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class PrintTreeCensusClosure : public AscendTreeCensusClosure { int _print_line; size_t _total_free; - FreeList _total; + FreeList_t _total; public: PrintTreeCensusClosure() { _print_line = 0; _total_free = 0; } - FreeList* total() { return &_total; } + FreeList_t* total() { return &_total; } size_t total_free() { return _total_free; } - void do_list(FreeList* fl) { + void do_list(FreeList* fl) { if (++_print_line >= 40) { - FreeList::print_labels_on(gclog_or_tty, "size"); + FreeList_t::print_labels_on(gclog_or_tty, "size"); _print_line = 0; } fl->print_on(gclog_or_tty); _total_free += fl->count() * fl->size() ; total()->set_count( total()->count() + fl->count() ); - total()->set_bfr_surp( total()->bfr_surp() + fl->bfr_surp() ); + } + +#ifndef SERIALGC + void do_list(AdaptiveFreeList* fl) { + if (++_print_line >= 40) { + FreeList_t::print_labels_on(gclog_or_tty, "size"); + _print_line = 0; + } + fl->print_on(gclog_or_tty); + _total_free += fl->count() * fl->size() ; + total()->set_count( total()->count() + fl->count() ); + total()->set_bfr_surp( total()->bfr_surp() + fl->bfr_surp() ); total()->set_surplus( total()->split_deaths() + fl->surplus() ); - total()->set_desired( total()->desired() + fl->desired() ); + total()->set_desired( total()->desired() + fl->desired() ); total()->set_prev_sweep( total()->prev_sweep() + fl->prev_sweep() ); total()->set_before_sweep(total()->before_sweep() + fl->before_sweep()); total()->set_coal_births( total()->coal_births() + fl->coal_births() ); @@ -1219,18 +1270,32 @@ total()->set_split_births(total()->split_births() + fl->split_births()); total()->set_split_deaths(total()->split_deaths() + fl->split_deaths()); } +#endif // SERIALGC }; -template -void BinaryTreeDictionary::print_dict_census(void) const { +template class FreeList_t> +void BinaryTreeDictionary::print_dict_census(void) const { gclog_or_tty->print("\nBinaryTree\n"); - FreeList::print_labels_on(gclog_or_tty, "size"); - PrintTreeCensusClosure ptc; + FreeList_t::print_labels_on(gclog_or_tty, "size"); + PrintTreeCensusClosure ptc; ptc.do_tree(root()); - FreeList* total = ptc.total(); - FreeList::print_labels_on(gclog_or_tty, " "); + FreeList_t* total = ptc.total(); + FreeList_t::print_labels_on(gclog_or_tty, " "); +} + +#ifndef SERIALGC +template <> +void BinaryTreeDictionary::print_dict_census(void) const { + + gclog_or_tty->print("\nBinaryTree\n"); + AdaptiveFreeList::print_labels_on(gclog_or_tty, "size"); + PrintTreeCensusClosure ptc; + ptc.do_tree(root()); + + AdaptiveFreeList* total = ptc.total(); + AdaptiveFreeList::print_labels_on(gclog_or_tty, " "); total->print_on(gclog_or_tty, "TOTAL\t"); gclog_or_tty->print( "total_free(words): " SIZE_FORMAT_W(16) @@ -1242,9 +1307,10 @@ (double)(total->desired() - total->count()) /(total->desired() != 0 ? (double)total->desired() : 1.0)); } +#endif // SERIALGC -template -class PrintFreeListsClosure : public AscendTreeCensusClosure { +template class FreeList_t> +class PrintFreeListsClosure : public AscendTreeCensusClosure { outputStream* _st; int _print_line; @@ -1253,14 +1319,14 @@ _st = st; _print_line = 0; } - void do_list(FreeList* fl) { + void do_list(FreeList_t* fl) { if (++_print_line >= 40) { - FreeList::print_labels_on(_st, "size"); + FreeList_t::print_labels_on(_st, "size"); _print_line = 0; } fl->print_on(gclog_or_tty); size_t sz = fl->size(); - for (Chunk* fc = fl->head(); fc != NULL; + for (Chunk_t* fc = fl->head(); fc != NULL; fc = fc->next()) { _st->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s", fc, (HeapWord*)fc + sz, @@ -1269,11 +1335,11 @@ } }; -template -void BinaryTreeDictionary::print_free_lists(outputStream* st) const { +template class FreeList_t> +void BinaryTreeDictionary::print_free_lists(outputStream* st) const { - FreeList::print_labels_on(st, "size"); - PrintFreeListsClosure pflc(st); + FreeList_t::print_labels_on(st, "size"); + PrintFreeListsClosure pflc(st); pflc.do_tree(root()); } @@ -1281,18 +1347,18 @@ // . _root has no parent // . parent and child point to each other // . each node's key correctly related to that of its child(ren) -template -void BinaryTreeDictionary::verify_tree() const { +template class FreeList_t> +void BinaryTreeDictionary::verify_tree() const { guarantee(root() == NULL || total_free_blocks() == 0 || total_size() != 0, "_total_size should't be 0?"); guarantee(root() == NULL || root()->parent() == NULL, "_root shouldn't have parent"); verify_tree_helper(root()); } -template -size_t BinaryTreeDictionary::verify_prev_free_ptrs(TreeList* tl) { +template class FreeList_t> +size_t BinaryTreeDictionary::verify_prev_free_ptrs(TreeList* tl) { size_t ct = 0; - for (Chunk* curFC = tl->head(); curFC != NULL; curFC = curFC->next()) { + for (Chunk_t* curFC = tl->head(); curFC != NULL; curFC = curFC->next()) { ct++; assert(curFC->prev() == NULL || curFC->prev()->is_free(), "Chunk should be free"); @@ -1303,8 +1369,8 @@ // Note: this helper is recursive rather than iterative, so use with // caution on very deep trees; and watch out for stack overflow errors; // In general, to be used only for debugging. -template -void BinaryTreeDictionary::verify_tree_helper(TreeList* tl) const { +template class FreeList_t> +void BinaryTreeDictionary::verify_tree_helper(TreeList* tl) const { if (tl == NULL) return; guarantee(tl->size() != 0, "A list must has a size"); @@ -1332,15 +1398,25 @@ verify_tree_helper(tl->right()); } -template -void BinaryTreeDictionary::verify() const { +template class FreeList_t> +void BinaryTreeDictionary::verify() const { verify_tree(); guarantee(total_size() == total_size_in_tree(root()), "Total Size inconsistency"); } +template class TreeList; +template class BinaryTreeDictionary; +template class TreeChunk; + +template class TreeList; +template class BinaryTreeDictionary; +template class TreeChunk; + + #ifndef SERIALGC // Explicitly instantiate these types for FreeChunk. -template class BinaryTreeDictionary; -template class TreeChunk; -template class TreeList; +template class TreeList; +template class BinaryTreeDictionary; +template class TreeChunk; + #endif // SERIALGC diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/binaryTreeDictionary.hpp --- a/hotspot/src/share/vm/memory/binaryTreeDictionary.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/binaryTreeDictionary.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -37,77 +37,78 @@ // A TreeList is a FreeList which can be used to maintain a // binary tree of free lists. -template class TreeChunk; -template class BinaryTreeDictionary; -template class AscendTreeCensusClosure; -template class DescendTreeCensusClosure; -template class DescendTreeSearchClosure; +template class FreeList_t> class TreeChunk; +template class FreeList_t> class BinaryTreeDictionary; +template class FreeList_t> class AscendTreeCensusClosure; +template class FreeList_t> class DescendTreeCensusClosure; +template class FreeList_t> class DescendTreeSearchClosure; -template -class TreeList: public FreeList { - friend class TreeChunk; - friend class BinaryTreeDictionary; - friend class AscendTreeCensusClosure; - friend class DescendTreeCensusClosure; - friend class DescendTreeSearchClosure; +template class FreeList_t> +class TreeList : public FreeList_t { + friend class TreeChunk; + friend class BinaryTreeDictionary; + friend class AscendTreeCensusClosure; + friend class DescendTreeCensusClosure; + friend class DescendTreeSearchClosure; - TreeList* _parent; - TreeList* _left; - TreeList* _right; + TreeList* _parent; + TreeList* _left; + TreeList* _right; protected: - TreeList* parent() const { return _parent; } - TreeList* left() const { return _left; } - TreeList* right() const { return _right; } - // Explicitly import these names into our namespace to fix name lookup with templates - using FreeList::head; - using FreeList::set_head; + TreeList* parent() const { return _parent; } + TreeList* left() const { return _left; } + TreeList* right() const { return _right; } - using FreeList::tail; - using FreeList::set_tail; - using FreeList::link_tail; + // Wrapper on call to base class, to get the template to compile. + Chunk_t* head() const { return FreeList_t::head(); } + Chunk_t* tail() const { return FreeList_t::tail(); } + void set_head(Chunk_t* head) { FreeList_t::set_head(head); } + void set_tail(Chunk_t* tail) { FreeList_t::set_tail(tail); } - using FreeList::increment_count; - NOT_PRODUCT(using FreeList::increment_returned_bytes_by;) - using FreeList::verify_chunk_in_free_list; - using FreeList::size; + size_t size() const { return FreeList_t::size(); } // Accessors for links in tree. - void set_left(TreeList* tl) { + void set_left(TreeList* tl) { _left = tl; if (tl != NULL) tl->set_parent(this); } - void set_right(TreeList* tl) { + void set_right(TreeList* tl) { _right = tl; if (tl != NULL) tl->set_parent(this); } - void set_parent(TreeList* tl) { _parent = tl; } + void set_parent(TreeList* tl) { _parent = tl; } - void clearLeft() { _left = NULL; } + void clear_left() { _left = NULL; } void clear_right() { _right = NULL; } void clear_parent() { _parent = NULL; } - void initialize() { clearLeft(); clear_right(), clear_parent(); } + void initialize() { clear_left(); clear_right(), clear_parent(); FreeList_t::initialize(); } // For constructing a TreeList from a Tree chunk or // address and size. - static TreeList* as_TreeList(TreeChunk* tc); - static TreeList* as_TreeList(HeapWord* addr, size_t size); + TreeList(); + static TreeList* + as_TreeList(TreeChunk* tc); + static TreeList* as_TreeList(HeapWord* addr, size_t size); // Returns the head of the free list as a pointer to a TreeChunk. - TreeChunk* head_as_TreeChunk(); + TreeChunk* head_as_TreeChunk(); // Returns the first available chunk in the free list as a pointer // to a TreeChunk. - TreeChunk* first_available(); + TreeChunk* first_available(); // Returns the block with the largest heap address amongst // those in the list for this size; potentially slow and expensive, // use with caution! - TreeChunk* largest_address(); + TreeChunk* largest_address(); + + TreeList* get_better_list( + BinaryTreeDictionary* dictionary); // remove_chunk_replace_if_needed() removes the given "tc" from the TreeList. // If "tc" is the first chunk in the list, it is also the @@ -115,10 +116,10 @@ // returns the possibly replaced TreeList* for the node in // the tree. It also updates the parent of the original // node to point to the new node. - TreeList* remove_chunk_replace_if_needed(TreeChunk* tc); + TreeList* remove_chunk_replace_if_needed(TreeChunk* tc); // See FreeList. - void return_chunk_at_head(TreeChunk* tc); - void return_chunk_at_tail(TreeChunk* tc); + void return_chunk_at_head(TreeChunk* tc); + void return_chunk_at_tail(TreeChunk* tc); }; // A TreeChunk is a subclass of a Chunk that additionally @@ -134,52 +135,54 @@ // on the free list for a node in the tree and is only removed if // it is the last chunk on the free list. -template -class TreeChunk : public Chunk { - friend class TreeList; - TreeList* _list; - TreeList _embedded_list; // if non-null, this chunk is on _list +template class FreeList_t> +class TreeChunk : public Chunk_t { + friend class TreeList; + TreeList* _list; + TreeList _embedded_list; // if non-null, this chunk is on _list + + static size_t _min_tree_chunk_size; + protected: - TreeList* embedded_list() const { return (TreeList*) &_embedded_list; } - void set_embedded_list(TreeList* v) { _embedded_list = *v; } + TreeList* embedded_list() const { return (TreeList*) &_embedded_list; } + void set_embedded_list(TreeList* v) { _embedded_list = *v; } public: - TreeList* list() { return _list; } - void set_list(TreeList* v) { _list = v; } - static TreeChunk* as_TreeChunk(Chunk* fc); + TreeList* list() { return _list; } + void set_list(TreeList* v) { _list = v; } + static TreeChunk* as_TreeChunk(Chunk_t* fc); // Initialize fields in a TreeChunk that should be // initialized when the TreeChunk is being added to // a free list in the tree. void initialize() { embedded_list()->initialize(); } - Chunk* next() const { return Chunk::next(); } - Chunk* prev() const { return Chunk::prev(); } - size_t size() const volatile { return Chunk::size(); } + Chunk_t* next() const { return Chunk_t::next(); } + Chunk_t* prev() const { return Chunk_t::prev(); } + size_t size() const volatile { return Chunk_t::size(); } + + static size_t min_size() { + return _min_tree_chunk_size; + } // debugging void verify_tree_chunk_list() const; + void assert_is_mangled() const; }; -template -class BinaryTreeDictionary: public FreeBlockDictionary { +template class FreeList_t> +class BinaryTreeDictionary: public FreeBlockDictionary { friend class VMStructs; - bool _splay; - bool _adaptive_freelists; size_t _total_size; size_t _total_free_blocks; - TreeList* _root; + TreeList* _root; // private accessors - bool splay() const { return _splay; } - void set_splay(bool v) { _splay = v; } void set_total_size(size_t v) { _total_size = v; } virtual void inc_total_size(size_t v); virtual void dec_total_size(size_t v); - size_t total_free_blocks() const { return _total_free_blocks; } void set_total_free_blocks(size_t v) { _total_free_blocks = v; } - TreeList* root() const { return _root; } - void set_root(TreeList* v) { _root = v; } - bool adaptive_freelists() { return _adaptive_freelists; } + TreeList* root() const { return _root; } + void set_root(TreeList* v) { _root = v; } // This field is added and can be set to point to the // the Mutex used to synchronize access to the @@ -191,54 +194,55 @@ // return it. If the chunk // is the last chunk of that size, remove the node for that size // from the tree. - TreeChunk* get_chunk_from_tree(size_t size, enum FreeBlockDictionary::Dither dither, bool splay); - // Return a list of the specified size or NULL from the tree. - // The list is not removed from the tree. - TreeList* find_list (size_t size) const; + TreeChunk* get_chunk_from_tree(size_t size, enum FreeBlockDictionary::Dither dither); // Remove this chunk from the tree. If the removal results // in an empty list in the tree, remove the empty list. - TreeChunk* remove_chunk_from_tree(TreeChunk* tc); + TreeChunk* remove_chunk_from_tree(TreeChunk* tc); // Remove the node in the trees starting at tl that has the // minimum value and return it. Repair the tree as needed. - TreeList* remove_tree_minimum(TreeList* tl); - void semi_splay_step(TreeList* tl); + TreeList* remove_tree_minimum(TreeList* tl); // Add this free chunk to the tree. - void insert_chunk_in_tree(Chunk* freeChunk); + void insert_chunk_in_tree(Chunk_t* freeChunk); public: - static const size_t min_tree_chunk_size = sizeof(TreeChunk)/HeapWordSize; + // Return a list of the specified size or NULL from the tree. + // The list is not removed from the tree. + TreeList* find_list (size_t size) const; void verify_tree() const; // verify that the given chunk is in the tree. - bool verify_chunk_in_free_list(Chunk* tc) const; + bool verify_chunk_in_free_list(Chunk_t* tc) const; private: - void verify_tree_helper(TreeList* tl) const; - static size_t verify_prev_free_ptrs(TreeList* tl); + void verify_tree_helper(TreeList* tl) const; + static size_t verify_prev_free_ptrs(TreeList* tl); // Returns the total number of chunks in the list. - size_t total_list_length(TreeList* tl) const; + size_t total_list_length(TreeList* tl) const; // Returns the total number of words in the chunks in the tree // starting at "tl". - size_t total_size_in_tree(TreeList* tl) const; + size_t total_size_in_tree(TreeList* tl) const; // Returns the sum of the square of the size of each block // in the tree starting at "tl". - double sum_of_squared_block_sizes(TreeList* const tl) const; + double sum_of_squared_block_sizes(TreeList* const tl) const; // Returns the total number of free blocks in the tree starting // at "tl". - size_t total_free_blocks_in_tree(TreeList* tl) const; - size_t num_free_blocks() const; - size_t treeHeight() const; - size_t tree_height_helper(TreeList* tl) const; - size_t total_nodes_in_tree(TreeList* tl) const; - size_t total_nodes_helper(TreeList* tl) const; + size_t total_free_blocks_in_tree(TreeList* tl) const; + size_t num_free_blocks() const; + size_t tree_height() const; + size_t tree_height_helper(TreeList* tl) const; + size_t total_nodes_in_tree(TreeList* tl) const; + size_t total_nodes_helper(TreeList* tl) const; public: // Constructor - BinaryTreeDictionary(bool adaptive_freelists, bool splay = false); - BinaryTreeDictionary(MemRegion mr, bool adaptive_freelists, bool splay = false); + BinaryTreeDictionary() : + _total_size(0), _total_free_blocks(0), _root(0) {} + + BinaryTreeDictionary(MemRegion mr); // Public accessors size_t total_size() const { return _total_size; } + size_t total_free_blocks() const { return _total_free_blocks; } // Reset the dictionary to the initial conditions with // a single free chunk. @@ -249,23 +253,24 @@ // Return a chunk of size "size" or greater from // the tree. - // want a better dynamic splay strategy for the future. - Chunk* get_chunk(size_t size, enum FreeBlockDictionary::Dither dither) { - FreeBlockDictionary::verify_par_locked(); - Chunk* res = get_chunk_from_tree(size, dither, splay()); + Chunk_t* get_chunk(size_t size, enum FreeBlockDictionary::Dither dither) { + FreeBlockDictionary::verify_par_locked(); + Chunk_t* res = get_chunk_from_tree(size, dither); assert(res == NULL || res->is_free(), "Should be returning a free chunk"); + assert(dither != FreeBlockDictionary::exactly || + res->size() == size, "Not correct size"); return res; } - void return_chunk(Chunk* chunk) { - FreeBlockDictionary::verify_par_locked(); + void return_chunk(Chunk_t* chunk) { + FreeBlockDictionary::verify_par_locked(); insert_chunk_in_tree(chunk); } - void remove_chunk(Chunk* chunk) { - FreeBlockDictionary::verify_par_locked(); - remove_chunk_from_tree((TreeChunk*)chunk); + void remove_chunk(Chunk_t* chunk) { + FreeBlockDictionary::verify_par_locked(); + remove_chunk_from_tree((TreeChunk*)chunk); assert(chunk->is_free(), "Should still be a free chunk"); } @@ -281,19 +286,19 @@ } size_t min_size() const { - return min_tree_chunk_size; + return TreeChunk::min_size(); } double sum_of_squared_block_sizes() const { return sum_of_squared_block_sizes(root()); } - Chunk* find_chunk_ends_at(HeapWord* target) const; + Chunk_t* find_chunk_ends_at(HeapWord* target) const; // Find the list with size "size" in the binary tree and update // the statistics in the list according to "split" (chunk was // split or coalesce) and "birth" (chunk was added or removed). - void dict_census_udpate(size_t size, bool split, bool birth); + void dict_census_update(size_t size, bool split, bool birth); // Return true if the dictionary is overpopulated (more chunks of // this size than desired) for size "size". bool coal_dict_over_populated(size_t size); @@ -307,7 +312,7 @@ // statistics for the sweep. void end_sweep_dict_census(double splitSurplusPercent); // Return the largest free chunk in the tree. - Chunk* find_largest_dict() const; + Chunk_t* find_largest_dict() const; // Accessors for statistics void set_tree_surplus(double splitSurplusPercent); void set_tree_hints(void); diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/freeBlockDictionary.cpp --- a/hotspot/src/share/vm/memory/freeBlockDictionary.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/freeBlockDictionary.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -27,6 +27,8 @@ #include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" #endif // SERIALGC #include "memory/freeBlockDictionary.hpp" +#include "memory/metablock.hpp" +#include "memory/metachunk.hpp" #ifdef TARGET_OS_FAMILY_linux # include "thread_linux.inline.hpp" #endif @@ -62,6 +64,9 @@ } #endif +template class FreeBlockDictionary; +template class FreeBlockDictionary; + #ifndef SERIALGC // Explicitly instantiate for FreeChunk template class FreeBlockDictionary; diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/freeBlockDictionary.hpp --- a/hotspot/src/share/vm/memory/freeBlockDictionary.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/freeBlockDictionary.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -66,7 +66,7 @@ virtual void reset(HeapWord* addr, size_t size) = 0; virtual void reset() = 0; - virtual void dict_census_udpate(size_t size, bool split, bool birth) = 0; + virtual void dict_census_update(size_t size, bool split, bool birth) = 0; virtual bool coal_dict_over_populated(size_t size) = 0; virtual void begin_sweep_dict_census(double coalSurplusPercent, float inter_sweep_current, float inter_sweep_estimate, diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/freeList.cpp --- a/hotspot/src/share/vm/memory/freeList.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/freeList.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -25,6 +25,8 @@ #include "precompiled.hpp" #include "memory/freeBlockDictionary.hpp" #include "memory/freeList.hpp" +#include "memory/metablock.hpp" +#include "memory/metachunk.hpp" #include "memory/sharedHeap.hpp" #include "runtime/globals.hpp" #include "runtime/mutex.hpp" @@ -49,8 +51,6 @@ { _size = 0; _count = 0; - _hint = 0; - init_statistics(); } template @@ -62,34 +62,50 @@ { _size = fc->size(); _count = 1; - _hint = 0; - init_statistics(); -#ifndef PRODUCT - _allocation_stats.set_returned_bytes(size() * HeapWordSize); -#endif } template -void FreeList::reset(size_t hint) { +void FreeList::link_head(Chunk* v) { + assert_proper_lock_protection(); + set_head(v); + // If this method is not used (just set the head instead), + // this check can be avoided. + if (v != NULL) { + v->link_prev(NULL); + } +} + + + +template +void FreeList::reset() { + // Don't set the _size to 0 because this method is + // used with a existing list that has a size but which has + // been emptied. + // Don't clear the _protecting_lock of an existing list. set_count(0); set_head(NULL); set_tail(NULL); - set_hint(hint); } template -void FreeList::init_statistics(bool split_birth) { - _allocation_stats.initialize(split_birth); +void FreeList::initialize() { +#ifdef ASSERT + // Needed early because it might be checked in other initializing code. + set_protecting_lock(NULL); +#endif + reset(); + set_size(0); } -template -Chunk* FreeList::get_chunk_at_head() { +template +Chunk_t* FreeList::get_chunk_at_head() { assert_proper_lock_protection(); assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); - Chunk* fc = head(); + Chunk_t* fc = head(); if (fc != NULL) { - Chunk* nextFC = fc->next(); + Chunk_t* nextFC = fc->next(); if (nextFC != NULL) { // The chunk fc being removed has a "next". Set the "next" to the // "prev" of fc. @@ -197,11 +213,6 @@ link_tail(chunk); } increment_count(); // of # of chunks in list - DEBUG_ONLY( - if (record_return) { - increment_returned_bytes_by(size()*HeapWordSize); - } - ) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); assert(head() == NULL || head()->size() == size(), "wrong item on list"); @@ -233,11 +244,6 @@ } link_tail(chunk); increment_count(); // of # of chunks in list - DEBUG_ONLY( - if (record_return) { - increment_returned_bytes_by(size()*HeapWordSize); - } - ) assert(head() == NULL || head()->prev() == NULL, "list invariant"); assert(tail() == NULL || tail()->next() == NULL, "list invariant"); assert(head() == NULL || head()->size() == size(), "wrong item on list"); @@ -273,7 +279,7 @@ } } -// verify_chunk_in_free_list() is used to verify that an item is in this free list. +// verify_chunk_in_free_lists() is used to verify that an item is in this free list. // It is used as a debugging aid. template bool FreeList::verify_chunk_in_free_list(Chunk* fc) const { @@ -294,40 +300,14 @@ #ifndef PRODUCT template -void FreeList::verify_stats() const { - // The +1 of the LH comparand is to allow some "looseness" in - // checking: we usually call this interface when adding a block - // and we'll subsequently update the stats; we cannot update the - // stats beforehand because in the case of the large-block BT - // dictionary for example, this might be the first block and - // in that case there would be no place that we could record - // the stats (which are kept in the block itself). - assert((_allocation_stats.prev_sweep() + _allocation_stats.split_births() - + _allocation_stats.coal_births() + 1) // Total Production Stock + 1 - >= (_allocation_stats.split_deaths() + _allocation_stats.coal_deaths() - + (ssize_t)count()), // Total Current Stock + depletion - err_msg("FreeList " PTR_FORMAT " of size " SIZE_FORMAT - " violates Conservation Principle: " - "prev_sweep(" SIZE_FORMAT ")" - " + split_births(" SIZE_FORMAT ")" - " + coal_births(" SIZE_FORMAT ") + 1 >= " - " split_deaths(" SIZE_FORMAT ")" - " coal_deaths(" SIZE_FORMAT ")" - " + count(" SSIZE_FORMAT ")", - this, _size, _allocation_stats.prev_sweep(), _allocation_stats.split_births(), - _allocation_stats.split_births(), _allocation_stats.split_deaths(), - _allocation_stats.coal_deaths(), count())); -} - -template void FreeList::assert_proper_lock_protection_work() const { - assert(_protecting_lock != NULL, "Don't call this directly"); + assert(protecting_lock() != NULL, "Don't call this directly"); assert(ParallelGCThreads > 0, "Don't call this directly"); Thread* thr = Thread::current(); if (thr->is_VM_thread() || thr->is_ConcurrentGC_thread()) { // assert that we are holding the freelist lock } else if (thr->is_GC_task_thread()) { - assert(_protecting_lock->owned_by_self(), "FreeList RACE DETECTED"); + assert(protecting_lock()->owned_by_self(), "FreeList RACE DETECTED"); } else if (thr->is_Java_thread()) { assert(!SafepointSynchronize::is_at_safepoint(), "Should not be executing"); } else { @@ -350,21 +330,17 @@ // to the call is a non-null string, it is printed in the first column; // otherwise, if the argument is null (the default), then the size of the // (free list) block is printed in the first column. -template -void FreeList::print_on(outputStream* st, const char* c) const { +template +void FreeList::print_on(outputStream* st, const char* c) const { if (c != NULL) { st->print("%16s", c); } else { st->print(SIZE_FORMAT_W(16), size()); } - st->print("\t" - SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" - SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\t" SSIZE_FORMAT_W(14) "\n", - bfr_surp(), surplus(), desired(), prev_sweep(), before_sweep(), - count(), coal_births(), coal_deaths(), split_births(), split_deaths()); } +template class FreeList; +template class FreeList; #ifndef SERIALGC -// Needs to be after the definitions have been seen. template class FreeList; #endif // SERIALGC diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/freeList.hpp --- a/hotspot/src/share/vm/memory/freeList.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/freeList.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -40,23 +40,19 @@ // for that implementation. class Mutex; -template class TreeList; -template class PrintTreeCensusClosure; -template +template class FreeList VALUE_OBJ_CLASS_SPEC { friend class CompactibleFreeListSpace; friend class VMStructs; - friend class PrintTreeCensusClosure; private: - Chunk* _head; // Head of list of free chunks - Chunk* _tail; // Tail of list of free chunks + Chunk_t* _head; // Head of list of free chunks + Chunk_t* _tail; // Tail of list of free chunks size_t _size; // Size in Heap words of each chunk ssize_t _count; // Number of entries in list - size_t _hint; // next larger size list with a positive surplus - AllocationStats _allocation_stats; // allocation-related statistics + protected: #ifdef ASSERT Mutex* _protecting_lock; @@ -71,10 +67,6 @@ #endif } - // Initialize the allocation statistics. - protected: - void init_statistics(bool split_birth = false); - void set_count(ssize_t v) { _count = v;} void increment_count() { _count++; } @@ -89,52 +81,48 @@ // Construct a list without any entries. FreeList(); // Construct a list with "fc" as the first (and lone) entry in the list. - FreeList(Chunk* fc); + FreeList(Chunk_t* fc); - // Reset the head, tail, hint, and count of a free list. - void reset(size_t hint); + // Do initialization + void initialize(); + + // Reset the head, tail, and count of a free list. + void reset(); // Declare the current free list to be protected by the given lock. #ifdef ASSERT - void set_protecting_lock(Mutex* protecting_lock) { - _protecting_lock = protecting_lock; + Mutex* protecting_lock() const { return _protecting_lock; } + void set_protecting_lock(Mutex* v) { + _protecting_lock = v; } #endif // Accessors. - Chunk* head() const { + Chunk_t* head() const { assert_proper_lock_protection(); return _head; } - void set_head(Chunk* v) { + void set_head(Chunk_t* v) { assert_proper_lock_protection(); _head = v; assert(!_head || _head->size() == _size, "bad chunk size"); } // Set the head of the list and set the prev field of non-null // values to NULL. - void link_head(Chunk* v) { - assert_proper_lock_protection(); - set_head(v); - // If this method is not used (just set the head instead), - // this check can be avoided. - if (v != NULL) { - v->link_prev(NULL); - } - } + void link_head(Chunk_t* v); - Chunk* tail() const { + Chunk_t* tail() const { assert_proper_lock_protection(); return _tail; } - void set_tail(Chunk* v) { + void set_tail(Chunk_t* v) { assert_proper_lock_protection(); _tail = v; assert(!_tail || _tail->size() == _size, "bad chunk size"); } // Set the tail of the list and set the next field of non-null // values to NULL. - void link_tail(Chunk* v) { + void link_tail(Chunk_t* v) { assert_proper_lock_protection(); set_tail(v); if (v != NULL) { @@ -152,174 +140,45 @@ assert_proper_lock_protection(); _size = v; } - ssize_t count() const { - return _count; - } - size_t hint() const { - return _hint; - } - void set_hint(size_t v) { - assert_proper_lock_protection(); - assert(v == 0 || _size < v, "Bad hint"); _hint = v; - } - - // Accessors for statistics - AllocationStats* allocation_stats() { - assert_proper_lock_protection(); - return &_allocation_stats; - } - - ssize_t desired() const { - return _allocation_stats.desired(); - } - void set_desired(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_desired(v); - } - void compute_desired(float inter_sweep_current, - float inter_sweep_estimate, - float intra_sweep_estimate) { - assert_proper_lock_protection(); - _allocation_stats.compute_desired(_count, - inter_sweep_current, - inter_sweep_estimate, - intra_sweep_estimate); - } - ssize_t coal_desired() const { - return _allocation_stats.coal_desired(); - } - void set_coal_desired(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_desired(v); - } - - ssize_t surplus() const { - return _allocation_stats.surplus(); - } - void set_surplus(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_surplus(v); - } - void increment_surplus() { - assert_proper_lock_protection(); - _allocation_stats.increment_surplus(); - } - void decrement_surplus() { - assert_proper_lock_protection(); - _allocation_stats.decrement_surplus(); - } + ssize_t count() const { return _count; } + void set_count(ssize_t v) { _count = v;} - ssize_t bfr_surp() const { - return _allocation_stats.bfr_surp(); - } - void set_bfr_surp(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_bfr_surp(v); - } - ssize_t prev_sweep() const { - return _allocation_stats.prev_sweep(); - } - void set_prev_sweep(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_prev_sweep(v); - } - ssize_t before_sweep() const { - return _allocation_stats.before_sweep(); - } - void set_before_sweep(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_before_sweep(v); - } - - ssize_t coal_births() const { - return _allocation_stats.coal_births(); - } - void set_coal_births(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_births(v); - } - void increment_coal_births() { - assert_proper_lock_protection(); - _allocation_stats.increment_coal_births(); - } + size_t get_better_size() { return size(); } - ssize_t coal_deaths() const { - return _allocation_stats.coal_deaths(); - } - void set_coal_deaths(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_coal_deaths(v); - } - void increment_coal_deaths() { - assert_proper_lock_protection(); - _allocation_stats.increment_coal_deaths(); - } - - ssize_t split_births() const { - return _allocation_stats.split_births(); - } - void set_split_births(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_split_births(v); - } - void increment_split_births() { - assert_proper_lock_protection(); - _allocation_stats.increment_split_births(); - } - - ssize_t split_deaths() const { - return _allocation_stats.split_deaths(); - } - void set_split_deaths(ssize_t v) { - assert_proper_lock_protection(); - _allocation_stats.set_split_deaths(v); - } - void increment_split_deaths() { - assert_proper_lock_protection(); - _allocation_stats.increment_split_deaths(); - } - - NOT_PRODUCT( - // For debugging. The "_returned_bytes" in all the lists are summed - // and compared with the total number of bytes swept during a - // collection. - size_t returned_bytes() const { return _allocation_stats.returned_bytes(); } - void set_returned_bytes(size_t v) { _allocation_stats.set_returned_bytes(v); } - void increment_returned_bytes_by(size_t v) { - _allocation_stats.set_returned_bytes(_allocation_stats.returned_bytes() + v); - } - ) + size_t returned_bytes() const { ShouldNotReachHere(); return 0; } + void set_returned_bytes(size_t v) {} + void increment_returned_bytes_by(size_t v) {} // Unlink head of list and return it. Returns NULL if // the list is empty. - Chunk* get_chunk_at_head(); + Chunk_t* get_chunk_at_head(); // Remove the first "n" or "count", whichever is smaller, chunks from the // list, setting "fl", which is required to be empty, to point to them. - void getFirstNChunksFromList(size_t n, FreeList* fl); + void getFirstNChunksFromList(size_t n, FreeList* fl); // Unlink this chunk from it's free list - void remove_chunk(Chunk* fc); + void remove_chunk(Chunk_t* fc); // Add this chunk to this free list. - void return_chunk_at_head(Chunk* fc); - void return_chunk_at_tail(Chunk* fc); + void return_chunk_at_head(Chunk_t* fc); + void return_chunk_at_tail(Chunk_t* fc); // Similar to returnChunk* but also records some diagnostic // information. - void return_chunk_at_head(Chunk* fc, bool record_return); - void return_chunk_at_tail(Chunk* fc, bool record_return); + void return_chunk_at_head(Chunk_t* fc, bool record_return); + void return_chunk_at_tail(Chunk_t* fc, bool record_return); // Prepend "fl" (whose size is required to be the same as that of "this") // to the front of "this" list. - void prepend(FreeList* fl); + void prepend(FreeList* fl); // Verify that the chunk is in the list. // found. Return NULL if "fc" is not found. - bool verify_chunk_in_free_list(Chunk* fc) const; + bool verify_chunk_in_free_list(Chunk_t* fc) const; // Stats verification - void verify_stats() const PRODUCT_RETURN; +// void verify_stats() const { ShouldNotReachHere(); }; // Printing support static void print_labels_on(outputStream* st, const char* c); diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/metablock.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/memory/metablock.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#ifndef SHARE_VM_MEMORY_METABLOCK_HPP +#define SHARE_VM_MEMORY_METABLOCK_HPP + +// Metablock are the unit of allocation from a Chunk. It is initialized +// with the size of the requested allocation. That size is overwritten +// once the allocation returns. +// +// A Metablock may be reused by its SpaceManager but are never moved between +// SpaceManagers. There is no explicit link to the Metachunk +// from which it was allocated. Metablock may be deallocated and +// put on a freelist but the space is never freed, rather +// the Metachunk it is a part of will be deallocated when it's +// associated class loader is collected. + +class Metablock VALUE_OBJ_CLASS_SPEC { + friend class VMStructs; + private: + // Used to align the allocation (see below). + union block_t { + void* _data[3]; + struct header_t { + size_t _word_size; + Metablock* _next; + Metablock* _prev; + } _header; + } _block; + static size_t _min_block_byte_size; + static size_t _overhead; + + typedef union block_t Block; + typedef struct header_t Header; + const Block* block() const { return &_block; } + const Block::header_t* header() const { return &(block()->_header); } + public: + + static Metablock* initialize(MetaWord* p, size_t word_size); + + // This places the body of the block at a 2 word boundary + // because every block starts on a 2 word boundary. Work out + // how to make the body on a 2 word boundary if the block + // starts on a arbitrary boundary. JJJ + + size_t word_size() const { return header()->_word_size; } + void set_word_size(size_t v) { _block._header._word_size = v; } + size_t size() const volatile { return _block._header._word_size; } + void set_size(size_t v) { _block._header._word_size = v; } + Metablock* next() const { return header()->_next; } + void set_next(Metablock* v) { _block._header._next = v; } + Metablock* prev() const { return header()->_prev; } + void set_prev(Metablock* v) { _block._header._prev = v; } + + static size_t min_block_byte_size() { return _min_block_byte_size; } + static size_t overhead() { return _overhead; } + + bool is_free() { return header()->_word_size != 0; } + void clear_next() { set_next(NULL); } + void link_prev(Metablock* ptr) { set_prev(ptr); } + uintptr_t* end() { return ((uintptr_t*) this) + size(); } + bool cantCoalesce() const { return false; } + void link_next(Metablock* ptr) { set_next(ptr); } + void link_after(Metablock* ptr){ + link_next(ptr); + if (ptr != NULL) ptr->link_prev(this); + } + + // Should not be needed in a free list of Metablocks + void markNotFree() { ShouldNotReachHere(); } + + // Debug support +#ifdef ASSERT + void* prev_addr() const { return (void*)&_block._header._prev; } + void* next_addr() const { return (void*)&_block._header._next; } + void* size_addr() const { return (void*)&_block._header._word_size; } +#endif + bool verify_chunk_in_free_list(Metablock* tc) const { return true; } + bool verify_par_locked() { return true; } + + void assert_is_mangled() const {/* Don't check "\*/} +}; +#endif // SHARE_VM_MEMORY_METABLOCK_HPP diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/metachunk.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/memory/metachunk.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#ifndef SHARE_VM_MEMORY_METACHUNK_HPP +#define SHARE_VM_MEMORY_METACHUNK_HPP + +// Metachunk - Quantum of allocation from a Virtualspace +// Metachunks are reused (when freed are put on a global freelist) and +// have no permanent association to a SpaceManager. + +// +--------------+ <- end +// | | --+ ---+ +// | | | free | +// | | | | +// | | | | capacity +// | | | | +// | | <- top --+ | +// | | ---+ | +// | | | used | +// | | | | +// | | | | +// +--------------+ <- bottom ---+ ---+ + +class Metachunk VALUE_OBJ_CLASS_SPEC { + // link to support lists of chunks + Metachunk* _next; + Metachunk* _prev; + + MetaWord* _bottom; + MetaWord* _end; + MetaWord* _top; + size_t _word_size; + // Used in a guarantee() so included in the Product builds + // even through it is only for debugging. + bool _is_free; + + // Metachunks are allocated out of a MetadataVirtualSpace and + // and use some of its space to describe itself (plus alignment + // considerations). Metadata is allocated in the rest of the chunk. + // This size is the overhead of maintaining the Metachunk within + // the space. + static size_t _overhead; + + void set_bottom(MetaWord* v) { _bottom = v; } + void set_end(MetaWord* v) { _end = v; } + void set_top(MetaWord* v) { _top = v; } + void set_word_size(size_t v) { _word_size = v; } + public: +#ifdef ASSERT + Metachunk() : _bottom(NULL), _end(NULL), _top(NULL), _is_free(false) {} +#else + Metachunk() : _bottom(NULL), _end(NULL), _top(NULL) {} +#endif + + // Used to add a Metachunk to a list of Metachunks + void set_next(Metachunk* v) { _next = v; assert(v != this, "Boom");} + void set_prev(Metachunk* v) { _prev = v; assert(v != this, "Boom");} + + MetaWord* allocate(size_t word_size); + static Metachunk* initialize(MetaWord* ptr, size_t word_size); + + // Accessors + Metachunk* next() const { return _next; } + Metachunk* prev() const { return _prev; } + MetaWord* bottom() const { return _bottom; } + MetaWord* end() const { return _end; } + MetaWord* top() const { return _top; } + size_t word_size() const { return _word_size; } + size_t size() const volatile { return _word_size; } + void set_size(size_t v) { _word_size = v; } + bool is_free() { return _is_free; } + void set_is_free(bool v) { _is_free = v; } + static size_t overhead() { return _overhead; } + void clear_next() { set_next(NULL); } + void link_prev(Metachunk* ptr) { set_prev(ptr); } + uintptr_t* end() { return ((uintptr_t*) this) + size(); } + bool cantCoalesce() const { return false; } + void link_next(Metachunk* ptr) { set_next(ptr); } + void link_after(Metachunk* ptr){ + link_next(ptr); + if (ptr != NULL) ptr->link_prev(this); + } + + // Reset top to bottom so chunk can be reused. + void reset_empty() { _top = (_bottom + _overhead); } + bool is_empty() { return _top == (_bottom + _overhead); } + + // used (has been allocated) + // free (available for future allocations) + // capacity (total size of chunk) + size_t used_word_size(); + size_t free_word_size(); + size_t capacity_word_size(); + + // Debug support +#ifdef ASSERT + void* prev_addr() const { return (void*)&_prev; } + void* next_addr() const { return (void*)&_next; } + void* size_addr() const { return (void*)&_word_size; } +#endif + bool verify_chunk_in_free_list(Metachunk* tc) const { return true; } + bool verify_par_locked() { return true; } + + void assert_is_mangled() const {/* Don't check "\*/} + +#ifdef ASSERT + void mangle(); +#endif // ASSERT + + void print_on(outputStream* st) const; + void verify(); +}; +#endif // SHARE_VM_MEMORY_METACHUNK_HPP diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/metaspace.cpp --- a/hotspot/src/share/vm/memory/metaspace.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/metaspace.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -24,9 +24,12 @@ #include "precompiled.hpp" #include "gc_interface/collectedHeap.hpp" #include "memory/binaryTreeDictionary.hpp" +#include "memory/freeList.hpp" #include "memory/collectorPolicy.hpp" #include "memory/filemap.hpp" #include "memory/freeList.hpp" +#include "memory/metablock.hpp" +#include "memory/metachunk.hpp" #include "memory/metaspace.hpp" #include "memory/metaspaceShared.hpp" #include "memory/resourceArea.hpp" @@ -37,15 +40,8 @@ #include "utilities/copy.hpp" #include "utilities/debug.hpp" -// Define this macro to deallocate Metablock. If not defined, -// blocks are not yet deallocated and are only mangled. -#undef DEALLOCATE_BLOCKS - -// Easily recognizable patterns -// These patterns can be the same in 32bit or 64bit since -// they only have to be easily recognizable. -const void* metaspace_allocation_leader = (void*) 0X11111111; -const void* metaspace_allocation_trailer = (void*) 0X77777777; +typedef BinaryTreeDictionary BlockTreeDictionary; +typedef BinaryTreeDictionary ChunkTreeDictionary; // Parameters for stress mode testing const uint metadata_deallocate_a_lot_block = 10; @@ -53,7 +49,6 @@ size_t const allocation_from_dictionary_limit = 64 * K; const size_t metadata_chunk_initialize = 0xf7f7f7f7; const size_t metadata_deallocate = 0xf5f5f5f5; -const size_t metadata_space_manager_allocate = 0xf3f3f3f3; MetaWord* last_allocated = 0; @@ -62,11 +57,12 @@ SmallIndex = 0, MediumIndex = 1, HumongousIndex = 2, - NumberOfFreeLists = 3 + NumberOfFreeLists = 2, + NumberOfInUseLists = 3 }; static ChunkIndex next_chunk_index(ChunkIndex i) { - assert(i < NumberOfFreeLists, "Out of bound"); + assert(i < NumberOfInUseLists, "Out of bound"); return (ChunkIndex) (i+1); } @@ -100,164 +96,13 @@ // the Chunk after the header for the Chunk) where as Metachunks // point to space in a VirtualSpace. To replace Metachunks with // Chunks, change Chunks so that they can be allocated out of a VirtualSpace. -// - -// Metablock are the unit of allocation from a Chunk. It contains -// the size of the requested allocation in a debug build. -// Also in a debug build it has a marker before and after the -// body of the block. The address of the body is the address returned -// by the allocation. -// -// Layout in a debug build. In a product build only the body is present. -// -// +-----------+-----------+------------+ +-----------+ -// | word size | leader | body | ... | trailer | -// +-----------+-----------+------------+ +-----------+ -// -// A Metablock may be reused by its SpaceManager but are never moved between -// SpaceManagers. There is no explicit link to the Metachunk -// from which it was allocated. Metablock are not deallocated, rather -// the Metachunk it is a part of will be deallocated when it's -// associated class loader is collected. -// -// When the word size of a block is passed in to the deallocation -// call the word size no longer needs to be part of a Metablock. - -class Metablock { - friend class VMStructs; - private: - // Used to align the allocation (see below) and for debugging. -#ifdef ASSERT - struct { - size_t _word_size; - void* _leader; - } _header; - void* _data[1]; -#endif - static size_t _overhead; - +size_t Metablock::_min_block_byte_size = sizeof(Metablock); #ifdef ASSERT - void set_word_size(size_t v) { _header._word_size = v; } - void* leader() { return _header._leader; } - void* trailer() { - jlong index = (jlong) _header._word_size - sizeof(_header)/BytesPerWord - 1; - assert(index > 0, err_msg("Bad indexling of trailer %d", index)); - void** ptr = &_data[index]; - return *ptr; - } - void set_leader(void* v) { _header._leader = v; } - void set_trailer(void* v) { - void** ptr = &_data[_header._word_size - sizeof(_header)/BytesPerWord - 1]; - *ptr = v; - } - public: - size_t word_size() { return _header._word_size; } -#endif - public: - - static Metablock* initialize(MetaWord* p, size_t word_size); - - // This places the body of the block at a 2 word boundary - // because every block starts on a 2 word boundary. Work out - // how to make the body on a 2 word boundary if the block - // starts on a arbitrary boundary. JJJ - -#ifdef ASSERT - MetaWord* data() { return (MetaWord*) &_data[0]; } + size_t Metablock::_overhead = + Chunk::aligned_overhead_size(sizeof(Metablock)) / BytesPerWord; #else - MetaWord* data() { return (MetaWord*) this; } -#endif - static Metablock* metablock_from_data(MetaWord* p) { -#ifdef ASSERT - size_t word_offset = offset_of(Metablock, _data)/BytesPerWord; - Metablock* result = (Metablock*) (p - word_offset); - return result; -#else - return (Metablock*) p; + size_t Metablock::_overhead = 0; #endif - } - - static size_t overhead() { return _overhead; } - void verify(); -}; - -// Metachunk - Quantum of allocation from a Virtualspace -// Metachunks are reused (when freed are put on a global freelist) and -// have no permanent association to a SpaceManager. - -// +--------------+ <- end -// | | --+ ---+ -// | | | free | -// | | | | -// | | | | capacity -// | | | | -// | | <- top --+ | -// | | ---+ | -// | | | used | -// | | | | -// | | | | -// +--------------+ <- bottom ---+ ---+ - -class Metachunk VALUE_OBJ_CLASS_SPEC { - // link to support lists of chunks - Metachunk* _next; - - MetaWord* _bottom; - MetaWord* _end; - MetaWord* _top; - size_t _word_size; - - // Metachunks are allocated out of a MetadataVirtualSpace and - // and use some of its space to describe itself (plus alignment - // considerations). Metadata is allocated in the rest of the chunk. - // This size is the overhead of maintaining the Metachunk within - // the space. - static size_t _overhead; - - void set_bottom(MetaWord* v) { _bottom = v; } - void set_end(MetaWord* v) { _end = v; } - void set_top(MetaWord* v) { _top = v; } - void set_word_size(size_t v) { _word_size = v; } - public: - - // Used to add a Metachunk to a list of Metachunks - void set_next(Metachunk* v) { _next = v; assert(v != this, "Boom");} - - Metablock* allocate(size_t word_size); - static Metachunk* initialize(MetaWord* ptr, size_t word_size); - - // Accessors - Metachunk* next() const { return _next; } - MetaWord* bottom() const { return _bottom; } - MetaWord* end() const { return _end; } - MetaWord* top() const { return _top; } - size_t word_size() const { return _word_size; } - static size_t overhead() { return _overhead; } - - // Reset top to bottom so chunk can be reused. - void reset_empty() { _top = (_bottom + _overhead); } - bool is_empty() { return _top == (_bottom + _overhead); } - - // used (has been allocated) - // free (available for future allocations) - // capacity (total size of chunk) - size_t used_word_size(); - size_t free_word_size(); - size_t capacity_word_size(); - -#ifdef ASSERT - void mangle() { - // Mangle the payload of the chunk and not the links that - // maintain list of chunks. - HeapWord* start = (HeapWord*)(bottom() + overhead()); - size_t word_size = capacity_word_size() - overhead(); - Copy::fill_to_words(start, word_size, metadata_chunk_initialize); - } -#endif // ASSERT - - void print_on(outputStream* st) const; - void verify(); -}; // Pointer to list of Metachunks. @@ -292,7 +137,10 @@ // SmallChunk // MediumChunk // HumongousChunk - ChunkList _free_chunks[3]; + ChunkList _free_chunks[NumberOfFreeLists]; + + // HumongousChunk + ChunkTreeDictionary _humongous_dictionary; // ChunkManager in all lists of this type size_t _free_chunks_total; @@ -337,7 +185,9 @@ } ChunkList* free_medium_chunks() { return &_free_chunks[1]; } ChunkList* free_small_chunks() { return &_free_chunks[0]; } - ChunkList* free_humongous_chunks() { return &_free_chunks[2]; } + ChunkTreeDictionary* humongous_dictionary() { + return &_humongous_dictionary; + } ChunkList* free_chunks(ChunkIndex index); @@ -356,41 +206,35 @@ void locked_print_free_chunks(outputStream* st); void locked_print_sum_free_chunks(outputStream* st); + + void print_on(outputStream* st); }; // Used to manage the free list of Metablocks (a block corresponds // to the allocation of a quantum of metadata). class BlockFreelist VALUE_OBJ_CLASS_SPEC { -#ifdef DEALLOCATE_BLOCKS - BinaryTreeDictionary* _dictionary; -#endif - static Metablock* initialize_free_chunk(Metablock* block, size_t word_size); - -#ifdef DEALLOCATE_BLOCKS + BlockTreeDictionary* _dictionary; + static Metablock* initialize_free_chunk(MetaWord* p, size_t word_size); + // Accessors - BinaryTreeDictionary* dictionary() const { return _dictionary; } -#endif + BlockTreeDictionary* dictionary() const { return _dictionary; } public: BlockFreelist(); ~BlockFreelist(); // Get and return a block to the free list - Metablock* get_block(size_t word_size); - void return_block(Metablock* block, size_t word_size); - - size_t totalSize() { -#ifdef DEALLOCATE_BLOCKS - if (dictionary() == NULL) { - return 0; - } else { - return dictionary()->totalSize(); - } -#else + MetaWord* get_block(size_t word_size); + void return_block(MetaWord* p, size_t word_size); + + size_t total_size() { + if (dictionary() == NULL) { return 0; -#endif + } else { + return dictionary()->total_size(); } +} void print_on(outputStream* st) const; }; @@ -600,7 +444,6 @@ }; }; - class Metadebug : AllStatic { // Debugging support for Metaspaces static int _deallocate_block_a_lot_count; @@ -655,7 +498,7 @@ // List of chunks in use by this SpaceManager. Allocations // are done from the current chunk. The list is used for deallocating // chunks when the SpaceManager is freed. - Metachunk* _chunks_in_use[NumberOfFreeLists]; + Metachunk* _chunks_in_use[NumberOfInUseLists]; Metachunk* _current_chunk; // Virtual space where allocation comes from. @@ -700,24 +543,6 @@ // Add chunk to the list of chunks in use void add_chunk(Metachunk* v, bool make_current); - // Debugging support - void verify_chunks_in_use_index(ChunkIndex index, Metachunk* v) { - switch (index) { - case 0: - assert(v->word_size() == SmallChunk, "Not a SmallChunk"); - break; - case 1: - assert(v->word_size() == MediumChunk, "Not a MediumChunk"); - break; - case 2: - assert(v->word_size() > MediumChunk, "Not a HumongousChunk"); - break; - default: - assert(false, "Wrong list."); - } - } - - protected: Mutex* lock() const { return _lock; } public: @@ -751,10 +576,10 @@ MetaWord* allocate(size_t word_size); // Helper for allocations - Metablock* allocate_work(size_t word_size); + MetaWord* allocate_work(size_t word_size); // Returns a block to the per manager freelist - void deallocate(MetaWord* p); + void deallocate(MetaWord* p, size_t word_size); // Based on the allocation size and a minimum chunk size, // returned chunk size (for expanding space for chunk allocation). @@ -763,7 +588,7 @@ // Called when an allocation from the current chunk fails. // Gets a new chunk (may require getting a new virtual space), // and allocates from that chunk. - Metablock* grow_and_allocate(size_t word_size); + MetaWord* grow_and_allocate(size_t word_size); // debugging support. @@ -780,6 +605,8 @@ uint const SpaceManager::_small_chunk_limit = 4; + + const char* SpaceManager::_expand_lock_name = "SpaceManager chunk allocation lock"; const int SpaceManager::_expand_lock_rank = Monitor::leaf - 1; @@ -788,39 +615,26 @@ SpaceManager::_expand_lock_name, Mutex::_allow_vm_block_flag); -#ifdef ASSERT -size_t Metablock::_overhead = - Chunk::aligned_overhead_size(sizeof(Metablock)) / BytesPerWord; -#else -size_t Metablock::_overhead = 0; -#endif size_t Metachunk::_overhead = Chunk::aligned_overhead_size(sizeof(Metachunk)) / BytesPerWord; // New blocks returned by the Metaspace are zero initialized. // We should fix the constructors to not assume this instead. Metablock* Metablock::initialize(MetaWord* p, size_t word_size) { + if (p == NULL) { + return NULL; + } + Metablock* result = (Metablock*) p; // Clear the memory Copy::fill_to_aligned_words((HeapWord*)result, word_size); #ifdef ASSERT result->set_word_size(word_size); - // Check after work size is set. - result->set_leader((void*) metaspace_allocation_leader); - result->set_trailer((void*) metaspace_allocation_trailer); #endif return result; } -void Metablock::verify() { -#ifdef ASSERT - assert(leader() == metaspace_allocation_leader && - trailer() == metaspace_allocation_trailer, - "block has been corrupted"); -#endif -} - // Metachunk methods Metachunk* Metachunk::initialize(MetaWord* ptr, size_t word_size) { @@ -843,18 +657,13 @@ } -Metablock* Metachunk::allocate(size_t word_size) { - Metablock* result = NULL; +MetaWord* Metachunk::allocate(size_t word_size) { + MetaWord* result = NULL; // If available, bump the pointer to allocate. if (free_word_size() >= word_size) { - result = Metablock::initialize(_top, word_size); + result = _top; _top = _top + word_size; } -#ifdef ASSERT - assert(result == NULL || - result->word_size() == word_size, - "Block size is not set correctly"); -#endif return result; } @@ -878,103 +687,85 @@ bottom(), top(), end(), word_size()); } +#ifdef ASSERT +void Metachunk::mangle() { + // Mangle the payload of the chunk and not the links that + // maintain list of chunks. + HeapWord* start = (HeapWord*)(bottom() + overhead()); + size_t word_size = capacity_word_size() - overhead(); + Copy::fill_to_words(start, word_size, metadata_chunk_initialize); +} +#endif // ASSERT void Metachunk::verify() { #ifdef ASSERT // Cannot walk through the blocks unless the blocks have // headers with sizes. - MetaWord* curr = bottom() + overhead(); - while (curr < top()) { - Metablock* block = (Metablock*) curr; - size_t word_size = block->word_size(); - block->verify(); - curr = curr + word_size; - } + assert(_bottom <= _top && + _top <= _end, + "Chunk has been smashed"); + assert(SpaceManager::is_humongous(_word_size) || + _word_size == SpaceManager::MediumChunk || + _word_size == SpaceManager::SmallChunk, + "Chunk size is wrong"); #endif return; } // BlockFreelist methods -#ifdef DEALLOCATE_BLOCKS BlockFreelist::BlockFreelist() : _dictionary(NULL) {} -#else -BlockFreelist::BlockFreelist() {} -#endif BlockFreelist::~BlockFreelist() { -#ifdef DEALLOCATE_BLOCKS if (_dictionary != NULL) { if (Verbose && TraceMetadataChunkAllocation) { _dictionary->print_free_lists(gclog_or_tty); } delete _dictionary; } -#endif } -Metablock* BlockFreelist::initialize_free_chunk(Metablock* block, size_t word_size) { -#ifdef DEALLOCATE_BLOCKS -#ifdef ASSERT - assert(word_size = block->word_size(), "Wrong chunk size"); -#endif - Metablock* result = block; - result->setSize(word_size); - result->linkPrev(NULL); - result->linkNext(NULL); - - return result; -#else - ShouldNotReachHere(); +Metablock* BlockFreelist::initialize_free_chunk(MetaWord* p, size_t word_size) { + Metablock* block = (Metablock*) p; + block->set_word_size(word_size); + block->set_prev(NULL); + block->set_next(NULL); + return block; -#endif } -void BlockFreelist::return_block(Metablock* block, size_t word_size) { -#ifdef ASSERT - assert(word_size = block->word_size(), "Block size is wrong");; -#endif - Metablock* free_chunk = initialize_free_chunk(block, word_size); -#ifdef DEALLOCATE_BLOCKS +void BlockFreelist::return_block(MetaWord* p, size_t word_size) { + Metablock* free_chunk = initialize_free_chunk(p, word_size); if (dictionary() == NULL) { - _dictionary = new BinaryTreeDictionary(false /* adaptive_freelists */); + _dictionary = new BlockTreeDictionary(); } - dictionary()->returnChunk(free_chunk); -#endif + dictionary()->return_chunk(free_chunk); } -Metablock* BlockFreelist::get_block(size_t word_size) { -#ifdef DEALLOCATE_BLOCKS +MetaWord* BlockFreelist::get_block(size_t word_size) { if (dictionary() == NULL) { return NULL; } - Metablock* free_chunk = - dictionary()->getChunk(word_size, FreeBlockDictionary::exactly); -#else - Metablock* free_chunk = NULL; -#endif - if (free_chunk == NULL) { + if (word_size < TreeChunk::min_size()) { + // Dark matter. Too small for dictionary. return NULL; } - assert(free_chunk->word_size() == word_size, "Size of chunk is incorrect"); - Metablock* block = Metablock::initialize((MetaWord*) free_chunk, word_size); -#ifdef ASSERT - assert(block->word_size() == word_size, "Block size is not set correctly"); -#endif - - return block; + + Metablock* free_block = + dictionary()->get_chunk(word_size, FreeBlockDictionary::exactly); + if (free_block == NULL) { + return NULL; + } + + return (MetaWord*) free_block; } void BlockFreelist::print_on(outputStream* st) const { -#ifdef DEALLOCATE_BLOCKS if (dictionary() == NULL) { return; } dictionary()->print_free_lists(st); -#else - return; -#endif } // VirtualSpaceNode methods @@ -1597,14 +1388,11 @@ Metadebug::deallocate_block_a_lot_count() % MetaDataDeallocateALotInterval == 0 ) { Metadebug::set_deallocate_block_a_lot_count(0); for (uint i = 0; i < metadata_deallocate_a_lot_block; i++) { - Metablock* dummy_block = sm->allocate_work(raw_word_size); + MetaWord* dummy_block = sm->allocate_work(raw_word_size); if (dummy_block == 0) { break; } -#ifdef ASSERT - assert(dummy_block->word_size() == raw_word_size, "Block size is not set correctly"); -#endif - sm->deallocate(dummy_block->data()); + sm->deallocate(dummy_block, raw_word_size); } } else { Metadebug::inc_deallocate_block_a_lot_count(); @@ -1784,8 +1572,8 @@ } void ChunkManager::locked_verify() { + locked_verify_free_chunks_count(); locked_verify_free_chunks_total(); - locked_verify_free_chunks_count(); } void ChunkManager::locked_print_free_chunks(outputStream* st) { @@ -1803,7 +1591,6 @@ return &_free_chunks[index]; } - // These methods that sum the free chunk lists are used in printing // methods that are used in product builds. size_t ChunkManager::sum_free_chunks() { @@ -1818,6 +1605,7 @@ result = result + list->sum_list_capacity(); } + result = result + humongous_dictionary()->total_size(); return result; } @@ -1831,6 +1619,7 @@ } count = count + list->sum_list_count(); } + count = count + humongous_dictionary()->total_free_blocks(); return count; } @@ -1875,23 +1664,24 @@ assert_lock_strong(SpaceManager::expand_lock()); locked_verify(); - ChunkList* free_list = find_free_chunks_list(word_size); - assert(free_list != NULL, "Sanity check"); - - Metachunk* chunk = free_list->head(); - debug_only(Metachunk* debug_head = chunk;) - - if (chunk == NULL) { - return NULL; - } - - Metachunk* prev_chunk = chunk; - if (chunk->word_size() == word_size) { - // Chunk is being removed from the chunks free list. - dec_free_chunks_total(chunk->capacity_word_size()); + + Metachunk* chunk = NULL; + if (!SpaceManager::is_humongous(word_size)) { + ChunkList* free_list = find_free_chunks_list(word_size); + assert(free_list != NULL, "Sanity check"); + + chunk = free_list->head(); + debug_only(Metachunk* debug_head = chunk;) + + if (chunk == NULL) { + return NULL; + } + // Remove the chunk as the head of the list. free_list->set_head(chunk->next()); chunk->set_next(NULL); + // Chunk has been removed from the chunks free list. + dec_free_chunks_total(chunk->capacity_word_size()); if (TraceMetadataChunkAllocation && Verbose) { tty->print_cr("ChunkManager::free_chunks_get: free_list " @@ -1899,79 +1689,24 @@ free_list, chunk, chunk->word_size()); } } else { - assert(SpaceManager::is_humongous(word_size), - "Should only need to check humongous"); - // This code to find the best fit is just for purposes of - // investigating the loss due to fragmentation on a humongous - // chunk. It will be replace by a binaryTreeDictionary for - // the humongous chunks. - uint count = 0; - Metachunk* best_fit = NULL; - Metachunk* best_fit_prev = NULL; - while (chunk != NULL) { - count++; - if (chunk->word_size() < word_size) { - prev_chunk = chunk; - chunk = chunk->next(); - } else if (chunk->word_size() == word_size) { - break; - } else { - if (best_fit == NULL || - best_fit->word_size() > chunk->word_size()) { - best_fit_prev = prev_chunk; - best_fit = chunk; - } - prev_chunk = chunk; - chunk = chunk->next(); - } - } - if (chunk == NULL) { - prev_chunk = best_fit_prev; - chunk = best_fit; + chunk = humongous_dictionary()->get_chunk( + word_size, + FreeBlockDictionary::atLeast); + + if (chunk != NULL) { + if (TraceMetadataHumongousAllocation) { + size_t waste = chunk->word_size() - word_size; + tty->print_cr("Free list allocate humongous chunk size " SIZE_FORMAT + " for requested size " SIZE_FORMAT + " waste " SIZE_FORMAT, + chunk->word_size(), word_size, waste); } - if (chunk != NULL) { - if (TraceMetadataHumongousAllocation) { - size_t waste = chunk->word_size() - word_size; - tty->print_cr("Free list allocate humongous chunk size " SIZE_FORMAT - " for requested size " SIZE_FORMAT - " waste " SIZE_FORMAT - " found at " SIZE_FORMAT " of " SIZE_FORMAT, - chunk->word_size(), word_size, waste, - count, free_list->sum_list_count()); - } - // Chunk is being removed from the chunks free list. - dec_free_chunks_total(chunk->capacity_word_size()); - // Remove the chunk if it is at the head of the list. - if (chunk == free_list->head()) { - free_list->set_head(chunk->next()); - - if (TraceMetadataHumongousAllocation) { - tty->print_cr("ChunkManager::free_chunks_get: humongous free_list " - PTR_FORMAT " chunk " PTR_FORMAT " size " SIZE_FORMAT - " new head " PTR_FORMAT, - free_list, chunk, chunk->word_size(), - free_list->head()); - } - } else { - // Remove a chunk in the interior of the list - prev_chunk->set_next(chunk->next()); - - if (TraceMetadataHumongousAllocation) { - tty->print_cr("ChunkManager::free_chunks_get: humongous free_list " - PTR_FORMAT " chunk " PTR_FORMAT " size " SIZE_FORMAT - PTR_FORMAT " prev " PTR_FORMAT " next " PTR_FORMAT, - free_list, chunk, chunk->word_size(), - prev_chunk, chunk->next()); - } - } - chunk->set_next(NULL); - } else { - if (TraceMetadataHumongousAllocation) { - tty->print_cr("ChunkManager::free_chunks_get: New humongous chunk of size " - SIZE_FORMAT, - word_size); - } - } + // Chunk is being removed from the chunks free list. + dec_free_chunks_total(chunk->capacity_word_size()); +#ifdef ASSERT + chunk->set_is_free(false); +#endif + } } locked_verify(); return chunk; @@ -2000,12 +1735,18 @@ return chunk; } +void ChunkManager::print_on(outputStream* out) { + if (PrintFLSStatistics != 0) { + humongous_dictionary()->report_statistics(); + } +} + // SpaceManager methods size_t SpaceManager::sum_free_in_chunks_in_use() const { MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); size_t free = 0; - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { Metachunk* chunk = chunks_in_use(i); while (chunk != NULL) { free += chunk->free_word_size(); @@ -2018,11 +1759,12 @@ size_t SpaceManager::sum_waste_in_chunks_in_use() const { MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); size_t result = 0; - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { - // Count the free space in all the chunk but not the - // current chunk from which allocations are still being done. + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { + + result += sum_waste_in_chunks_in_use(i); } + return result; } @@ -2033,10 +1775,10 @@ // Count the free space in all the chunk but not the // current chunk from which allocations are still being done. if (chunk != NULL) { - while (chunk != NULL) { - if (chunk != current_chunk()) { - result += chunk->free_word_size(); - } + Metachunk* prev = chunk; + while (chunk != NULL && chunk != current_chunk()) { + result += chunk->free_word_size(); + prev = chunk; chunk = chunk->next(); count++; } @@ -2047,7 +1789,7 @@ size_t SpaceManager::sum_capacity_in_chunks_in_use() const { MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); size_t sum = 0; - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { Metachunk* chunk = chunks_in_use(i); while (chunk != NULL) { // Just changed this sum += chunk->capacity_word_size(); @@ -2061,9 +1803,10 @@ size_t SpaceManager::sum_count_in_chunks_in_use() { size_t count = 0; - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { count = count + sum_count_in_chunks_in_use(i); } + return count; } @@ -2081,7 +1824,7 @@ size_t SpaceManager::sum_used_in_chunks_in_use() const { MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); size_t used = 0; - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { Metachunk* chunk = chunks_in_use(i); while (chunk != NULL) { used += chunk->used_word_size(); @@ -2139,15 +1882,13 @@ gclog_or_tty->print_cr(" word_size " PTR_FORMAT, word_size); gclog_or_tty->print_cr(" chunk_word_size " PTR_FORMAT, chunk_word_size); - gclog_or_tty->print_cr(" block overhead " PTR_FORMAT - " chunk overhead " PTR_FORMAT, - Metablock::overhead(), + gclog_or_tty->print_cr(" chunk overhead " PTR_FORMAT, Metachunk::overhead()); } return chunk_word_size; } -Metablock* SpaceManager::grow_and_allocate(size_t word_size) { +MetaWord* SpaceManager::grow_and_allocate(size_t word_size) { assert(vs_list()->current_virtual_space() != NULL, "Should have been set"); assert(current_chunk() == NULL || @@ -2180,7 +1921,7 @@ void SpaceManager::print_on(outputStream* st) const { for (ChunkIndex i = SmallIndex; - i < NumberOfFreeLists ; + i < NumberOfInUseLists ; i = next_chunk_index(i) ) { st->print_cr(" chunks_in_use " PTR_FORMAT " chunk size " PTR_FORMAT, chunks_in_use(i), @@ -2191,8 +1932,11 @@ sum_waste_in_chunks_in_use(SmallIndex), sum_waste_in_chunks_in_use(MediumIndex), sum_waste_in_chunks_in_use(HumongousIndex)); - // Nothing in them yet - // block_freelists()->print_on(st); + // block free lists + if (block_freelists() != NULL) { + st->print_cr("total in block free lists " SIZE_FORMAT, + block_freelists()->total_size()); + } } SpaceManager::SpaceManager(Mutex* lock, VirtualSpaceList* vs_list) : @@ -2200,7 +1944,7 @@ _allocation_total(0), _lock(lock) { Metadebug::init_allocation_fail_alot_count(); - for (ChunkIndex i = SmallIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = SmallIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { _chunks_in_use[i] = NULL; } _current_chunk = NULL; @@ -2262,22 +2006,24 @@ // Humongous chunks are never the current chunk. Metachunk* humongous_chunks = chunks_in_use(HumongousIndex); - if (humongous_chunks != NULL) { - chunk_manager->free_humongous_chunks()->add_at_head(humongous_chunks); - set_chunks_in_use(HumongousIndex, NULL); + while (humongous_chunks != NULL) { +#ifdef ASSERT + humongous_chunks->set_is_free(true); +#endif + Metachunk* next_humongous_chunks = humongous_chunks->next(); + chunk_manager->humongous_dictionary()->return_chunk(humongous_chunks); + humongous_chunks = next_humongous_chunks; } + set_chunks_in_use(HumongousIndex, NULL); chunk_manager->locked_verify(); } -void SpaceManager::deallocate(MetaWord* p) { +void SpaceManager::deallocate(MetaWord* p, size_t word_size) { assert_lock_strong(_lock); - ShouldNotReachHere(); // Where is this needed. -#ifdef DEALLOCATE_BLOCKS - Metablock* block = Metablock::metablock_from_data(p); - // This is expense but kept it until integration JJJ - assert(contains((address)block), "Block does not belong to this metaspace"); - block_freelists()->return_block(block, word_size); -#endif + size_t min_size = TreeChunk::min_size(); + assert(word_size >= min_size, + err_msg("Should not deallocate dark matter " SIZE_FORMAT, word_size)); + block_freelists()->return_block(p, word_size); } // Adds a chunk to the list of chunks in use. @@ -2366,50 +2112,40 @@ MetaWord* SpaceManager::allocate(size_t word_size) { MutexLockerEx cl(lock(), Mutex::_no_safepoint_check_flag); - size_t block_overhead = Metablock::overhead(); // If only the dictionary is going to be used (i.e., no // indexed free list), then there is a minimum size requirement. // MinChunkSize is a placeholder for the real minimum size JJJ - size_t byte_size_with_overhead = (word_size + block_overhead) * BytesPerWord; -#ifdef DEALLOCATE_BLOCKS - size_t raw_bytes_size = MAX2(ARENA_ALIGN(byte_size_with_overhead), - MinChunkSize * BytesPerWord); -#else - size_t raw_bytes_size = ARENA_ALIGN(byte_size_with_overhead); -#endif + size_t byte_size = word_size * BytesPerWord; + + size_t byte_size_with_overhead = byte_size + Metablock::overhead(); + + size_t raw_bytes_size = MAX2(byte_size_with_overhead, + Metablock::min_block_byte_size()); + raw_bytes_size = ARENA_ALIGN(raw_bytes_size); size_t raw_word_size = raw_bytes_size / BytesPerWord; assert(raw_word_size * BytesPerWord == raw_bytes_size, "Size problem"); BlockFreelist* fl = block_freelists(); - Metablock* block = NULL; + MetaWord* p = NULL; // Allocation from the dictionary is expensive in the sense that // the dictionary has to be searched for a size. Don't allocate // from the dictionary until it starts to get fat. Is this // a reasonable policy? Maybe an skinny dictionary is fast enough // for allocations. Do some profiling. JJJ - if (fl->totalSize() > allocation_from_dictionary_limit) { - block = fl->get_block(raw_word_size); + if (fl->total_size() > allocation_from_dictionary_limit) { + p = fl->get_block(raw_word_size); } - if (block == NULL) { - block = allocate_work(raw_word_size); - if (block == NULL) { - return NULL; - } + if (p == NULL) { + p = allocate_work(raw_word_size); } Metadebug::deallocate_block_a_lot(this, raw_word_size); - // Push the allocation past the word containing the size and leader. -#ifdef ASSERT - MetaWord* result = block->data(); - return result; -#else - return (MetaWord*) block; -#endif + return p; } // Returns the address of spaced allocated for "word_size". // This methods does not know about blocks (Metablocks) -Metablock* SpaceManager::allocate_work(size_t word_size) { +MetaWord* SpaceManager::allocate_work(size_t word_size) { assert_lock_strong(_lock); #ifdef ASSERT if (Metadebug::test_metadata_failure()) { @@ -2417,7 +2153,7 @@ } #endif // Is there space in the current chunk? - Metablock* result = NULL; + MetaWord* result = NULL; // For DumpSharedSpaces, only allocate out of the current chunk which is // never null because we gave it the size we wanted. Caller reports out @@ -2436,8 +2172,8 @@ } if (result > 0) { inc_allocation_total(word_size); - assert(result != (Metablock*) chunks_in_use(MediumIndex), "Head of the list is being allocated"); - assert(result->word_size() == word_size, "Size not set correctly"); + assert(result != (MetaWord*) chunks_in_use(MediumIndex), + "Head of the list is being allocated"); } return result; @@ -2447,13 +2183,13 @@ // If there are blocks in the dictionary, then // verfication of chunks does not work since // being in the dictionary alters a chunk. - if (block_freelists()->totalSize() == 0) { + if (block_freelists()->total_size() == 0) { // Skip the small chunks because their next link points to // medium chunks. This is because the small chunk is the // current chunk (for allocations) until it is full and the // the addition of the next chunk does not NULL the next // like of the small chunk. - for (ChunkIndex i = MediumIndex; i < NumberOfFreeLists; i = next_chunk_index(i)) { + for (ChunkIndex i = MediumIndex; i < NumberOfInUseLists; i = next_chunk_index(i)) { Metachunk* curr = chunks_in_use(i); while (curr != NULL) { curr->verify(); @@ -2492,7 +2228,7 @@ // Add up statistics for all chunks in this SpaceManager. for (ChunkIndex index = SmallIndex; - index < NumberOfFreeLists; + index < NumberOfInUseLists; index = next_chunk_index(index)) { for (Metachunk* curr = chunks_in_use(index); curr != NULL; @@ -2521,7 +2257,7 @@ #ifdef ASSERT void SpaceManager::mangle_freed_chunks() { for (ChunkIndex index = SmallIndex; - index < NumberOfFreeLists; + index < NumberOfInUseLists; index = next_chunk_index(index)) { for (Metachunk* curr = chunks_in_use(index); curr != NULL; @@ -2833,13 +2569,12 @@ } } - MetaWord* Metaspace::allocate(size_t word_size, MetadataType mdtype) { // DumpSharedSpaces doesn't use class metadata area (yet) if (mdtype == ClassType && !DumpSharedSpaces) { - return class_vsm()->allocate(word_size); + return class_vsm()->allocate(word_size); } else { - return vsm()->allocate(word_size); + return vsm()->allocate(word_size); } } @@ -2853,6 +2588,7 @@ gclog_or_tty->print_cr("Increase capacity to GC from " SIZE_FORMAT " to " SIZE_FORMAT, before_inc, MetaspaceGC::capacity_until_GC()); } + result = allocate(word_size, mdtype); return result; @@ -2889,37 +2625,39 @@ void Metaspace::deallocate(MetaWord* ptr, size_t word_size, bool is_class) { if (SafepointSynchronize::is_at_safepoint()) { assert(Thread::current()->is_VM_thread(), "should be the VM thread"); - // Don't take lock -#ifdef DEALLOCATE_BLOCKS - if (is_class) { - class_vsm()->deallocate(ptr); - } else { - vsm()->deallocate(ptr); + // Don't take Heap_lock + MutexLocker ml(vsm()->lock()); + if (word_size < TreeChunk::min_size()) { + // Dark matter. Too small for dictionary. +#ifdef ASSERT + Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5); +#endif + return; } -#else -#ifdef ASSERT - Copy::fill_to_words((HeapWord*)ptr, word_size, metadata_deallocate); -#endif -#endif - + if (is_class) { + class_vsm()->deallocate(ptr, word_size); + } else { + vsm()->deallocate(ptr, word_size); + } } else { MutexLocker ml(vsm()->lock()); -#ifdef DEALLOCATE_BLOCKS - if (is_class) { - class_vsm()->deallocate(ptr); - } else { - vsm()->deallocate(ptr); + if (word_size < TreeChunk::min_size()) { + // Dark matter. Too small for dictionary. +#ifdef ASSERT + Copy::fill_to_words((HeapWord*)ptr, word_size, 0xf5f5f5f5); +#endif + return; } -#else -#ifdef ASSERT - Copy::fill_to_words((HeapWord*)ptr, word_size, metadata_deallocate); -#endif -#endif + if (is_class) { + class_vsm()->deallocate(ptr, word_size); + } else { + vsm()->deallocate(ptr, word_size); + } } } -MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, +Metablock* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, bool read_only, MetadataType mdtype, TRAPS) { if (HAS_PENDING_EXCEPTION) { assert(false, "Should not allocate with exception pending"); @@ -2943,7 +2681,7 @@ if (result == NULL) { report_out_of_shared_space(read_only ? SharedReadOnly : SharedReadWrite); } - return result; + return Metablock::initialize(result, word_size); } result = loader_data->metaspace_non_null()->allocate(word_size, mdtype); @@ -2951,7 +2689,7 @@ if (result == NULL) { // Try to clean out some memory and retry. result = - Universe::heap()->collector_policy()->satisfy_failed_metadata_allocation( + Universe::heap()->collector_policy()->satisfy_failed_metadata_allocation( loader_data, word_size, mdtype); // If result is still null, we are out of memory. @@ -2967,7 +2705,7 @@ THROW_OOP_0(Universe::out_of_memory_error_perm_gen()); } } - return result; + return Metablock::initialize(result, word_size); } void Metaspace::print_on(outputStream* out) const { diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/memory/metaspace.hpp --- a/hotspot/src/share/vm/memory/metaspace.hpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/memory/metaspace.hpp Tue Sep 18 23:35:42 2012 -0700 @@ -57,12 +57,10 @@ // class ClassLoaderData; +class Metablock; class MetaWord; class Mutex; class outputStream; -class FreeChunk; -template class FreeList; -template class BinaryTreeDictionary; class SpaceManager; // Metaspaces each have a SpaceManager and allocations @@ -128,7 +126,7 @@ size_t capacity_words(MetadataType mdtype) const; size_t waste_words(MetadataType mdtype) const; - static MetaWord* allocate(ClassLoaderData* loader_data, size_t size, + static Metablock* allocate(ClassLoaderData* loader_data, size_t size, bool read_only, MetadataType mdtype, TRAPS); void deallocate(MetaWord* ptr, size_t byte_size, bool is_class); diff -r f5e31fb61738 -r 944e56f74fba hotspot/src/share/vm/runtime/vmStructs.cpp --- a/hotspot/src/share/vm/runtime/vmStructs.cpp Fri Oct 19 11:26:17 2012 -0700 +++ b/hotspot/src/share/vm/runtime/vmStructs.cpp Tue Sep 18 23:35:42 2012 -0700 @@ -59,6 +59,7 @@ #include "memory/generation.hpp" #include "memory/generationSpec.hpp" #include "memory/heap.hpp" +#include "memory/metablock.hpp" #include "memory/space.hpp" #include "memory/tenuredGeneration.hpp" #include "memory/universe.hpp" @@ -249,6 +250,7 @@ typedef Hashtable KlassHashtable; typedef HashtableEntry KlassHashtableEntry; typedef TwoOopHashtable SymbolTwoOopHashtable; +typedef BinaryTreeDictionary MetablockTreeDictionary; //-------------------------------------------------------------------------------- // VM_STRUCTS @@ -1237,7 +1239,15 @@ nonstatic_field(AccessFlags, _flags, jint) \ nonstatic_field(elapsedTimer, _counter, jlong) \ nonstatic_field(elapsedTimer, _active, bool) \ - nonstatic_field(InvocationCounter, _counter, unsigned int) + nonstatic_field(InvocationCounter, _counter, unsigned int) \ + volatile_nonstatic_field(FreeChunk, _size, size_t) \ + nonstatic_field(FreeChunk, _next, FreeChunk*) \ + nonstatic_field(FreeChunk, _prev, FreeChunk*) \ + nonstatic_field(FreeList, _size, size_t) \ + nonstatic_field(FreeList, _size, size_t) \ + nonstatic_field(FreeList, _count, ssize_t) \ + nonstatic_field(FreeList, _count, ssize_t) \ + nonstatic_field(MetablockTreeDictionary, _total_size, size_t) /* NOTE that we do not use the last_entry() macro here; it is used */ /* in vmStructs__.hpp's VM_STRUCTS_OS_CPU macro (and must */ @@ -2080,7 +2090,24 @@ declare_toplevel_type(Universe) \ declare_toplevel_type(vframeArray) \ declare_toplevel_type(vframeArrayElement) \ - declare_toplevel_type(Annotations*) + declare_toplevel_type(Annotations*) \ + \ + /***************/ \ + /* Miscellaneous types */ \ + /***************/ \ + \ + /* freelist */ \ + declare_toplevel_type(FreeChunk*) \ + declare_toplevel_type(Metablock*) \ + declare_toplevel_type(FreeBlockDictionary*) \ + declare_toplevel_type(FreeList*) \ + declare_toplevel_type(FreeList) \ + declare_toplevel_type(FreeBlockDictionary*) \ + declare_toplevel_type(FreeList*) \ + declare_toplevel_type(FreeList) \ + declare_toplevel_type(MetablockTreeDictionary*) \ + declare_type(MetablockTreeDictionary, FreeBlockDictionary) \ + declare_type(MetablockTreeDictionary, FreeBlockDictionary) /* NOTE that we do not use the last_entry() macro here; it is used */