diff -r 351481a5cf31 -r 6182ca66bc7b hotspot/src/share/vm/memory/binaryTreeDictionary.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/hotspot/src/share/vm/memory/binaryTreeDictionary.cpp Thu Mar 29 19:46:24 2012 -0700 @@ -0,0 +1,1343 @@ +/* + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc_implementation/shared/allocationStats.hpp" +#include "memory/binaryTreeDictionary.hpp" +#include "runtime/globals.hpp" +#include "utilities/ostream.hpp" +#ifndef SERIALGC +#include "gc_implementation/shared/spaceDecorator.hpp" +#include "gc_implementation/concurrentMarkSweep/freeChunk.hpp" +#endif // SERIALGC + +//////////////////////////////////////////////////////////////////////////////// +// A binary tree based search structure for free blocks. +// This is currently used in the Concurrent Mark&Sweep implementation. +//////////////////////////////////////////////////////////////////////////////// + +template +TreeChunk* TreeChunk::as_TreeChunk(Chunk* fc) { + // Do some assertion checking here. + return (TreeChunk*) fc; +} + +template +void TreeChunk::verifyTreeChunkList() const { + TreeChunk* nextTC = (TreeChunk*)next(); + if (prev() != NULL) { // interior list node shouldn'r have tree fields + guarantee(embedded_list()->parent() == NULL && embedded_list()->left() == NULL && + embedded_list()->right() == NULL, "should be clear"); + } + if (nextTC != NULL) { + guarantee(as_TreeChunk(nextTC->prev()) == this, "broken chain"); + guarantee(nextTC->size() == size(), "wrong size"); + nextTC->verifyTreeChunkList(); + } +} + + +template +TreeList* TreeList::as_TreeList(TreeChunk* tc) { + // This first free chunk in the list will be the tree list. + assert(tc->size() >= BinaryTreeDictionary::min_tree_chunk_size, "Chunk is too small for a TreeChunk"); + TreeList* tl = tc->embedded_list(); + tc->set_list(tl); +#ifdef ASSERT + tl->set_protecting_lock(NULL); +#endif + tl->set_hint(0); + tl->set_size(tc->size()); + tl->link_head(tc); + tl->link_tail(tc); + tl->set_count(1); + tl->init_statistics(true /* split_birth */); + tl->setParent(NULL); + tl->setLeft(NULL); + tl->setRight(NULL); + return tl; +} + +template +TreeList* TreeList::as_TreeList(HeapWord* addr, size_t size) { + TreeChunk* tc = (TreeChunk*) addr; + assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "Chunk is too small for a TreeChunk"); + // The space in the heap will have been mangled initially but + // is not remangled when a free chunk is returned to the free list + // (since it is used to maintain the chunk on the free list). + assert((ZapUnusedHeapArea && + SpaceMangler::is_mangled((HeapWord*) tc->size_addr()) && + SpaceMangler::is_mangled((HeapWord*) tc->prev_addr()) && + SpaceMangler::is_mangled((HeapWord*) tc->next_addr())) || + (tc->size() == 0 && tc->prev() == NULL && tc->next() == NULL), + "Space should be clear or mangled"); + tc->setSize(size); + tc->linkPrev(NULL); + tc->linkNext(NULL); + TreeList* tl = TreeList::as_TreeList(tc); + return tl; +} + +template +TreeList* TreeList::removeChunkReplaceIfNeeded(TreeChunk* tc) { + + TreeList* retTL = this; + Chunk* list = head(); + assert(!list || list != list->next(), "Chunk on list twice"); + assert(tc != NULL, "Chunk being removed is NULL"); + assert(parent() == NULL || this == parent()->left() || + this == parent()->right(), "list is inconsistent"); + assert(tc->isFree(), "Header is not marked correctly"); + assert(head() == NULL || head()->prev() == NULL, "list invariant"); + assert(tail() == NULL || tail()->next() == NULL, "list invariant"); + + Chunk* prevFC = tc->prev(); + TreeChunk* nextTC = TreeChunk::as_TreeChunk(tc->next()); + assert(list != NULL, "should have at least the target chunk"); + + // Is this the first item on the list? + if (tc == list) { + // The "getChunk..." functions for a TreeList will not return the + // first chunk in the list unless it is the last chunk in the list + // because the first chunk is also acting as the tree node. + // When coalescing happens, however, the first chunk in the a tree + // list can be the start of a free range. Free ranges are removed + // from the free lists so that they are not available to be + // allocated when the sweeper yields (giving up the free list lock) + // to allow mutator activity. If this chunk is the first in the + // list and is not the last in the list, do the work to copy the + // TreeList from the first chunk to the next chunk and update all + // the TreeList pointers in the chunks in the list. + if (nextTC == NULL) { + assert(prevFC == NULL, "Not last chunk in the list"); + set_tail(NULL); + set_head(NULL); + } else { + // copy embedded list. + nextTC->set_embedded_list(tc->embedded_list()); + retTL = nextTC->embedded_list(); + // Fix the pointer to the list in each chunk in the list. + // This can be slow for a long list. Consider having + // an option that does not allow the first chunk on the + // list to be coalesced. + for (TreeChunk* curTC = nextTC; curTC != NULL; + curTC = TreeChunk::as_TreeChunk(curTC->next())) { + curTC->set_list(retTL); + } + // Fix the parent to point to the new TreeList. + if (retTL->parent() != NULL) { + if (this == retTL->parent()->left()) { + retTL->parent()->setLeft(retTL); + } else { + assert(this == retTL->parent()->right(), "Parent is incorrect"); + retTL->parent()->setRight(retTL); + } + } + // Fix the children's parent pointers to point to the + // new list. + assert(right() == retTL->right(), "Should have been copied"); + if (retTL->right() != NULL) { + retTL->right()->setParent(retTL); + } + assert(left() == retTL->left(), "Should have been copied"); + if (retTL->left() != NULL) { + retTL->left()->setParent(retTL); + } + retTL->link_head(nextTC); + assert(nextTC->isFree(), "Should be a free chunk"); + } + } else { + if (nextTC == NULL) { + // Removing chunk at tail of list + link_tail(prevFC); + } + // Chunk is interior to the list + prevFC->linkAfter(nextTC); + } + + // Below this point the embeded TreeList being used for the + // tree node may have changed. Don't use "this" + // TreeList*. + // chunk should still be a free chunk (bit set in _prev) + assert(!retTL->head() || retTL->size() == retTL->head()->size(), + "Wrong sized chunk in list"); + debug_only( + tc->linkPrev(NULL); + tc->linkNext(NULL); + tc->set_list(NULL); + bool prev_found = false; + bool next_found = false; + for (Chunk* curFC = retTL->head(); + curFC != NULL; curFC = curFC->next()) { + assert(curFC != tc, "Chunk is still in list"); + if (curFC == prevFC) { + prev_found = true; + } + if (curFC == nextTC) { + next_found = true; + } + } + assert(prevFC == NULL || prev_found, "Chunk was lost from list"); + assert(nextTC == NULL || next_found, "Chunk was lost from list"); + assert(retTL->parent() == NULL || + retTL == retTL->parent()->left() || + retTL == retTL->parent()->right(), + "list is inconsistent"); + ) + retTL->decrement_count(); + + assert(tc->isFree(), "Should still be a free chunk"); + assert(retTL->head() == NULL || retTL->head()->prev() == NULL, + "list invariant"); + assert(retTL->tail() == NULL || retTL->tail()->next() == NULL, + "list invariant"); + return retTL; +} + +template +void TreeList::returnChunkAtTail(TreeChunk* chunk) { + assert(chunk != NULL, "returning NULL chunk"); + assert(chunk->list() == this, "list should be set for chunk"); + assert(tail() != NULL, "The tree list is embedded in the first chunk"); + // which means that the list can never be empty. + assert(!verifyChunkInFreeLists(chunk), "Double entry"); + assert(head() == NULL || head()->prev() == NULL, "list invariant"); + assert(tail() == NULL || tail()->next() == NULL, "list invariant"); + + Chunk* fc = tail(); + fc->linkAfter(chunk); + link_tail(chunk); + + assert(!tail() || size() == tail()->size(), "Wrong sized chunk in list"); + FreeList::increment_count(); + debug_only(increment_returnedBytes_by(chunk->size()*sizeof(HeapWord));) + assert(head() == NULL || head()->prev() == NULL, "list invariant"); + assert(tail() == NULL || tail()->next() == NULL, "list invariant"); +} + +// Add this chunk at the head of the list. "At the head of the list" +// is defined to be after the chunk pointer to by head(). This is +// because the TreeList is embedded in the first TreeChunk in the +// list. See the definition of TreeChunk. +template +void TreeList::returnChunkAtHead(TreeChunk* chunk) { + assert(chunk->list() == this, "list should be set for chunk"); + assert(head() != NULL, "The tree list is embedded in the first chunk"); + assert(chunk != NULL, "returning NULL chunk"); + assert(!verifyChunkInFreeLists(chunk), "Double entry"); + assert(head() == NULL || head()->prev() == NULL, "list invariant"); + assert(tail() == NULL || tail()->next() == NULL, "list invariant"); + + Chunk* fc = head()->next(); + if (fc != NULL) { + chunk->linkAfter(fc); + } else { + assert(tail() == NULL, "List is inconsistent"); + link_tail(chunk); + } + head()->linkAfter(chunk); + assert(!head() || size() == head()->size(), "Wrong sized chunk in list"); + FreeList::increment_count(); + debug_only(increment_returnedBytes_by(chunk->size()*sizeof(HeapWord));) + assert(head() == NULL || head()->prev() == NULL, "list invariant"); + assert(tail() == NULL || tail()->next() == NULL, "list invariant"); +} + +template +TreeChunk* TreeList::head_as_TreeChunk() { + assert(head() == NULL || TreeChunk::as_TreeChunk(head())->list() == this, + "Wrong type of chunk?"); + return TreeChunk::as_TreeChunk(head()); +} + +template +TreeChunk* TreeList::first_available() { + assert(head() != NULL, "The head of the list cannot be NULL"); + Chunk* fc = head()->next(); + TreeChunk* retTC; + if (fc == NULL) { + retTC = head_as_TreeChunk(); + } else { + retTC = TreeChunk::as_TreeChunk(fc); + } + assert(retTC->list() == this, "Wrong type of chunk."); + return retTC; +} + +// Returns the block with the largest heap address amongst +// those in the list for this size; potentially slow and expensive, +// use with caution! +template +TreeChunk* TreeList::largest_address() { + assert(head() != NULL, "The head of the list cannot be NULL"); + Chunk* fc = head()->next(); + TreeChunk* retTC; + if (fc == NULL) { + retTC = head_as_TreeChunk(); + } else { + // walk down the list and return the one with the highest + // heap address among chunks of this size. + Chunk* last = fc; + while (fc->next() != NULL) { + if ((HeapWord*)last < (HeapWord*)fc) { + last = fc; + } + fc = fc->next(); + } + retTC = TreeChunk::as_TreeChunk(last); + } + assert(retTC->list() == this, "Wrong type of chunk."); + return retTC; +} + +template +BinaryTreeDictionary::BinaryTreeDictionary(bool adaptive_freelists, bool splay) : + _splay(splay), _adaptive_freelists(adaptive_freelists), + _totalSize(0), _totalFreeBlocks(0), _root(0) {} + +template +BinaryTreeDictionary::BinaryTreeDictionary(MemRegion mr, + bool adaptive_freelists, + bool splay): + _adaptive_freelists(adaptive_freelists), _splay(splay) +{ + assert(mr.word_size() >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); + + reset(mr); + assert(root()->left() == NULL, "reset check failed"); + assert(root()->right() == NULL, "reset check failed"); + assert(root()->head()->next() == NULL, "reset check failed"); + assert(root()->head()->prev() == NULL, "reset check failed"); + assert(totalSize() == root()->size(), "reset check failed"); + assert(totalFreeBlocks() == 1, "reset check failed"); +} + +template +void BinaryTreeDictionary::inc_totalSize(size_t inc) { + _totalSize = _totalSize + inc; +} + +template +void BinaryTreeDictionary::dec_totalSize(size_t dec) { + _totalSize = _totalSize - dec; +} + +template +void BinaryTreeDictionary::reset(MemRegion mr) { + assert(mr.word_size() >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); + set_root(TreeList::as_TreeList(mr.start(), mr.word_size())); + set_totalSize(mr.word_size()); + set_totalFreeBlocks(1); +} + +template +void BinaryTreeDictionary::reset(HeapWord* addr, size_t byte_size) { + MemRegion mr(addr, heap_word_size(byte_size)); + reset(mr); +} + +template +void BinaryTreeDictionary::reset() { + set_root(NULL); + set_totalSize(0); + set_totalFreeBlocks(0); +} + +// Get a free block of size at least size from tree, or NULL. +// If a splay step is requested, the removal algorithm (only) incorporates +// a splay step as follows: +// . the search proceeds down the tree looking for a possible +// match. At the (closest) matching location, an appropriate splay step is applied +// (zig, zig-zig or zig-zag). A chunk of the appropriate size is then returned +// if available, and if it's the last chunk, the node is deleted. A deteleted +// node is replaced in place by its tree successor. +template +TreeChunk* +BinaryTreeDictionary::getChunkFromTree(size_t size, enum FreeBlockDictionary::Dither dither, bool splay) +{ + TreeList *curTL, *prevTL; + TreeChunk* retTC = NULL; + assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "minimum chunk size"); + if (FLSVerifyDictionary) { + verifyTree(); + } + // starting at the root, work downwards trying to find match. + // Remember the last node of size too great or too small. + for (prevTL = curTL = root(); curTL != NULL;) { + if (curTL->size() == size) { // exact match + break; + } + prevTL = curTL; + if (curTL->size() < size) { // proceed to right sub-tree + curTL = curTL->right(); + } else { // proceed to left sub-tree + assert(curTL->size() > size, "size inconsistency"); + curTL = curTL->left(); + } + } + if (curTL == NULL) { // couldn't find exact match + + if (dither == FreeBlockDictionary::exactly) return NULL; + + // try and find the next larger size by walking back up the search path + for (curTL = prevTL; curTL != NULL;) { + if (curTL->size() >= size) break; + else curTL = curTL->parent(); + } + assert(curTL == NULL || curTL->count() > 0, + "An empty list should not be in the tree"); + } + if (curTL != NULL) { + assert(curTL->size() >= size, "size inconsistency"); + if (adaptive_freelists()) { + + // A candidate chunk has been found. If it is already under + // populated, get a chunk associated with the hint for this + // chunk. + if (curTL->surplus() <= 0) { + /* Use the hint to find a size with a surplus, and reset the hint. */ + TreeList* hintTL = curTL; + while (hintTL->hint() != 0) { + assert(hintTL->hint() == 0 || hintTL->hint() > hintTL->size(), + "hint points in the wrong direction"); + hintTL = findList(hintTL->hint()); + assert(curTL != hintTL, "Infinite loop"); + if (hintTL == NULL || + hintTL == curTL /* Should not happen but protect against it */ ) { + // No useful hint. Set the hint to NULL and go on. + curTL->set_hint(0); + break; + } + assert(hintTL->size() > size, "hint is inconsistent"); + if (hintTL->surplus() > 0) { + // The hint led to a list that has a surplus. Use it. + // Set the hint for the candidate to an overpopulated + // size. + curTL->set_hint(hintTL->size()); + // Change the candidate. + curTL = hintTL; + break; + } + // The evm code reset the hint of the candidate as + // at an interim point. Why? Seems like this leaves + // the hint pointing to a list that didn't work. + // curTL->set_hint(hintTL->size()); + } + } + } + // don't waste time splaying if chunk's singleton + if (splay && curTL->head()->next() != NULL) { + semiSplayStep(curTL); + } + retTC = curTL->first_available(); + assert((retTC != NULL) && (curTL->count() > 0), + "A list in the binary tree should not be NULL"); + assert(retTC->size() >= size, + "A chunk of the wrong size was found"); + removeChunkFromTree(retTC); + assert(retTC->isFree(), "Header is not marked correctly"); + } + + if (FLSVerifyDictionary) { + verify(); + } + return retTC; +} + +template +TreeList* BinaryTreeDictionary::findList(size_t size) const { + TreeList* curTL; + for (curTL = root(); curTL != NULL;) { + if (curTL->size() == size) { // exact match + break; + } + + if (curTL->size() < size) { // proceed to right sub-tree + curTL = curTL->right(); + } else { // proceed to left sub-tree + assert(curTL->size() > size, "size inconsistency"); + curTL = curTL->left(); + } + } + return curTL; +} + + +template +bool BinaryTreeDictionary::verifyChunkInFreeLists(Chunk* tc) const { + size_t size = tc->size(); + TreeList* tl = findList(size); + if (tl == NULL) { + return false; + } else { + return tl->verifyChunkInFreeLists(tc); + } +} + +template +Chunk* BinaryTreeDictionary::findLargestDict() const { + TreeList *curTL = root(); + if (curTL != NULL) { + while(curTL->right() != NULL) curTL = curTL->right(); + return curTL->largest_address(); + } else { + return NULL; + } +} + +// Remove the current chunk from the tree. If it is not the last +// chunk in a list on a tree node, just unlink it. +// If it is the last chunk in the list (the next link is NULL), +// remove the node and repair the tree. +template +TreeChunk* +BinaryTreeDictionary::removeChunkFromTree(TreeChunk* tc) { + assert(tc != NULL, "Should not call with a NULL chunk"); + assert(tc->isFree(), "Header is not marked correctly"); + + TreeList *newTL, *parentTL; + TreeChunk* retTC; + TreeList* tl = tc->list(); + debug_only( + bool removing_only_chunk = false; + if (tl == _root) { + if ((_root->left() == NULL) && (_root->right() == NULL)) { + if (_root->count() == 1) { + assert(_root->head() == tc, "Should only be this one chunk"); + removing_only_chunk = true; + } + } + } + ) + assert(tl != NULL, "List should be set"); + assert(tl->parent() == NULL || tl == tl->parent()->left() || + tl == tl->parent()->right(), "list is inconsistent"); + + bool complicatedSplice = false; + + retTC = tc; + // Removing this chunk can have the side effect of changing the node + // (TreeList*) in the tree. If the node is the root, update it. + TreeList* replacementTL = tl->removeChunkReplaceIfNeeded(tc); + assert(tc->isFree(), "Chunk should still be free"); + assert(replacementTL->parent() == NULL || + replacementTL == replacementTL->parent()->left() || + replacementTL == replacementTL->parent()->right(), + "list is inconsistent"); + if (tl == root()) { + assert(replacementTL->parent() == NULL, "Incorrectly replacing root"); + set_root(replacementTL); + } + debug_only( + if (tl != replacementTL) { + assert(replacementTL->head() != NULL, + "If the tree list was replaced, it should not be a NULL list"); + TreeList* rhl = replacementTL->head_as_TreeChunk()->list(); + TreeList* rtl = TreeChunk::as_TreeChunk(replacementTL->tail())->list(); + assert(rhl == replacementTL, "Broken head"); + assert(rtl == replacementTL, "Broken tail"); + assert(replacementTL->size() == tc->size(), "Broken size"); + } + ) + + // Does the tree need to be repaired? + if (replacementTL->count() == 0) { + assert(replacementTL->head() == NULL && + replacementTL->tail() == NULL, "list count is incorrect"); + // Find the replacement node for the (soon to be empty) node being removed. + // if we have a single (or no) child, splice child in our stead + if (replacementTL->left() == NULL) { + // left is NULL so pick right. right may also be NULL. + newTL = replacementTL->right(); + debug_only(replacementTL->clearRight();) + } else if (replacementTL->right() == NULL) { + // right is NULL + newTL = replacementTL->left(); + debug_only(replacementTL->clearLeft();) + } else { // we have both children, so, by patriarchal convention, + // my replacement is least node in right sub-tree + complicatedSplice = true; + newTL = removeTreeMinimum(replacementTL->right()); + assert(newTL != NULL && newTL->left() == NULL && + newTL->right() == NULL, "sub-tree minimum exists"); + } + // newTL is the replacement for the (soon to be empty) node. + // newTL may be NULL. + // should verify; we just cleanly excised our replacement + if (FLSVerifyDictionary) { + verifyTree(); + } + // first make newTL my parent's child + if ((parentTL = replacementTL->parent()) == NULL) { + // newTL should be root + assert(tl == root(), "Incorrectly replacing root"); + set_root(newTL); + if (newTL != NULL) { + newTL->clearParent(); + } + } else if (parentTL->right() == replacementTL) { + // replacementTL is a right child + parentTL->setRight(newTL); + } else { // replacementTL is a left child + assert(parentTL->left() == replacementTL, "should be left child"); + parentTL->setLeft(newTL); + } + debug_only(replacementTL->clearParent();) + if (complicatedSplice) { // we need newTL to get replacementTL's + // two children + assert(newTL != NULL && + newTL->left() == NULL && newTL->right() == NULL, + "newTL should not have encumbrances from the past"); + // we'd like to assert as below: + // assert(replacementTL->left() != NULL && replacementTL->right() != NULL, + // "else !complicatedSplice"); + // ... however, the above assertion is too strong because we aren't + // guaranteed that replacementTL->right() is still NULL. + // Recall that we removed + // the right sub-tree minimum from replacementTL. + // That may well have been its right + // child! So we'll just assert half of the above: + assert(replacementTL->left() != NULL, "else !complicatedSplice"); + newTL->setLeft(replacementTL->left()); + newTL->setRight(replacementTL->right()); + debug_only( + replacementTL->clearRight(); + replacementTL->clearLeft(); + ) + } + assert(replacementTL->right() == NULL && + replacementTL->left() == NULL && + replacementTL->parent() == NULL, + "delete without encumbrances"); + } + + assert(totalSize() >= retTC->size(), "Incorrect total size"); + dec_totalSize(retTC->size()); // size book-keeping + assert(totalFreeBlocks() > 0, "Incorrect total count"); + set_totalFreeBlocks(totalFreeBlocks() - 1); + + assert(retTC != NULL, "null chunk?"); + assert(retTC->prev() == NULL && retTC->next() == NULL, + "should return without encumbrances"); + if (FLSVerifyDictionary) { + verifyTree(); + } + assert(!removing_only_chunk || _root == NULL, "root should be NULL"); + return TreeChunk::as_TreeChunk(retTC); +} + +// Remove the leftmost node (lm) in the tree and return it. +// If lm has a right child, link it to the left node of +// the parent of lm. +template +TreeList* BinaryTreeDictionary::removeTreeMinimum(TreeList* tl) { + assert(tl != NULL && tl->parent() != NULL, "really need a proper sub-tree"); + // locate the subtree minimum by walking down left branches + TreeList* curTL = tl; + for (; curTL->left() != NULL; curTL = curTL->left()); + // obviously curTL now has at most one child, a right child + if (curTL != root()) { // Should this test just be removed? + TreeList* parentTL = curTL->parent(); + if (parentTL->left() == curTL) { // curTL is a left child + parentTL->setLeft(curTL->right()); + } else { + // If the list tl has no left child, then curTL may be + // the right child of parentTL. + assert(parentTL->right() == curTL, "should be a right child"); + parentTL->setRight(curTL->right()); + } + } else { + // The only use of this method would not pass the root of the + // tree (as indicated by the assertion above that the tree list + // has a parent) but the specification does not explicitly exclude the + // passing of the root so accomodate it. + set_root(NULL); + } + debug_only( + curTL->clearParent(); // Test if this needs to be cleared + curTL->clearRight(); // recall, above, left child is already null + ) + // we just excised a (non-root) node, we should still verify all tree invariants + if (FLSVerifyDictionary) { + verifyTree(); + } + return curTL; +} + +// Based on a simplification of the algorithm by Sleator and Tarjan (JACM 1985). +// The simplifications are the following: +// . we splay only when we delete (not when we insert) +// . we apply a single spay step per deletion/access +// By doing such partial splaying, we reduce the amount of restructuring, +// while getting a reasonably efficient search tree (we think). +// [Measurements will be needed to (in)validate this expectation.] + +template +void BinaryTreeDictionary::semiSplayStep(TreeList* tc) { + // apply a semi-splay step at the given node: + // . if root, norting needs to be done + // . if child of root, splay once + // . else zig-zig or sig-zag depending on path from grandparent + if (root() == tc) return; + warning("*** Splaying not yet implemented; " + "tree operations may be inefficient ***"); +} + +template +void BinaryTreeDictionary::insertChunkInTree(Chunk* fc) { + TreeList *curTL, *prevTL; + size_t size = fc->size(); + + assert(size >= BinaryTreeDictionary::min_tree_chunk_size, "too small to be a TreeList"); + if (FLSVerifyDictionary) { + verifyTree(); + } + + fc->clearNext(); + fc->linkPrev(NULL); + + // work down from the _root, looking for insertion point + for (prevTL = curTL = root(); curTL != NULL;) { + if (curTL->size() == size) // exact match + break; + prevTL = curTL; + if (curTL->size() > size) { // follow left branch + curTL = curTL->left(); + } else { // follow right branch + assert(curTL->size() < size, "size inconsistency"); + curTL = curTL->right(); + } + } + TreeChunk* tc = TreeChunk::as_TreeChunk(fc); + // This chunk is being returned to the binary tree. Its embedded + // TreeList should be unused at this point. + tc->initialize(); + if (curTL != NULL) { // exact match + tc->set_list(curTL); + curTL->returnChunkAtTail(tc); + } else { // need a new node in tree + tc->clearNext(); + tc->linkPrev(NULL); + TreeList* newTL = TreeList::as_TreeList(tc); + assert(((TreeChunk*)tc)->list() == newTL, + "List was not initialized correctly"); + if (prevTL == NULL) { // we are the only tree node + assert(root() == NULL, "control point invariant"); + set_root(newTL); + } else { // insert under prevTL ... + if (prevTL->size() < size) { // am right child + assert(prevTL->right() == NULL, "control point invariant"); + prevTL->setRight(newTL); + } else { // am left child + assert(prevTL->size() > size && prevTL->left() == NULL, "cpt pt inv"); + prevTL->setLeft(newTL); + } + } + } + assert(tc->list() != NULL, "Tree list should be set"); + + inc_totalSize(size); + // Method 'totalSizeInTree' walks through the every block in the + // tree, so it can cause significant performance loss if there are + // many blocks in the tree + assert(!FLSVerifyDictionary || totalSizeInTree(root()) == totalSize(), "_totalSize inconsistency"); + set_totalFreeBlocks(totalFreeBlocks() + 1); + if (FLSVerifyDictionary) { + verifyTree(); + } +} + +template +size_t BinaryTreeDictionary::maxChunkSize() const { + FreeBlockDictionary::verify_par_locked(); + TreeList* tc = root(); + if (tc == NULL) return 0; + for (; tc->right() != NULL; tc = tc->right()); + return tc->size(); +} + +template +size_t BinaryTreeDictionary::totalListLength(TreeList* tl) const { + size_t res; + res = tl->count(); +#ifdef ASSERT + size_t cnt; + Chunk* tc = tl->head(); + for (cnt = 0; tc != NULL; tc = tc->next(), cnt++); + assert(res == cnt, "The count is not being maintained correctly"); +#endif + return res; +} + +template +size_t BinaryTreeDictionary::totalSizeInTree(TreeList* tl) const { + if (tl == NULL) + return 0; + return (tl->size() * totalListLength(tl)) + + totalSizeInTree(tl->left()) + + totalSizeInTree(tl->right()); +} + +template +double BinaryTreeDictionary::sum_of_squared_block_sizes(TreeList* const tl) const { + if (tl == NULL) { + return 0.0; + } + double size = (double)(tl->size()); + double curr = size * size * totalListLength(tl); + curr += sum_of_squared_block_sizes(tl->left()); + curr += sum_of_squared_block_sizes(tl->right()); + return curr; +} + +template +size_t BinaryTreeDictionary::totalFreeBlocksInTree(TreeList* tl) const { + if (tl == NULL) + return 0; + return totalListLength(tl) + + totalFreeBlocksInTree(tl->left()) + + totalFreeBlocksInTree(tl->right()); +} + +template +size_t BinaryTreeDictionary::numFreeBlocks() const { + assert(totalFreeBlocksInTree(root()) == totalFreeBlocks(), + "_totalFreeBlocks inconsistency"); + return totalFreeBlocks(); +} + +template +size_t BinaryTreeDictionary::treeHeightHelper(TreeList* tl) const { + if (tl == NULL) + return 0; + return 1 + MAX2(treeHeightHelper(tl->left()), + treeHeightHelper(tl->right())); +} + +template +size_t BinaryTreeDictionary::treeHeight() const { + return treeHeightHelper(root()); +} + +template +size_t BinaryTreeDictionary::totalNodesHelper(TreeList* tl) const { + if (tl == NULL) { + return 0; + } + return 1 + totalNodesHelper(tl->left()) + + totalNodesHelper(tl->right()); +} + +template +size_t BinaryTreeDictionary::totalNodesInTree(TreeList* tl) const { + return totalNodesHelper(root()); +} + +template +void BinaryTreeDictionary::dictCensusUpdate(size_t size, bool split, bool birth){ + TreeList* nd = findList(size); + if (nd) { + if (split) { + if (birth) { + nd->increment_splitBirths(); + nd->increment_surplus(); + } else { + nd->increment_splitDeaths(); + nd->decrement_surplus(); + } + } else { + if (birth) { + nd->increment_coalBirths(); + nd->increment_surplus(); + } else { + nd->increment_coalDeaths(); + nd->decrement_surplus(); + } + } + } + // A list for this size may not be found (nd == 0) if + // This is a death where the appropriate list is now + // empty and has been removed from the list. + // This is a birth associated with a LinAB. The chunk + // for the LinAB is not in the dictionary. +} + +template +bool BinaryTreeDictionary::coalDictOverPopulated(size_t size) { + if (FLSAlwaysCoalesceLarge) return true; + + TreeList* list_of_size = findList(size); + // None of requested size implies overpopulated. + return list_of_size == NULL || list_of_size->coalDesired() <= 0 || + list_of_size->count() > list_of_size->coalDesired(); +} + +// Closures for walking the binary tree. +// do_list() walks the free list in a node applying the closure +// to each free chunk in the list +// do_tree() walks the nodes in the binary tree applying do_list() +// to each list at each node. + +template +class TreeCensusClosure : public StackObj { + protected: + virtual void do_list(FreeList* fl) = 0; + public: + virtual void do_tree(TreeList* tl) = 0; +}; + +template +class AscendTreeCensusClosure : public TreeCensusClosure { + public: + void do_tree(TreeList* tl) { + if (tl != NULL) { + do_tree(tl->left()); + do_list(tl); + do_tree(tl->right()); + } + } +}; + +template +class DescendTreeCensusClosure : public TreeCensusClosure { + public: + void do_tree(TreeList* tl) { + if (tl != NULL) { + do_tree(tl->right()); + do_list(tl); + do_tree(tl->left()); + } + } +}; + +// For each list in the tree, calculate the desired, desired +// coalesce, count before sweep, and surplus before sweep. +template +class BeginSweepClosure : public AscendTreeCensusClosure { + double _percentage; + float _inter_sweep_current; + float _inter_sweep_estimate; + float _intra_sweep_estimate; + + public: + BeginSweepClosure(double p, float inter_sweep_current, + float inter_sweep_estimate, + float intra_sweep_estimate) : + _percentage(p), + _inter_sweep_current(inter_sweep_current), + _inter_sweep_estimate(inter_sweep_estimate), + _intra_sweep_estimate(intra_sweep_estimate) { } + + void do_list(FreeList* fl) { + double coalSurplusPercent = _percentage; + fl->compute_desired(_inter_sweep_current, _inter_sweep_estimate, _intra_sweep_estimate); + fl->set_coalDesired((ssize_t)((double)fl->desired() * coalSurplusPercent)); + fl->set_beforeSweep(fl->count()); + fl->set_bfrSurp(fl->surplus()); + } +}; + +// Used to search the tree until a condition is met. +// Similar to TreeCensusClosure but searches the +// tree and returns promptly when found. + +template +class TreeSearchClosure : public StackObj { + protected: + virtual bool do_list(FreeList* fl) = 0; + public: + virtual bool do_tree(TreeList* tl) = 0; +}; + +#if 0 // Don't need this yet but here for symmetry. +template +class AscendTreeSearchClosure : public TreeSearchClosure { + public: + bool do_tree(TreeList* tl) { + if (tl != NULL) { + if (do_tree(tl->left())) return true; + if (do_list(tl)) return true; + if (do_tree(tl->right())) return true; + } + return false; + } +}; +#endif + +template +class DescendTreeSearchClosure : public TreeSearchClosure { + public: + bool do_tree(TreeList* tl) { + if (tl != NULL) { + if (do_tree(tl->right())) return true; + if (do_list(tl)) return true; + if (do_tree(tl->left())) return true; + } + return false; + } +}; + +// Searches the tree for a chunk that ends at the +// specified address. +template +class EndTreeSearchClosure : public DescendTreeSearchClosure { + HeapWord* _target; + Chunk* _found; + + public: + EndTreeSearchClosure(HeapWord* target) : _target(target), _found(NULL) {} + bool do_list(FreeList* fl) { + Chunk* item = fl->head(); + while (item != NULL) { + if (item->end() == _target) { + _found = item; + return true; + } + item = item->next(); + } + return false; + } + Chunk* found() { return _found; } +}; + +template +Chunk* BinaryTreeDictionary::find_chunk_ends_at(HeapWord* target) const { + EndTreeSearchClosure etsc(target); + bool found_target = etsc.do_tree(root()); + assert(found_target || etsc.found() == NULL, "Consistency check"); + assert(!found_target || etsc.found() != NULL, "Consistency check"); + return etsc.found(); +} + +template +void BinaryTreeDictionary::beginSweepDictCensus(double coalSurplusPercent, + float inter_sweep_current, float inter_sweep_estimate, float intra_sweep_estimate) { + BeginSweepClosure bsc(coalSurplusPercent, inter_sweep_current, + inter_sweep_estimate, + intra_sweep_estimate); + bsc.do_tree(root()); +} + +// Closures and methods for calculating total bytes returned to the +// free lists in the tree. +#ifndef PRODUCT +template +class InitializeDictReturnedBytesClosure : public AscendTreeCensusClosure { + public: + void do_list(FreeList* fl) { + fl->set_returnedBytes(0); + } +}; + +template +void BinaryTreeDictionary::initializeDictReturnedBytes() { + InitializeDictReturnedBytesClosure idrb; + idrb.do_tree(root()); +} + +template +class ReturnedBytesClosure : public AscendTreeCensusClosure { + size_t _dictReturnedBytes; + public: + ReturnedBytesClosure() { _dictReturnedBytes = 0; } + void do_list(FreeList* fl) { + _dictReturnedBytes += fl->returnedBytes(); + } + size_t dictReturnedBytes() { return _dictReturnedBytes; } +}; + +template +size_t BinaryTreeDictionary::sumDictReturnedBytes() { + ReturnedBytesClosure rbc; + rbc.do_tree(root()); + + return rbc.dictReturnedBytes(); +} + +// Count the number of entries in the tree. +template +class treeCountClosure : public DescendTreeCensusClosure { + public: + uint count; + treeCountClosure(uint c) { count = c; } + void do_list(FreeList* fl) { + count++; + } +}; + +template +size_t BinaryTreeDictionary::totalCount() { + treeCountClosure ctc(0); + ctc.do_tree(root()); + return ctc.count; +} +#endif // PRODUCT + +// Calculate surpluses for the lists in the tree. +template +class setTreeSurplusClosure : public AscendTreeCensusClosure { + double percentage; + public: + setTreeSurplusClosure(double v) { percentage = v; } + void do_list(FreeList* fl) { + double splitSurplusPercent = percentage; + fl->set_surplus(fl->count() - + (ssize_t)((double)fl->desired() * splitSurplusPercent)); + } +}; + +template +void BinaryTreeDictionary::setTreeSurplus(double splitSurplusPercent) { + setTreeSurplusClosure sts(splitSurplusPercent); + sts.do_tree(root()); +} + +// Set hints for the lists in the tree. +template +class setTreeHintsClosure : public DescendTreeCensusClosure { + size_t hint; + public: + setTreeHintsClosure(size_t v) { hint = v; } + void do_list(FreeList* fl) { + fl->set_hint(hint); + assert(fl->hint() == 0 || fl->hint() > fl->size(), + "Current hint is inconsistent"); + if (fl->surplus() > 0) { + hint = fl->size(); + } + } +}; + +template +void BinaryTreeDictionary::setTreeHints(void) { + setTreeHintsClosure sth(0); + sth.do_tree(root()); +} + +// Save count before previous sweep and splits and coalesces. +template +class clearTreeCensusClosure : public AscendTreeCensusClosure { + void do_list(FreeList* fl) { + fl->set_prevSweep(fl->count()); + fl->set_coalBirths(0); + fl->set_coalDeaths(0); + fl->set_splitBirths(0); + fl->set_splitDeaths(0); + } +}; + +template +void BinaryTreeDictionary::clearTreeCensus(void) { + clearTreeCensusClosure ctc; + ctc.do_tree(root()); +} + +// Do reporting and post sweep clean up. +template +void BinaryTreeDictionary::endSweepDictCensus(double splitSurplusPercent) { + // Does walking the tree 3 times hurt? + setTreeSurplus(splitSurplusPercent); + setTreeHints(); + if (PrintGC && Verbose) { + reportStatistics(); + } + clearTreeCensus(); +} + +// Print summary statistics +template +void BinaryTreeDictionary::reportStatistics() const { + FreeBlockDictionary::verify_par_locked(); + gclog_or_tty->print("Statistics for BinaryTreeDictionary:\n" + "------------------------------------\n"); + size_t totalSize = totalChunkSize(debug_only(NULL)); + size_t freeBlocks = numFreeBlocks(); + gclog_or_tty->print("Total Free Space: %d\n", totalSize); + gclog_or_tty->print("Max Chunk Size: %d\n", maxChunkSize()); + gclog_or_tty->print("Number of Blocks: %d\n", freeBlocks); + if (freeBlocks > 0) { + gclog_or_tty->print("Av. Block Size: %d\n", totalSize/freeBlocks); + } + gclog_or_tty->print("Tree Height: %d\n", treeHeight()); +} + +// Print census information - counts, births, deaths, etc. +// for each list in the tree. Also print some summary +// information. +template +class PrintTreeCensusClosure : public AscendTreeCensusClosure { + int _print_line; + size_t _totalFree; + FreeList _total; + + public: + PrintTreeCensusClosure() { + _print_line = 0; + _totalFree = 0; + } + FreeList* total() { return &_total; } + size_t totalFree() { return _totalFree; } + void do_list(FreeList* fl) { + if (++_print_line >= 40) { + FreeList::print_labels_on(gclog_or_tty, "size"); + _print_line = 0; + } + fl->print_on(gclog_or_tty); + _totalFree += fl->count() * fl->size() ; + total()->set_count( total()->count() + fl->count() ); + total()->set_bfrSurp( total()->bfrSurp() + fl->bfrSurp() ); + total()->set_surplus( total()->splitDeaths() + fl->surplus() ); + total()->set_desired( total()->desired() + fl->desired() ); + total()->set_prevSweep( total()->prevSweep() + fl->prevSweep() ); + total()->set_beforeSweep(total()->beforeSweep() + fl->beforeSweep()); + total()->set_coalBirths( total()->coalBirths() + fl->coalBirths() ); + total()->set_coalDeaths( total()->coalDeaths() + fl->coalDeaths() ); + total()->set_splitBirths(total()->splitBirths() + fl->splitBirths()); + total()->set_splitDeaths(total()->splitDeaths() + fl->splitDeaths()); + } +}; + +template +void BinaryTreeDictionary::printDictCensus(void) const { + + gclog_or_tty->print("\nBinaryTree\n"); + FreeList::print_labels_on(gclog_or_tty, "size"); + PrintTreeCensusClosure ptc; + ptc.do_tree(root()); + + FreeList* total = ptc.total(); + FreeList::print_labels_on(gclog_or_tty, " "); + total->print_on(gclog_or_tty, "TOTAL\t"); + gclog_or_tty->print( + "totalFree(words): " SIZE_FORMAT_W(16) + " growth: %8.5f deficit: %8.5f\n", + ptc.totalFree(), + (double)(total->splitBirths() + total->coalBirths() + - total->splitDeaths() - total->coalDeaths()) + /(total->prevSweep() != 0 ? (double)total->prevSweep() : 1.0), + (double)(total->desired() - total->count()) + /(total->desired() != 0 ? (double)total->desired() : 1.0)); +} + +template +class PrintFreeListsClosure : public AscendTreeCensusClosure { + outputStream* _st; + int _print_line; + + public: + PrintFreeListsClosure(outputStream* st) { + _st = st; + _print_line = 0; + } + void do_list(FreeList* fl) { + if (++_print_line >= 40) { + FreeList::print_labels_on(_st, "size"); + _print_line = 0; + } + fl->print_on(gclog_or_tty); + size_t sz = fl->size(); + for (Chunk* fc = fl->head(); fc != NULL; + fc = fc->next()) { + _st->print_cr("\t[" PTR_FORMAT "," PTR_FORMAT ") %s", + fc, (HeapWord*)fc + sz, + fc->cantCoalesce() ? "\t CC" : ""); + } + } +}; + +template +void BinaryTreeDictionary::print_free_lists(outputStream* st) const { + + FreeList::print_labels_on(st, "size"); + PrintFreeListsClosure pflc(st); + pflc.do_tree(root()); +} + +// Verify the following tree invariants: +// . _root has no parent +// . parent and child point to each other +// . each node's key correctly related to that of its child(ren) +template +void BinaryTreeDictionary::verifyTree() const { + guarantee(root() == NULL || totalFreeBlocks() == 0 || + totalSize() != 0, "_totalSize should't be 0?"); + guarantee(root() == NULL || root()->parent() == NULL, "_root shouldn't have parent"); + verifyTreeHelper(root()); +} + +template +size_t BinaryTreeDictionary::verifyPrevFreePtrs(TreeList* tl) { + size_t ct = 0; + for (Chunk* curFC = tl->head(); curFC != NULL; curFC = curFC->next()) { + ct++; + assert(curFC->prev() == NULL || curFC->prev()->isFree(), + "Chunk should be free"); + } + return ct; +} + +// Note: this helper is recursive rather than iterative, so use with +// caution on very deep trees; and watch out for stack overflow errors; +// In general, to be used only for debugging. +template +void BinaryTreeDictionary::verifyTreeHelper(TreeList* tl) const { + if (tl == NULL) + return; + guarantee(tl->size() != 0, "A list must has a size"); + guarantee(tl->left() == NULL || tl->left()->parent() == tl, + "parent<-/->left"); + guarantee(tl->right() == NULL || tl->right()->parent() == tl, + "parent<-/->right");; + guarantee(tl->left() == NULL || tl->left()->size() < tl->size(), + "parent !> left"); + guarantee(tl->right() == NULL || tl->right()->size() > tl->size(), + "parent !< left"); + guarantee(tl->head() == NULL || tl->head()->isFree(), "!Free"); + guarantee(tl->head() == NULL || tl->head_as_TreeChunk()->list() == tl, + "list inconsistency"); + guarantee(tl->count() > 0 || (tl->head() == NULL && tl->tail() == NULL), + "list count is inconsistent"); + guarantee(tl->count() > 1 || tl->head() == tl->tail(), + "list is incorrectly constructed"); + size_t count = verifyPrevFreePtrs(tl); + guarantee(count == (size_t)tl->count(), "Node count is incorrect"); + if (tl->head() != NULL) { + tl->head_as_TreeChunk()->verifyTreeChunkList(); + } + verifyTreeHelper(tl->left()); + verifyTreeHelper(tl->right()); +} + +template +void BinaryTreeDictionary::verify() const { + verifyTree(); + guarantee(totalSize() == totalSizeInTree(root()), "Total Size inconsistency"); +} + +#ifndef SERIALGC +// Explicitly instantiate these types for FreeChunk. +template class BinaryTreeDictionary; +template class TreeChunk; +template class TreeList; +#endif // SERIALGC