jdk-sandbox: comparison hotspot/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp

equal deleted inserted replaced

-:01f84e2c3fc0
+:305a76435cf1
 }
 res = new_region_try_secondary_free_list();
 }
 if (res == NULL && do_expand) {
 if (expand(word_size * HeapWordSize)) {
-// The expansion succeeded and so we should have at least one
+// Even though the heap was expanded, it might not have reached
-// region on the free list.
+// the desired size. So, we cannot assume that the allocation
-res = _free_list.remove_head();
+// will succeed.
+res = _free_list.remove_head_or_null();
 }
 }
 if (res != NULL) {
 if (G1PrintHeapRegions) {
-gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT","PTR_FORMAT"], "
+gclog_or_tty->print_cr("new alloc region "HR_FORMAT,
-"top "PTR_FORMAT, res->hrs_index(),
+HR_FORMAT_PARAMS(res));
-res->bottom(), res->end(), res->top());
 }
 }
 return res;
 }
 g1_policy()->note_alloc_region_limit_reached(purpose);
 }
 return alloc_region;
 }
-int G1CollectedHeap::humongous_obj_allocate_find_first(size_t num_regions,
+size_t G1CollectedHeap::humongous_obj_allocate_find_first(size_t num_regions,
 size_t word_size) {
 assert(isHumongous(word_size), "word_size should be humongous");
 assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
-int first = -1;
+size_t first = G1_NULL_HRS_INDEX;
 if (num_regions == 1) {
 // Only one region to allocate, no need to go through the slower
 // path. The caller will attempt the expasion if this fails, so
 // let's not try to expand here too.
 HeapRegion* hr = new_region(word_size, false /* do_expand */);
 if (hr != NULL) {
 first = hr->hrs_index();
 } else {
-first = -1;
+first = G1_NULL_HRS_INDEX;
 }
 } else {
 // We can't allocate humongous regions while cleanupComplete() is
 // running, since some of the regions we find to be empty might not
 // yet be added to the free list and it is not straightforward to
 // region allocation code (see above).
 wait_while_free_regions_coming();
 append_secondary_free_list_if_not_empty_with_lock();
 if (free_regions() >= num_regions) {
-first = _hrs->find_contiguous(num_regions);
+first = _hrs.find_contiguous(num_regions);
-if (first != -1) {
+if (first != G1_NULL_HRS_INDEX) {
-for (int i = first; i < first + (int) num_regions; ++i) {
+for (size_t i = first; i < first + num_regions; ++i) {
-HeapRegion* hr = _hrs->at(i);
+HeapRegion* hr = region_at(i);
 assert(hr->is_empty(), "sanity");
 assert(is_on_master_free_list(hr), "sanity");
 hr->set_pending_removal(true);
 }
 _free_list.remove_all_pending(num_regions);
 }
 return first;
 }
 HeapWord*
-G1CollectedHeap::humongous_obj_allocate_initialize_regions(int first,
+G1CollectedHeap::humongous_obj_allocate_initialize_regions(size_t first,
 size_t num_regions,
 size_t word_size) {
-assert(first != -1, "pre-condition");
+assert(first != G1_NULL_HRS_INDEX, "pre-condition");
 assert(isHumongous(word_size), "word_size should be humongous");
 assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition");
 // Index of last region in the series + 1.
-int last = first + (int) num_regions;
+size_t last = first + num_regions;
 // We need to initialize the region(s) we just discovered. This is
 // a bit tricky given that it can happen concurrently with
 // refinement threads refining cards on these regions and
 // potentially wanting to refine the BOT as they are scanning
 // The word size sum of all the regions we will allocate.
 size_t word_size_sum = num_regions * HeapRegion::GrainWords;
 assert(word_size <= word_size_sum, "sanity");
 // This will be the "starts humongous" region.
-HeapRegion* first_hr = _hrs->at(first);
+HeapRegion* first_hr = region_at(first);
 // The header of the new object will be placed at the bottom of
 // the first region.
 HeapWord* new_obj = first_hr->bottom();
 // This will be the new end of the first region in the series that
 // should also match the end of the last region in the seriers.
 first_hr->set_startsHumongous(new_top, new_end);
 // Then, if there are any, we will set up the "continues
 // humongous" regions.
 HeapRegion* hr = NULL;
-for (int i = first + 1; i < last; ++i) {
+for (size_t i = first + 1; i < last; ++i) {
-hr = _hrs->at(i);
+hr = region_at(i);
 hr->set_continuesHumongous(first_hr);
 }
 // If we have "continues humongous" regions (hr != NULL), then the
 // end of the last one should match new_end.
 assert(hr == NULL || hr->end() == new_end, "sanity");
 // fields here. The way we set top for all regions (i.e., top ==
 // end for all regions but the last one, top == new_top for the
 // last one) is actually used when we will free up the humongous
 // region in free_humongous_region().
 hr = NULL;
-for (int i = first + 1; i < last; ++i) {
+for (size_t i = first + 1; i < last; ++i) {
-hr = _hrs->at(i);
+hr = region_at(i);
 if ((i + 1) == last) {
 // last continues humongous region
 assert(hr->bottom() < new_top && new_top <= hr->end(),
 "new_top should fall on this region");
 hr->set_top(new_top);
 verify_region_sets_optional();
 size_t num_regions =
 round_to(word_size, HeapRegion::GrainWords) / HeapRegion::GrainWords;
 size_t x_size = expansion_regions();
-size_t fs = _hrs->free_suffix();
+size_t fs = _hrs.free_suffix();
-int first = humongous_obj_allocate_find_first(num_regions, word_size);
+size_t first = humongous_obj_allocate_find_first(num_regions, word_size);
-if (first == -1) {
+if (first == G1_NULL_HRS_INDEX) {
 // The only thing we can do now is attempt expansion.
 if (fs + x_size >= num_regions) {
 // If the number of regions we're trying to allocate for this
 // object is at most the number of regions in the free suffix,
 // then the call to humongous_obj_allocate_find_first() above
 // not sufficient for the object _and_ we have some expansion
 // room available.
 assert(num_regions > fs, "earlier allocation should have succeeded");
 if (expand((num_regions - fs) * HeapRegion::GrainBytes)) {
+// Even though the heap was expanded, it might not have
+// reached the desired size. So, we cannot assume that the
+// allocation will succeed.
 first = humongous_obj_allocate_find_first(num_regions, word_size);
-// If the expansion was successful then the allocation
-// should have been successful.
-assert(first != -1, "this should have worked");
 }
 }
 }
 HeapWord* result = NULL;
-if (first != -1) {
+if (first != G1_NULL_HRS_INDEX) {
 result =
 humongous_obj_allocate_initialize_regions(first, num_regions, word_size);
 assert(result != NULL, "it should always return a valid result");
 }
 }
 // Update the number of full collections that have been completed.
 increment_full_collections_completed(false /* concurrent */);
+_hrs.verify_optional();
 verify_region_sets_optional();
 if (PrintHeapAtGC) {
 Universe::print_heap_after_gc();
 }
 verify_region_sets_optional();
 size_t expand_bytes = MAX2(word_size * HeapWordSize, MinHeapDeltaBytes);
 if (expand(expand_bytes)) {
+_hrs.verify_optional();
 verify_region_sets_optional();
 return attempt_allocation_at_safepoint(word_size,
 false /* expect_null_mutator_alloc_region */);
 }
 return NULL;
+}
+void G1CollectedHeap::update_committed_space(HeapWord* old_end,
+HeapWord* new_end) {
+assert(old_end != new_end, "don't call this otherwise");
+assert((HeapWord*) _g1_storage.high() == new_end, "invariant");
+// Update the committed mem region.
+_g1_committed.set_end(new_end);
+// Tell the card table about the update.
+Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);
+// Tell the BOT about the update.
+_bot_shared->resize(_g1_committed.word_size());
 }
 bool G1CollectedHeap::expand(size_t expand_bytes) {
 size_t old_mem_size = _g1_storage.committed_size();
 size_t aligned_expand_bytes = ReservedSpace::page_align_size_up(expand_bytes);
 if (Verbose && PrintGC) {
 gclog_or_tty->print("Expanding garbage-first heap from %ldK by %ldK",
 old_mem_size/K, aligned_expand_bytes/K);
 }
-HeapWord* old_end = (HeapWord*)_g1_storage.high();
+// First commit the memory.
+HeapWord* old_end = (HeapWord*) _g1_storage.high();
 bool successful = _g1_storage.expand_by(aligned_expand_bytes);
 if (successful) {
-HeapWord* new_end = (HeapWord*)_g1_storage.high();
+// Then propagate this update to the necessary data structures.
+HeapWord* new_end = (HeapWord*) _g1_storage.high();
-// Expand the committed region.
+update_committed_space(old_end, new_end);
-_g1_committed.set_end(new_end);
+FreeRegionList expansion_list("Local Expansion List");
-// Tell the cardtable about the expansion.
+MemRegion mr = _hrs.expand_by(old_end, new_end, &expansion_list);
-Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);
+assert(mr.start() == old_end, "post-condition");
+// mr might be a smaller region than what was requested if
-// And the offset table as well.
+// expand_by() was unable to allocate the HeapRegion instances
-_bot_shared->resize(_g1_committed.word_size());
+assert(mr.end() <= new_end, "post-condition");
-expand_bytes = aligned_expand_bytes;
+size_t actual_expand_bytes = mr.byte_size();
-HeapWord* base = old_end;
+assert(actual_expand_bytes <= aligned_expand_bytes, "post-condition");
+assert(actual_expand_bytes == expansion_list.total_capacity_bytes(),
-// Create the heap regions for [old_end, new_end)
+"post-condition");
-while (expand_bytes > 0) {
+if (actual_expand_bytes < aligned_expand_bytes) {
-HeapWord* high = base + HeapRegion::GrainWords;
+// We could not expand _hrs to the desired size. In this case we
+// need to shrink the committed space accordingly.
-// Create a new HeapRegion.
+assert(mr.end() < new_end, "invariant");
-MemRegion mr(base, high);
-bool is_zeroed = !_g1_max_committed.contains(base);
+size_t diff_bytes = aligned_expand_bytes - actual_expand_bytes;
-HeapRegion* hr = new HeapRegion(_bot_shared, mr, is_zeroed);
+// First uncommit the memory.
+_g1_storage.shrink_by(diff_bytes);
-// Add it to the HeapRegionSeq.
+// Then propagate this update to the necessary data structures.
-_hrs->insert(hr);
+update_committed_space(new_end, mr.end());
-_free_list.add_as_tail(hr);
+}
+_free_list.add_as_tail(&expansion_list);
-// And we used up an expansion region to create it.
-_expansion_regions--;
-expand_bytes -= HeapRegion::GrainBytes;
-base += HeapRegion::GrainWords;
-}
-assert(base == new_end, "sanity");
-// Now update max_committed if necessary.
-_g1_max_committed.set_end(MAX2(_g1_max_committed.end(), new_end));
 } else {
 // The expansion of the virtual storage space was unsuccessful.
 // Let's see if it was because we ran out of swap.
 if (G1ExitOnExpansionFailure &&
 _g1_storage.uncommitted_size() >= aligned_expand_bytes) {
 new_mem_size/K);
 }
 return successful;
 }
-void G1CollectedHeap::shrink_helper(size_t shrink_bytes)
+void G1CollectedHeap::shrink_helper(size_t shrink_bytes) {
-{
 size_t old_mem_size = _g1_storage.committed_size();
 size_t aligned_shrink_bytes =
 ReservedSpace::page_align_size_down(shrink_bytes);
 aligned_shrink_bytes = align_size_down(aligned_shrink_bytes,
 HeapRegion::GrainBytes);
 size_t num_regions_deleted = 0;
-MemRegion mr = _hrs->shrink_by(aligned_shrink_bytes, num_regions_deleted);
+MemRegion mr = _hrs.shrink_by(aligned_shrink_bytes, &num_regions_deleted);
+HeapWord* old_end = (HeapWord*) _g1_storage.high();
-assert(mr.end() == (HeapWord*)_g1_storage.high(), "Bad shrink!");
+assert(mr.end() == old_end, "post-condition");
-if (mr.byte_size() > 0)
+if (mr.byte_size() > 0) {
 _g1_storage.shrink_by(mr.byte_size());
-assert(mr.start() == (HeapWord*)_g1_storage.high(), "Bad shrink!");
+HeapWord* new_end = (HeapWord*) _g1_storage.high();
+assert(mr.start() == new_end, "post-condition");
-_g1_committed.set_end(mr.start());
 _expansion_regions += num_regions_deleted;
+update_committed_space(old_end, new_end);
-// Tell the cardtable about it.
+HeapRegionRemSet::shrink_heap(n_regions());
-Universe::heap()->barrier_set()->resize_covered_region(_g1_committed);
+if (Verbose && PrintGC) {
-// And the offset table as well.
+size_t new_mem_size = _g1_storage.committed_size();
-_bot_shared->resize(_g1_committed.word_size());
+gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",
+old_mem_size/K, aligned_shrink_bytes/K,
-HeapRegionRemSet::shrink_heap(n_regions());
+new_mem_size/K);
+}
-if (Verbose && PrintGC) {
-size_t new_mem_size = _g1_storage.committed_size();
-gclog_or_tty->print_cr("Shrinking garbage-first heap from %ldK by %ldK to %ldK",
-old_mem_size/K, aligned_shrink_bytes/K,
-new_mem_size/K);
 }
 }
 void G1CollectedHeap::shrink(size_t shrink_bytes) {
 verify_region_sets_optional();
 // remove only the ones that we need to remove.
 tear_down_region_lists();  // We will rebuild them in a moment.
 shrink_helper(shrink_bytes);
 rebuild_region_lists();
+_hrs.verify_optional();
 verify_region_sets_optional();
 }
 // Public methods.
 _perm_gen = pgs->init(perm_gen_rs, pgs->init_size(), rem_set());
 _g1_storage.initialize(g1_rs, 0);
 _g1_committed = MemRegion((HeapWord*)_g1_storage.low(), (size_t) 0);
-_g1_max_committed = _g1_committed;
+_hrs.initialize((HeapWord*) _g1_reserved.start(),
-_hrs = new HeapRegionSeq(_expansion_regions);
+(HeapWord*) _g1_reserved.end(),
-guarantee(_hrs != NULL, "Couldn't allocate HeapRegionSeq");
+_expansion_regions);
 // 6843694 - ensure that the maximum region index can fit
 // in the remembered set structures.
 const size_t max_region_idx = ((size_t)1 << (sizeof(RegionIdx_t)*BitsPerByte-1)) - 1;
 guarantee((max_regions() - 1) <= max_region_idx, "too many regions");
 _cg1r->init();
 // Here we allocate the dummy full region that is required by the
 // G1AllocRegion class. If we don't pass an address in the reserved
 // space here, lots of asserts fire.
-MemRegion mr(_g1_reserved.start(), HeapRegion::GrainWords);
-HeapRegion* dummy_region = new HeapRegion(_bot_shared, mr, true);
+HeapRegion* dummy_region = new_heap_region(0 /* index of bottom region */,
+_g1_reserved.start());
 // We'll re-use the same region whether the alloc region will
 // require BOT updates or not and, if it doesn't, then a non-young
 // region will complain that it cannot support allocations without
 // BOT updates. So we'll tag the dummy region as young to avoid that.
 dummy_region->set_young();
 size_t result() { return _used; }
 };
 size_t G1CollectedHeap::recalculate_used() const {
 SumUsedClosure blk;
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 return blk.result();
 }
 #ifndef PRODUCT
 class SumUsedRegionsClosure: public HeapRegionClosure {
 size_t result() { return _num; }
 };
 size_t G1CollectedHeap::recalculate_used_regions() const {
 SumUsedRegionsClosure blk;
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 return blk.result();
 }
 #endif // PRODUCT
 size_t G1CollectedHeap::unsafe_max_alloc() {
 }
 }
 }
 bool G1CollectedHeap::is_in(const void* p) const {
-if (_g1_committed.contains(p)) {
+HeapRegion* hr = _hrs.addr_to_region((HeapWord*) p);
-HeapRegion* hr = _hrs->addr_to_region(p);
+if (hr != NULL) {
 return hr->is_in(p);
 } else {
 return _perm_gen->as_gen()->is_in(p);
 }
 }
 }
 };
 void G1CollectedHeap::oop_iterate(OopClosure* cl, bool do_perm) {
 IterateOopClosureRegionClosure blk(_g1_committed, cl);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 if (do_perm) {
 perm_gen()->oop_iterate(cl);
 }
 }
 void G1CollectedHeap::oop_iterate(MemRegion mr, OopClosure* cl, bool do_perm) {
 IterateOopClosureRegionClosure blk(mr, cl);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 if (do_perm) {
 perm_gen()->oop_iterate(cl);
 }
 }
 }
 };
 void G1CollectedHeap::object_iterate(ObjectClosure* cl, bool do_perm) {
 IterateObjectClosureRegionClosure blk(cl);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 if (do_perm) {
 perm_gen()->object_iterate(cl);
 }
 }
 }
 };
 void G1CollectedHeap::space_iterate(SpaceClosure* cl) {
 SpaceClosureRegionClosure blk(cl);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 }
-void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) {
+void G1CollectedHeap::heap_region_iterate(HeapRegionClosure* cl) const {
-_hrs->iterate(cl);
+_hrs.iterate(cl);
 }
 void G1CollectedHeap::heap_region_iterate_from(HeapRegion* r,
-HeapRegionClosure* cl) {
+HeapRegionClosure* cl) const {
-_hrs->iterate_from(r, cl);
+_hrs.iterate_from(r, cl);
 }
-void
-G1CollectedHeap::heap_region_iterate_from(int idx, HeapRegionClosure* cl) {
-_hrs->iterate_from(idx, cl);
-}
-HeapRegion* G1CollectedHeap::region_at(size_t idx) { return _hrs->at(idx); }
 void
 G1CollectedHeap::heap_region_par_iterate_chunked(HeapRegionClosure* cl,
 int worker,
 jint claim_value) {
 cur = next;
 }
 }
 CompactibleSpace* G1CollectedHeap::first_compactible_space() {
-return _hrs->length() > 0 ? _hrs->at(0) : NULL;
+return n_regions() > 0 ? region_at(0) : NULL;
 }
 Space* G1CollectedHeap::space_containing(const void* addr) const {
 Space* res = heap_region_containing(addr);
 assert(check_heap_region_claim_values(HeapRegion::InitialClaimValue),
 "sanity check");
 } else {
 VerifyRegionClosure blk(allow_dirty, false, use_prev_marking);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 if (blk.failures()) {
 failures = true;
 }
 }
 if (!silent) gclog_or_tty->print("RemSet ");
 }
 }
 void G1CollectedHeap::print_on_extended(outputStream* st) const {
 PrintRegionClosure blk(st);
-_hrs->iterate(&blk);
+heap_region_iterate(&blk);
 }
 void G1CollectedHeap::print_gc_threads_on(outputStream* st) const {
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 workers()->print_worker_threads_on(st);
 if (G1SummarizeConcMark) {
 concurrent_mark()->print_summary_info();
 }
 g1_policy()->print_yg_surv_rate_info();
 SpecializationStats::print();
-}
-int G1CollectedHeap::addr_to_arena_id(void* addr) const {
-HeapRegion* hr = heap_region_containing(addr);
-if (hr == NULL) {
-return 0;
-} else {
-return 1;
-}
 }
 G1CollectedHeap* G1CollectedHeap::heap() {
 assert(_sh->kind() == CollectedHeap::G1CollectedHeap,
 "not a garbage-first heap");
 print_tracing_info();
 vm_exit(-1);
 }
 }
+_hrs.verify_optional();
 verify_region_sets_optional();
 TASKQUEUE_STATS_ONLY(if (ParallelGCVerbose) print_taskqueue_stats());
 TASKQUEUE_STATS_ONLY(reset_taskqueue_stats());
 #ifdef G1_DEBUG
 class FindGCAllocRegion: public HeapRegionClosure {
 public:
 bool doHeapRegion(HeapRegion* r) {
 if (r->is_gc_alloc_region()) {
-gclog_or_tty->print_cr("Region %d ["PTR_FORMAT"...] is still a gc_alloc_region.",
+gclog_or_tty->print_cr("Region "HR_FORMAT" is still a GC alloc region",
-r->hrs_index(), r->bottom());
+HR_FORMAT_PARAMS(r));
 }
 return false;
 }
 };
 #endif // G1_DEBUG
 alloc_region = new_gc_alloc_region(ap, HeapRegion::GrainWords);
 } else {
 // the region was retained from the last collection
 ++_gc_alloc_region_counts[ap];
 if (G1PrintHeapRegions) {
-gclog_or_tty->print_cr("new alloc region %d:["PTR_FORMAT", "PTR_FORMAT"], "
+gclog_or_tty->print_cr("new alloc region "HR_FORMAT,
-"top "PTR_FORMAT,
+HR_FORMAT_PARAMS(alloc_region));
-alloc_region->hrs_index(), alloc_region->bottom(), alloc_region->end(), alloc_region->top());
 }
 }
 if (alloc_region != NULL) {
 assert(_gc_alloc_regions[ap] == NULL, "pre-condition");
 size_t hr_pre_used = 0;
 _humongous_set.remove_with_proxy(hr, humongous_proxy_set);
 hr->set_notHumongous();
 free_region(hr, &hr_pre_used, free_list, par);
-int i = hr->hrs_index() + 1;
+size_t i = hr->hrs_index() + 1;
 size_t num = 1;
-while ((size_t) i < n_regions()) {
+while (i < n_regions()) {
-HeapRegion* curr_hr = _hrs->at(i);
+HeapRegion* curr_hr = region_at(i);
 if (!curr_hr->continuesHumongous()) {
 break;
 }
 curr_hr->set_notHumongous();
 free_region(curr_hr, &hr_pre_used, free_list, par);
 gclog_or_tty->print_cr("G1ConcRegionFreeing [other] : "
 "done waiting for free regions");
 }
 }
-size_t G1CollectedHeap::n_regions() {
-return _hrs->length();
-}
-size_t G1CollectedHeap::max_regions() {
-return
-(size_t)align_size_up(max_capacity(), HeapRegion::GrainBytes) /
-HeapRegion::GrainBytes;
-}
 void G1CollectedHeap::set_region_short_lived_locked(HeapRegion* hr) {
 assert(heap_lock_held_for_gc(),
 "the heap lock should already be held by or for this thread");
 _young_list->push_region(hr);
 g1_policy()->set_region_short_lived(hr);
 }
 return false;
 }
 };
+HeapRegion* G1CollectedHeap::new_heap_region(size_t hrs_index,
+HeapWord* bottom) {
+HeapWord* end = bottom + HeapRegion::GrainWords;
+MemRegion mr(bottom, end);
+assert(_g1_reserved.contains(mr), "invariant");
+// This might return NULL if the allocation fails
+return new HeapRegion(hrs_index, _bot_shared, mr, true /* is_zeroed */);
+}
 void G1CollectedHeap::verify_region_sets() {
 assert_heap_locked_or_at_safepoint(true /* should_be_vm_thread */);
 // First, check the explicit lists.
 _free_list.verify();

changeset 9989	305a76435cf1
parent 9987	6e2b9796996f
child 9995	290620c08233