28 #include "gc/shared/markBitMap.inline.hpp" |
28 #include "gc/shared/markBitMap.inline.hpp" |
29 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" |
29 #include "gc/shared/threadLocalAllocBuffer.inline.hpp" |
30 #include "gc/shared/suspendibleThreadSet.hpp" |
30 #include "gc/shared/suspendibleThreadSet.hpp" |
31 #include "gc/shenandoah/shenandoahAsserts.hpp" |
31 #include "gc/shenandoah/shenandoahAsserts.hpp" |
32 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" |
32 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" |
33 #include "gc/shenandoah/shenandoahBrooksPointer.inline.hpp" |
|
34 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" |
33 #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" |
|
34 #include "gc/shenandoah/shenandoahForwarding.inline.hpp" |
35 #include "gc/shenandoah/shenandoahWorkGroup.hpp" |
35 #include "gc/shenandoah/shenandoahWorkGroup.hpp" |
36 #include "gc/shenandoah/shenandoahHeap.hpp" |
36 #include "gc/shenandoah/shenandoahHeap.hpp" |
37 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" |
37 #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" |
38 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" |
38 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" |
39 #include "gc/shenandoah/shenandoahControlThread.hpp" |
39 #include "gc/shenandoah/shenandoahControlThread.hpp" |
233 } |
233 } |
234 |
234 |
235 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); |
235 assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); |
236 |
236 |
237 size_t size_no_fwdptr = (size_t) p->size(); |
237 size_t size_no_fwdptr = (size_t) p->size(); |
238 size_t size_with_fwdptr = size_no_fwdptr + ShenandoahBrooksPointer::word_size(); |
238 size_t size_with_fwdptr = size_no_fwdptr + ShenandoahForwarding::word_size(); |
239 |
239 |
240 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); |
240 assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); |
241 |
241 |
242 bool alloc_from_gclab = true; |
242 bool alloc_from_gclab = true; |
243 HeapWord* filler = NULL; |
243 HeapWord* filler = NULL; |
267 |
267 |
268 return ShenandoahBarrierSet::resolve_forwarded(p); |
268 return ShenandoahBarrierSet::resolve_forwarded(p); |
269 } |
269 } |
270 |
270 |
271 // Copy the object and initialize its forwarding ptr: |
271 // Copy the object and initialize its forwarding ptr: |
272 HeapWord* copy = filler + ShenandoahBrooksPointer::word_size(); |
272 HeapWord* copy = filler + ShenandoahForwarding::word_size(); |
273 oop copy_val = oop(copy); |
273 oop copy_val = oop(copy); |
274 |
274 |
275 Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr); |
275 Copy::aligned_disjoint_words((HeapWord*) p, copy, size_no_fwdptr); |
276 ShenandoahBrooksPointer::initialize(oop(copy)); |
276 ShenandoahForwarding::initialize(oop(copy)); |
277 |
277 |
278 // Try to install the new forwarding pointer. |
278 // Try to install the new forwarding pointer. |
279 oop result = ShenandoahBrooksPointer::try_update_forwardee(p, copy_val); |
279 oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); |
280 |
280 |
281 if (oopDesc::equals_raw(result, p)) { |
281 if (oopDesc::equals_raw(result, p)) { |
282 // Successfully evacuated. Our copy is now the public one! |
282 // Successfully evacuated. Our copy is now the public one! |
283 shenandoah_assert_correct(NULL, copy_val); |
283 shenandoah_assert_correct(NULL, copy_val); |
284 return copy_val; |
284 return copy_val; |
368 marked_object_iterate(region, cl, region->top()); |
368 marked_object_iterate(region, cl, region->top()); |
369 } |
369 } |
370 |
370 |
371 template<class T> |
371 template<class T> |
372 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) { |
372 inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) { |
373 assert(ShenandoahBrooksPointer::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj"); |
373 assert(ShenandoahForwarding::word_offset() < 0, "skip_delta calculation below assumes the forwarding ptr is before obj"); |
374 assert(! region->is_humongous_continuation(), "no humongous continuation regions here"); |
374 assert(! region->is_humongous_continuation(), "no humongous continuation regions here"); |
375 |
375 |
376 ShenandoahMarkingContext* const ctx = complete_marking_context(); |
376 ShenandoahMarkingContext* const ctx = complete_marking_context(); |
377 assert(ctx->is_complete(), "sanity"); |
377 assert(ctx->is_complete(), "sanity"); |
378 |
378 |
379 MarkBitMap* mark_bit_map = ctx->mark_bit_map(); |
379 MarkBitMap* mark_bit_map = ctx->mark_bit_map(); |
380 HeapWord* tams = ctx->top_at_mark_start(region); |
380 HeapWord* tams = ctx->top_at_mark_start(region); |
381 |
381 |
382 size_t skip_bitmap_delta = ShenandoahBrooksPointer::word_size() + 1; |
382 size_t skip_bitmap_delta = ShenandoahForwarding::word_size() + 1; |
383 size_t skip_objsize_delta = ShenandoahBrooksPointer::word_size() /* + actual obj.size() below */; |
383 size_t skip_objsize_delta = ShenandoahForwarding::word_size() /* + actual obj.size() below */; |
384 HeapWord* start = region->bottom() + ShenandoahBrooksPointer::word_size(); |
384 HeapWord* start = region->bottom() + ShenandoahForwarding::word_size(); |
385 HeapWord* end = MIN2(tams + ShenandoahBrooksPointer::word_size(), region->end()); |
385 HeapWord* end = MIN2(tams + ShenandoahForwarding::word_size(), region->end()); |
386 |
386 |
387 // Step 1. Scan below the TAMS based on bitmap data. |
387 // Step 1. Scan below the TAMS based on bitmap data. |
388 HeapWord* limit_bitmap = MIN2(limit, tams); |
388 HeapWord* limit_bitmap = MIN2(limit, tams); |
389 |
389 |
390 // Try to scan the initial candidate. If the candidate is above the TAMS, it would |
390 // Try to scan the initial candidate. If the candidate is above the TAMS, it would |
410 |
410 |
411 int avail; |
411 int avail; |
412 do { |
412 do { |
413 avail = 0; |
413 avail = 0; |
414 for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) { |
414 for (int c = 0; (c < dist) && (cb < limit_bitmap); c++) { |
415 Prefetch::read(cb, ShenandoahBrooksPointer::byte_offset()); |
415 Prefetch::read(cb, ShenandoahForwarding::byte_offset()); |
416 slots[avail++] = cb; |
416 slots[avail++] = cb; |
417 cb += skip_bitmap_delta; |
417 cb += skip_bitmap_delta; |
418 if (cb < limit_bitmap) { |
418 if (cb < limit_bitmap) { |
419 cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap); |
419 cb = mark_bit_map->get_next_marked_addr(cb, limit_bitmap); |
420 } |
420 } |
445 } |
445 } |
446 |
446 |
447 // Step 2. Accurate size-based traversal, happens past the TAMS. |
447 // Step 2. Accurate size-based traversal, happens past the TAMS. |
448 // This restarts the scan at TAMS, which makes sure we traverse all objects, |
448 // This restarts the scan at TAMS, which makes sure we traverse all objects, |
449 // regardless of what happened at Step 1. |
449 // regardless of what happened at Step 1. |
450 HeapWord* cs = tams + ShenandoahBrooksPointer::word_size(); |
450 HeapWord* cs = tams + ShenandoahForwarding::word_size(); |
451 while (cs < limit) { |
451 while (cs < limit) { |
452 assert (cs > tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams)); |
452 assert (cs > tams, "only objects past TAMS here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(tams)); |
453 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit)); |
453 assert (cs < limit, "only objects below limit here: " PTR_FORMAT " (" PTR_FORMAT ")", p2i(cs), p2i(limit)); |
454 oop obj = oop(cs); |
454 oop obj = oop(cs); |
455 assert(oopDesc::is_oop(obj), "sanity"); |
455 assert(oopDesc::is_oop(obj), "sanity"); |