72 const HeapWord* limit) const { |
72 const HeapWord* limit) const { |
73 // First we must round addr *up* to a possible object boundary. |
73 // First we must round addr *up* to a possible object boundary. |
74 addr = (HeapWord*)align_size_up((intptr_t)addr, |
74 addr = (HeapWord*)align_size_up((intptr_t)addr, |
75 HeapWordSize << _shifter); |
75 HeapWordSize << _shifter); |
76 size_t addrOffset = heapWordToOffset(addr); |
76 size_t addrOffset = heapWordToOffset(addr); |
77 if (limit == NULL) { |
77 assert(limit != NULL, "limit must not be NULL"); |
78 limit = _bmStartWord + _bmWordSize; |
|
79 } |
|
80 size_t limitOffset = heapWordToOffset(limit); |
78 size_t limitOffset = heapWordToOffset(limit); |
81 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); |
79 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); |
82 HeapWord* nextAddr = offsetToHeapWord(nextOffset); |
80 HeapWord* nextAddr = offsetToHeapWord(nextOffset); |
83 assert(nextAddr >= addr, "get_next_one postcondition"); |
81 assert(nextAddr >= addr, "get_next_one postcondition"); |
84 assert(nextAddr == limit || isMarked(nextAddr), |
82 assert(nextAddr == limit || isMarked(nextAddr), |
85 "get_next_one postcondition"); |
83 "get_next_one postcondition"); |
86 return nextAddr; |
84 return nextAddr; |
87 } |
|
88 |
|
89 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(const HeapWord* addr, |
|
90 const HeapWord* limit) const { |
|
91 size_t addrOffset = heapWordToOffset(addr); |
|
92 if (limit == NULL) { |
|
93 limit = _bmStartWord + _bmWordSize; |
|
94 } |
|
95 size_t limitOffset = heapWordToOffset(limit); |
|
96 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); |
|
97 HeapWord* nextAddr = offsetToHeapWord(nextOffset); |
|
98 assert(nextAddr >= addr, "get_next_one postcondition"); |
|
99 assert(nextAddr == limit || !isMarked(nextAddr), |
|
100 "get_next_one postcondition"); |
|
101 return nextAddr; |
|
102 } |
|
103 |
|
104 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { |
|
105 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); |
|
106 return (int) (diff >> _shifter); |
|
107 } |
85 } |
108 |
86 |
109 #ifndef PRODUCT |
87 #ifndef PRODUCT |
110 bool CMBitMapRO::covers(MemRegion heap_rs) const { |
88 bool CMBitMapRO::covers(MemRegion heap_rs) const { |
111 // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); |
89 // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); |
209 g1h->workers()->run_task(&task); |
187 g1h->workers()->run_task(&task); |
210 guarantee(cl.complete(), "Must have completed iteration."); |
188 guarantee(cl.complete(), "Must have completed iteration."); |
211 return; |
189 return; |
212 } |
190 } |
213 |
191 |
214 void CMBitMap::markRange(MemRegion mr) { |
|
215 mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); |
|
216 assert(!mr.is_empty(), "unexpected empty region"); |
|
217 assert((offsetToHeapWord(heapWordToOffset(mr.end())) == |
|
218 ((HeapWord *) mr.end())), |
|
219 "markRange memory region end is not card aligned"); |
|
220 // convert address range into offset range |
|
221 _bm.at_put_range(heapWordToOffset(mr.start()), |
|
222 heapWordToOffset(mr.end()), true); |
|
223 } |
|
224 |
|
225 void CMBitMap::clearRange(MemRegion mr) { |
192 void CMBitMap::clearRange(MemRegion mr) { |
226 mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); |
193 mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); |
227 assert(!mr.is_empty(), "unexpected empty region"); |
194 assert(!mr.is_empty(), "unexpected empty region"); |
228 // convert address range into offset range |
195 // convert address range into offset range |
229 _bm.at_put_range(heapWordToOffset(mr.start()), |
196 _bm.at_put_range(heapWordToOffset(mr.start()), |
230 heapWordToOffset(mr.end()), false); |
197 heapWordToOffset(mr.end()), false); |
231 } |
|
232 |
|
233 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, |
|
234 HeapWord* end_addr) { |
|
235 HeapWord* start = getNextMarkedWordAddress(addr); |
|
236 start = MIN2(start, end_addr); |
|
237 HeapWord* end = getNextUnmarkedWordAddress(start); |
|
238 end = MIN2(end, end_addr); |
|
239 assert(start <= end, "Consistency check"); |
|
240 MemRegion mr(start, end); |
|
241 if (!mr.is_empty()) { |
|
242 clearRange(mr); |
|
243 } |
|
244 return mr; |
|
245 } |
198 } |
246 |
199 |
247 CMMarkStack::CMMarkStack(ConcurrentMark* cm) : |
200 CMMarkStack::CMMarkStack(ConcurrentMark* cm) : |
248 _base(NULL), _cm(cm) |
201 _base(NULL), _cm(cm) |
249 {} |
202 {} |
464 _markBitMap2(), |
417 _markBitMap2(), |
465 _parallel_marking_threads(0), |
418 _parallel_marking_threads(0), |
466 _max_parallel_marking_threads(0), |
419 _max_parallel_marking_threads(0), |
467 _sleep_factor(0.0), |
420 _sleep_factor(0.0), |
468 _marking_task_overhead(1.0), |
421 _marking_task_overhead(1.0), |
469 _cleanup_sleep_factor(0.0), |
|
470 _cleanup_task_overhead(1.0), |
|
471 _cleanup_list("Cleanup List"), |
422 _cleanup_list("Cleanup List"), |
472 _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), |
423 _region_bm((BitMap::idx_t)(g1h->max_regions()), false /* in_resource_area*/), |
473 _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >> |
424 _card_bm((g1h->reserved_region().byte_size() + CardTableModRefBS::card_size - 1) >> |
474 CardTableModRefBS::card_shift, |
425 CardTableModRefBS::card_shift, |
475 false /* in_resource_area*/), |
426 false /* in_resource_area*/), |
566 |
517 |
567 assert(ConcGCThreads > 0, "Should have been set"); |
518 assert(ConcGCThreads > 0, "Should have been set"); |
568 _parallel_marking_threads = ConcGCThreads; |
519 _parallel_marking_threads = ConcGCThreads; |
569 _max_parallel_marking_threads = _parallel_marking_threads; |
520 _max_parallel_marking_threads = _parallel_marking_threads; |
570 |
521 |
571 if (parallel_marking_threads() > 1) { |
|
572 _cleanup_task_overhead = 1.0; |
|
573 } else { |
|
574 _cleanup_task_overhead = marking_task_overhead(); |
|
575 } |
|
576 _cleanup_sleep_factor = |
|
577 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead(); |
|
578 |
|
579 #if 0 |
|
580 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads()); |
|
581 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead()); |
|
582 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor()); |
|
583 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead()); |
|
584 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor()); |
|
585 #endif |
|
586 |
|
587 _parallel_workers = new WorkGang("G1 Marker", |
522 _parallel_workers = new WorkGang("G1 Marker", |
588 _max_parallel_marking_threads, false, true); |
523 _max_parallel_marking_threads, false, true); |
589 if (_parallel_workers == NULL) { |
524 if (_parallel_workers == NULL) { |
590 vm_exit_during_initialization("Failed necessary allocation."); |
525 vm_exit_during_initialization("Failed necessary allocation."); |
591 } else { |
526 } else { |
837 } |
772 } |
838 |
773 |
839 |
774 |
840 void ConcurrentMark::checkpointRootsInitialPost() { |
775 void ConcurrentMark::checkpointRootsInitialPost() { |
841 G1CollectedHeap* g1h = G1CollectedHeap::heap(); |
776 G1CollectedHeap* g1h = G1CollectedHeap::heap(); |
842 |
|
843 // If we force an overflow during remark, the remark operation will |
|
844 // actually abort and we'll restart concurrent marking. If we always |
|
845 // force an overflow during remark we'll never actually complete the |
|
846 // marking phase. So, we initialize this here, at the start of the |
|
847 // cycle, so that at the remaining overflow number will decrease at |
|
848 // every remark and we'll eventually not need to cause one. |
|
849 force_overflow_stw()->init(); |
|
850 |
777 |
851 // Start Concurrent Marking weak-reference discovery. |
778 // Start Concurrent Marking weak-reference discovery. |
852 ReferenceProcessor* rp = g1h->ref_processor_cm(); |
779 ReferenceProcessor* rp = g1h->ref_processor_cm(); |
853 // enable ("weak") refs discovery |
780 // enable ("weak") refs discovery |
854 rp->enable_discovery(); |
781 rp->enable_discovery(); |
918 // We should be here because of an overflow. During STW we should |
845 // We should be here because of an overflow. During STW we should |
919 // not clear the overflow flag since we rely on it being true when |
846 // not clear the overflow flag since we rely on it being true when |
920 // we exit this method to abort the pause and restart concurrent |
847 // we exit this method to abort the pause and restart concurrent |
921 // marking. |
848 // marking. |
922 reset_marking_state(true /* clear_overflow */); |
849 reset_marking_state(true /* clear_overflow */); |
923 force_overflow()->update(); |
|
924 |
850 |
925 if (G1Log::fine()) { |
851 if (G1Log::fine()) { |
926 gclog_or_tty->gclog_stamp(); |
852 gclog_or_tty->gclog_stamp(); |
927 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); |
853 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); |
928 } |
854 } |
937 SuspendibleThreadSetLeaver sts_leave(concurrent()); |
863 SuspendibleThreadSetLeaver sts_leave(concurrent()); |
938 _second_overflow_barrier_sync.enter(); |
864 _second_overflow_barrier_sync.enter(); |
939 |
865 |
940 // at this point everything should be re-initialized and ready to go |
866 // at this point everything should be re-initialized and ready to go |
941 } |
867 } |
942 |
|
943 #ifndef PRODUCT |
|
944 void ForceOverflowSettings::init() { |
|
945 _num_remaining = G1ConcMarkForceOverflow; |
|
946 _force = false; |
|
947 update(); |
|
948 } |
|
949 |
|
950 void ForceOverflowSettings::update() { |
|
951 if (_num_remaining > 0) { |
|
952 _num_remaining -= 1; |
|
953 _force = true; |
|
954 } else { |
|
955 _force = false; |
|
956 } |
|
957 } |
|
958 |
|
959 bool ForceOverflowSettings::should_force() { |
|
960 if (_force) { |
|
961 _force = false; |
|
962 return true; |
|
963 } else { |
|
964 return false; |
|
965 } |
|
966 } |
|
967 #endif // !PRODUCT |
|
968 |
868 |
969 class CMConcurrentMarkingTask: public AbstractGangTask { |
869 class CMConcurrentMarkingTask: public AbstractGangTask { |
970 private: |
870 private: |
971 ConcurrentMark* _cm; |
871 ConcurrentMark* _cm; |
972 ConcurrentMarkThread* _cmt; |
872 ConcurrentMarkThread* _cmt; |
1129 // However that wouldn't be right, because it's possible that |
1029 // However that wouldn't be right, because it's possible that |
1130 // a safepoint is indeed in progress as a younger generation |
1030 // a safepoint is indeed in progress as a younger generation |
1131 // stop-the-world GC happens even as we mark in this generation. |
1031 // stop-the-world GC happens even as we mark in this generation. |
1132 |
1032 |
1133 _restart_for_overflow = false; |
1033 _restart_for_overflow = false; |
1134 force_overflow_conc()->init(); |
|
1135 |
1034 |
1136 // _g1h has _n_par_threads |
1035 // _g1h has _n_par_threads |
1137 _parallel_marking_threads = calc_parallel_marking_threads(); |
1036 _parallel_marking_threads = calc_parallel_marking_threads(); |
1138 assert(parallel_marking_threads() <= max_parallel_marking_threads(), |
1037 assert(parallel_marking_threads() <= max_parallel_marking_threads(), |
1139 "Maximum number of marking threads exceeded"); |
1038 "Maximum number of marking threads exceeded"); |
2436 |
2335 |
2437 void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { |
2336 void ConcurrentMark::clearRangePrevBitmap(MemRegion mr) { |
2438 // Note we are overriding the read-only view of the prev map here, via |
2337 // Note we are overriding the read-only view of the prev map here, via |
2439 // the cast. |
2338 // the cast. |
2440 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); |
2339 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); |
2441 } |
|
2442 |
|
2443 void ConcurrentMark::clearRangeNextBitmap(MemRegion mr) { |
|
2444 _nextMarkBitMap->clearRange(mr); |
|
2445 } |
2340 } |
2446 |
2341 |
2447 HeapRegion* |
2342 HeapRegion* |
2448 ConcurrentMark::claim_region(uint worker_id) { |
2343 ConcurrentMark::claim_region(uint worker_id) { |
2449 // "checkpoint" the finger |
2344 // "checkpoint" the finger |
3530 drain_local_queue(false); |
3425 drain_local_queue(false); |
3531 drain_global_stack(false); |
3426 drain_global_stack(false); |
3532 } else { |
3427 } else { |
3533 break; |
3428 break; |
3534 } |
3429 } |
3535 } |
|
3536 } |
|
3537 |
|
3538 // If we are about to wrap up and go into termination, check if we |
|
3539 // should raise the overflow flag. |
|
3540 if (do_termination && !has_aborted()) { |
|
3541 if (_cm->force_overflow()->should_force()) { |
|
3542 _cm->set_has_overflown(); |
|
3543 regular_clock_call(); |
|
3544 } |
3430 } |
3545 } |
3431 } |
3546 |
3432 |
3547 // We still haven't aborted. Now, let's try to get into the |
3433 // We still haven't aborted. Now, let's try to get into the |
3548 // termination protocol. |
3434 // termination protocol. |