|
1 /* |
|
2 * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. |
|
3 * |
|
4 * This code is free software; you can redistribute it and/or modify it |
|
5 * under the terms of the GNU General Public License version 2 only, as |
|
6 * published by the Free Software Foundation. |
|
7 * |
|
8 * This code is distributed in the hope that it will be useful, but WITHOUT |
|
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
11 * version 2 for more details (a copy is included in the LICENSE file that |
|
12 * accompanied this code). |
|
13 * |
|
14 * You should have received a copy of the GNU General Public License version |
|
15 * 2 along with this work; if not, write to the Free Software Foundation, |
|
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
17 * |
|
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
19 * or visit www.oracle.com if you need additional information or have any |
|
20 * questions. |
|
21 * |
|
22 */ |
|
23 |
|
24 #include "precompiled.hpp" |
|
25 |
|
26 #include "gc/shenandoah/shenandoahAsserts.hpp" |
|
27 #include "gc/shenandoah/shenandoahBrooksPointer.hpp" |
|
28 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" |
|
29 #include "gc/shenandoah/shenandoahHeap.hpp" |
|
30 #include "gc/shenandoah/shenandoahHeap.inline.hpp" |
|
31 #include "gc/shenandoah/shenandoahRootProcessor.hpp" |
|
32 #include "gc/shenandoah/shenandoahTaskqueue.hpp" |
|
33 #include "gc/shenandoah/shenandoahUtils.hpp" |
|
34 #include "gc/shenandoah/shenandoahVerifier.hpp" |
|
35 #include "memory/allocation.hpp" |
|
36 #include "memory/iterator.inline.hpp" |
|
37 #include "memory/resourceArea.hpp" |
|
38 |
|
39 // Avoid name collision on verify_oop (defined in macroAssembler_arm.hpp) |
|
40 #ifdef verify_oop |
|
41 #undef verify_oop |
|
42 #endif |
|
43 |
|
44 class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { |
|
45 private: |
|
46 const char* _phase; |
|
47 ShenandoahVerifier::VerifyOptions _options; |
|
48 ShenandoahVerifierStack* _stack; |
|
49 ShenandoahHeap* _heap; |
|
50 MarkBitMap* _map; |
|
51 ShenandoahLivenessData* _ld; |
|
52 void* _interior_loc; |
|
53 oop _loc; |
|
54 |
|
55 public: |
|
56 ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, |
|
57 const char* phase, ShenandoahVerifier::VerifyOptions options) : |
|
58 _phase(phase), |
|
59 _options(options), |
|
60 _stack(stack), |
|
61 _heap(ShenandoahHeap::heap()), |
|
62 _map(map), |
|
63 _ld(ld), |
|
64 _interior_loc(NULL), |
|
65 _loc(NULL) { } |
|
66 |
|
67 private: |
|
68 void check(ShenandoahAsserts::SafeLevel level, oop obj, bool test, const char* label) { |
|
69 if (!test) { |
|
70 ShenandoahAsserts::print_failure(level, obj, _interior_loc, _loc, _phase, label, __FILE__, __LINE__); |
|
71 } |
|
72 } |
|
73 |
|
74 template <class T> |
|
75 void do_oop_work(T* p) { |
|
76 T o = RawAccess<>::oop_load(p); |
|
77 if (!CompressedOops::is_null(o)) { |
|
78 oop obj = CompressedOops::decode_not_null(o); |
|
79 |
|
80 // Single threaded verification can use faster non-atomic stack and bitmap |
|
81 // methods. |
|
82 // |
|
83 // For performance reasons, only fully verify non-marked field values. |
|
84 // We are here when the host object for *p is already marked. |
|
85 |
|
86 HeapWord* addr = (HeapWord*) obj; |
|
87 if (_map->par_mark(addr)) { |
|
88 verify_oop_at(p, obj); |
|
89 _stack->push(ShenandoahVerifierTask(obj)); |
|
90 } |
|
91 } |
|
92 } |
|
93 |
|
94 void verify_oop(oop obj) { |
|
95 // Perform consistency checks with gradually decreasing safety level. This guarantees |
|
96 // that failure report would not try to touch something that was not yet verified to be |
|
97 // safe to process. |
|
98 |
|
99 check(ShenandoahAsserts::_safe_unknown, obj, _heap->is_in(obj), |
|
100 "oop must be in heap"); |
|
101 check(ShenandoahAsserts::_safe_unknown, obj, check_obj_alignment(obj), |
|
102 "oop must be aligned"); |
|
103 |
|
104 ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); |
|
105 Klass* obj_klass = obj->klass_or_null(); |
|
106 |
|
107 // Verify that obj is not in dead space: |
|
108 { |
|
109 // Do this before touching obj->size() |
|
110 check(ShenandoahAsserts::_safe_unknown, obj, obj_klass != NULL, |
|
111 "Object klass pointer should not be NULL"); |
|
112 check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), |
|
113 "Object klass pointer must go to metaspace"); |
|
114 |
|
115 HeapWord *obj_addr = (HeapWord *) obj; |
|
116 check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), |
|
117 "Object start should be within the region"); |
|
118 |
|
119 if (!obj_reg->is_humongous()) { |
|
120 check(ShenandoahAsserts::_safe_unknown, obj, (obj_addr + obj->size()) <= obj_reg->top(), |
|
121 "Object end should be within the region"); |
|
122 } else { |
|
123 size_t humongous_start = obj_reg->region_number(); |
|
124 size_t humongous_end = humongous_start + (obj->size() >> ShenandoahHeapRegion::region_size_words_shift()); |
|
125 for (size_t idx = humongous_start + 1; idx < humongous_end; idx++) { |
|
126 check(ShenandoahAsserts::_safe_unknown, obj, _heap->get_region(idx)->is_humongous_continuation(), |
|
127 "Humongous object is in continuation that fits it"); |
|
128 } |
|
129 } |
|
130 |
|
131 // ------------ obj is safe at this point -------------- |
|
132 |
|
133 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->is_active(), |
|
134 "Object should be in active region"); |
|
135 |
|
136 switch (_options._verify_liveness) { |
|
137 case ShenandoahVerifier::_verify_liveness_disable: |
|
138 // skip |
|
139 break; |
|
140 case ShenandoahVerifier::_verify_liveness_complete: |
|
141 Atomic::add(obj->size() + ShenandoahBrooksPointer::word_size(), &_ld[obj_reg->region_number()]); |
|
142 // fallthrough for fast failure for un-live regions: |
|
143 case ShenandoahVerifier::_verify_liveness_conservative: |
|
144 check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), |
|
145 "Object must belong to region with live data"); |
|
146 break; |
|
147 default: |
|
148 assert(false, "Unhandled liveness verification"); |
|
149 } |
|
150 } |
|
151 |
|
152 oop fwd = (oop) ShenandoahBrooksPointer::get_raw_unchecked(obj); |
|
153 |
|
154 ShenandoahHeapRegion* fwd_reg = NULL; |
|
155 |
|
156 if (!oopDesc::equals_raw(obj, fwd)) { |
|
157 check(ShenandoahAsserts::_safe_oop, obj, _heap->is_in(fwd), |
|
158 "Forwardee must be in heap"); |
|
159 check(ShenandoahAsserts::_safe_oop, obj, !CompressedOops::is_null(fwd), |
|
160 "Forwardee is set"); |
|
161 check(ShenandoahAsserts::_safe_oop, obj, check_obj_alignment(fwd), |
|
162 "Forwardee must be aligned"); |
|
163 |
|
164 // Do this before touching fwd->size() |
|
165 Klass* fwd_klass = fwd->klass_or_null(); |
|
166 check(ShenandoahAsserts::_safe_oop, obj, fwd_klass != NULL, |
|
167 "Forwardee klass pointer should not be NULL"); |
|
168 check(ShenandoahAsserts::_safe_oop, obj, Metaspace::contains(fwd_klass), |
|
169 "Forwardee klass pointer must go to metaspace"); |
|
170 check(ShenandoahAsserts::_safe_oop, obj, obj_klass == fwd_klass, |
|
171 "Forwardee klass pointer must go to metaspace"); |
|
172 |
|
173 fwd_reg = _heap->heap_region_containing(fwd); |
|
174 |
|
175 // Verify that forwardee is not in the dead space: |
|
176 check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), |
|
177 "Should have no humongous forwardees"); |
|
178 |
|
179 HeapWord *fwd_addr = (HeapWord *) fwd; |
|
180 check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), |
|
181 "Forwardee start should be within the region"); |
|
182 check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), |
|
183 "Forwardee end should be within the region"); |
|
184 |
|
185 oop fwd2 = (oop) ShenandoahBrooksPointer::get_raw_unchecked(fwd); |
|
186 check(ShenandoahAsserts::_safe_oop, obj, oopDesc::equals_raw(fwd, fwd2), |
|
187 "Double forwarding"); |
|
188 } else { |
|
189 fwd_reg = obj_reg; |
|
190 } |
|
191 |
|
192 // ------------ obj and fwd are safe at this point -------------- |
|
193 |
|
194 switch (_options._verify_marked) { |
|
195 case ShenandoahVerifier::_verify_marked_disable: |
|
196 // skip |
|
197 break; |
|
198 case ShenandoahVerifier::_verify_marked_incomplete: |
|
199 check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj), |
|
200 "Must be marked in incomplete bitmap"); |
|
201 break; |
|
202 case ShenandoahVerifier::_verify_marked_complete: |
|
203 check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj), |
|
204 "Must be marked in complete bitmap"); |
|
205 break; |
|
206 default: |
|
207 assert(false, "Unhandled mark verification"); |
|
208 } |
|
209 |
|
210 switch (_options._verify_forwarded) { |
|
211 case ShenandoahVerifier::_verify_forwarded_disable: |
|
212 // skip |
|
213 break; |
|
214 case ShenandoahVerifier::_verify_forwarded_none: { |
|
215 check(ShenandoahAsserts::_safe_all, obj, oopDesc::equals_raw(obj, fwd), |
|
216 "Should not be forwarded"); |
|
217 break; |
|
218 } |
|
219 case ShenandoahVerifier::_verify_forwarded_allow: { |
|
220 if (!oopDesc::equals_raw(obj, fwd)) { |
|
221 check(ShenandoahAsserts::_safe_all, obj, obj_reg != fwd_reg, |
|
222 "Forwardee should be in another region"); |
|
223 } |
|
224 break; |
|
225 } |
|
226 default: |
|
227 assert(false, "Unhandled forwarding verification"); |
|
228 } |
|
229 |
|
230 switch (_options._verify_cset) { |
|
231 case ShenandoahVerifier::_verify_cset_disable: |
|
232 // skip |
|
233 break; |
|
234 case ShenandoahVerifier::_verify_cset_none: |
|
235 check(ShenandoahAsserts::_safe_all, obj, !_heap->in_collection_set(obj), |
|
236 "Should not have references to collection set"); |
|
237 break; |
|
238 case ShenandoahVerifier::_verify_cset_forwarded: |
|
239 if (_heap->in_collection_set(obj)) { |
|
240 check(ShenandoahAsserts::_safe_all, obj, !oopDesc::equals_raw(obj, fwd), |
|
241 "Object in collection set, should have forwardee"); |
|
242 } |
|
243 break; |
|
244 default: |
|
245 assert(false, "Unhandled cset verification"); |
|
246 } |
|
247 |
|
248 } |
|
249 |
|
250 public: |
|
251 /** |
|
252 * Verify object with known interior reference. |
|
253 * @param p interior reference where the object is referenced from; can be off-heap |
|
254 * @param obj verified object |
|
255 */ |
|
256 template <class T> |
|
257 void verify_oop_at(T* p, oop obj) { |
|
258 _interior_loc = p; |
|
259 verify_oop(obj); |
|
260 _interior_loc = NULL; |
|
261 } |
|
262 |
|
263 /** |
|
264 * Verify object without known interior reference. |
|
265 * Useful when picking up the object at known offset in heap, |
|
266 * but without knowing what objects reference it. |
|
267 * @param obj verified object |
|
268 */ |
|
269 void verify_oop_standalone(oop obj) { |
|
270 _interior_loc = NULL; |
|
271 verify_oop(obj); |
|
272 _interior_loc = NULL; |
|
273 } |
|
274 |
|
275 /** |
|
276 * Verify oop fields from this object. |
|
277 * @param obj host object for verified fields |
|
278 */ |
|
279 void verify_oops_from(oop obj) { |
|
280 _loc = obj; |
|
281 obj->oop_iterate(this); |
|
282 _loc = NULL; |
|
283 } |
|
284 |
|
285 virtual void do_oop(oop* p) { do_oop_work(p); } |
|
286 virtual void do_oop(narrowOop* p) { do_oop_work(p); } |
|
287 }; |
|
288 |
|
289 class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { |
|
290 private: |
|
291 size_t _used, _committed, _garbage; |
|
292 public: |
|
293 ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; |
|
294 |
|
295 void heap_region_do(ShenandoahHeapRegion* r) { |
|
296 _used += r->used(); |
|
297 _garbage += r->garbage(); |
|
298 _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; |
|
299 } |
|
300 |
|
301 size_t used() { return _used; } |
|
302 size_t committed() { return _committed; } |
|
303 size_t garbage() { return _garbage; } |
|
304 }; |
|
305 |
|
306 class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { |
|
307 private: |
|
308 ShenandoahHeap* _heap; |
|
309 const char* _phase; |
|
310 ShenandoahVerifier::VerifyRegions _regions; |
|
311 public: |
|
312 ShenandoahVerifyHeapRegionClosure(const char* phase, ShenandoahVerifier::VerifyRegions regions) : |
|
313 _heap(ShenandoahHeap::heap()), |
|
314 _phase(phase), |
|
315 _regions(regions) {}; |
|
316 |
|
317 void print_failure(ShenandoahHeapRegion* r, const char* label) { |
|
318 ResourceMark rm; |
|
319 |
|
320 ShenandoahMessageBuffer msg("Shenandoah verification failed; %s: %s\n\n", _phase, label); |
|
321 |
|
322 stringStream ss; |
|
323 r->print_on(&ss); |
|
324 msg.append("%s", ss.as_string()); |
|
325 |
|
326 report_vm_error(__FILE__, __LINE__, msg.buffer()); |
|
327 } |
|
328 |
|
329 void verify(ShenandoahHeapRegion* r, bool test, const char* msg) { |
|
330 if (!test) { |
|
331 print_failure(r, msg); |
|
332 } |
|
333 } |
|
334 |
|
335 void heap_region_do(ShenandoahHeapRegion* r) { |
|
336 switch (_regions) { |
|
337 case ShenandoahVerifier::_verify_regions_disable: |
|
338 break; |
|
339 case ShenandoahVerifier::_verify_regions_notrash: |
|
340 verify(r, !r->is_trash(), |
|
341 "Should not have trash regions"); |
|
342 break; |
|
343 case ShenandoahVerifier::_verify_regions_nocset: |
|
344 verify(r, !r->is_cset(), |
|
345 "Should not have cset regions"); |
|
346 break; |
|
347 case ShenandoahVerifier::_verify_regions_notrash_nocset: |
|
348 verify(r, !r->is_trash(), |
|
349 "Should not have trash regions"); |
|
350 verify(r, !r->is_cset(), |
|
351 "Should not have cset regions"); |
|
352 break; |
|
353 default: |
|
354 ShouldNotReachHere(); |
|
355 } |
|
356 |
|
357 verify(r, r->capacity() == ShenandoahHeapRegion::region_size_bytes(), |
|
358 "Capacity should match region size"); |
|
359 |
|
360 verify(r, r->bottom() <= r->top(), |
|
361 "Region top should not be less than bottom"); |
|
362 |
|
363 verify(r, r->bottom() <= _heap->marking_context()->top_at_mark_start(r), |
|
364 "Region TAMS should not be less than bottom"); |
|
365 |
|
366 verify(r, _heap->marking_context()->top_at_mark_start(r) <= r->top(), |
|
367 "Complete TAMS should not be larger than top"); |
|
368 |
|
369 verify(r, r->get_live_data_bytes() <= r->capacity(), |
|
370 "Live data cannot be larger than capacity"); |
|
371 |
|
372 verify(r, r->garbage() <= r->capacity(), |
|
373 "Garbage cannot be larger than capacity"); |
|
374 |
|
375 verify(r, r->used() <= r->capacity(), |
|
376 "Used cannot be larger than capacity"); |
|
377 |
|
378 verify(r, r->get_shared_allocs() <= r->capacity(), |
|
379 "Shared alloc count should not be larger than capacity"); |
|
380 |
|
381 verify(r, r->get_tlab_allocs() <= r->capacity(), |
|
382 "TLAB alloc count should not be larger than capacity"); |
|
383 |
|
384 verify(r, r->get_gclab_allocs() <= r->capacity(), |
|
385 "GCLAB alloc count should not be larger than capacity"); |
|
386 |
|
387 verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), |
|
388 "Accurate accounting: shared + TLAB + GCLAB = used"); |
|
389 |
|
390 verify(r, !r->is_empty() || !r->has_live(), |
|
391 "Empty regions should not have live data"); |
|
392 |
|
393 verify(r, r->is_cset() == _heap->collection_set()->is_in(r), |
|
394 "Transitional: region flags and collection set agree"); |
|
395 |
|
396 verify(r, r->is_empty() || r->seqnum_first_alloc() != 0, |
|
397 "Non-empty regions should have first seqnum set"); |
|
398 |
|
399 verify(r, r->is_empty() || (r->seqnum_first_alloc_mutator() != 0 || r->seqnum_first_alloc_gc() != 0), |
|
400 "Non-empty regions should have first seqnum set to either GC or mutator"); |
|
401 |
|
402 verify(r, r->is_empty() || r->seqnum_last_alloc() != 0, |
|
403 "Non-empty regions should have last seqnum set"); |
|
404 |
|
405 verify(r, r->is_empty() || (r->seqnum_last_alloc_mutator() != 0 || r->seqnum_last_alloc_gc() != 0), |
|
406 "Non-empty regions should have last seqnum set to either GC or mutator"); |
|
407 |
|
408 verify(r, r->seqnum_first_alloc() <= r->seqnum_last_alloc(), |
|
409 "First seqnum should not be greater than last timestamp"); |
|
410 |
|
411 verify(r, r->seqnum_first_alloc_mutator() <= r->seqnum_last_alloc_mutator(), |
|
412 "First mutator seqnum should not be greater than last seqnum"); |
|
413 |
|
414 verify(r, r->seqnum_first_alloc_gc() <= r->seqnum_last_alloc_gc(), |
|
415 "First GC seqnum should not be greater than last seqnum"); |
|
416 } |
|
417 }; |
|
418 |
|
419 class ShenandoahVerifierReachableTask : public AbstractGangTask { |
|
420 private: |
|
421 const char* _label; |
|
422 ShenandoahRootProcessor* _rp; |
|
423 ShenandoahVerifier::VerifyOptions _options; |
|
424 ShenandoahHeap* _heap; |
|
425 ShenandoahLivenessData* _ld; |
|
426 MarkBitMap* _bitmap; |
|
427 volatile size_t _processed; |
|
428 |
|
429 public: |
|
430 ShenandoahVerifierReachableTask(MarkBitMap* bitmap, |
|
431 ShenandoahLivenessData* ld, |
|
432 ShenandoahRootProcessor* rp, |
|
433 const char* label, |
|
434 ShenandoahVerifier::VerifyOptions options) : |
|
435 AbstractGangTask("Shenandoah Parallel Verifier Reachable Task"), |
|
436 _label(label), |
|
437 _rp(rp), |
|
438 _options(options), |
|
439 _heap(ShenandoahHeap::heap()), |
|
440 _ld(ld), |
|
441 _bitmap(bitmap), |
|
442 _processed(0) {}; |
|
443 |
|
444 size_t processed() { |
|
445 return _processed; |
|
446 } |
|
447 |
|
448 virtual void work(uint worker_id) { |
|
449 ResourceMark rm; |
|
450 ShenandoahVerifierStack stack; |
|
451 |
|
452 // On level 2, we need to only check the roots once. |
|
453 // On level 3, we want to check the roots, and seed the local stack. |
|
454 // It is a lesser evil to accept multiple root scans at level 3, because |
|
455 // extended parallelism would buy us out. |
|
456 if (((ShenandoahVerifyLevel == 2) && (worker_id == 0)) |
|
457 || (ShenandoahVerifyLevel >= 3)) { |
|
458 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, |
|
459 ShenandoahMessageBuffer("%s, Roots", _label), |
|
460 _options); |
|
461 _rp->process_all_roots_slow(&cl); |
|
462 } |
|
463 |
|
464 size_t processed = 0; |
|
465 |
|
466 if (ShenandoahVerifyLevel >= 3) { |
|
467 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, |
|
468 ShenandoahMessageBuffer("%s, Reachable", _label), |
|
469 _options); |
|
470 while (!stack.is_empty()) { |
|
471 processed++; |
|
472 ShenandoahVerifierTask task = stack.pop(); |
|
473 cl.verify_oops_from(task.obj()); |
|
474 } |
|
475 } |
|
476 |
|
477 Atomic::add(processed, &_processed); |
|
478 } |
|
479 }; |
|
480 |
|
481 class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { |
|
482 private: |
|
483 const char* _label; |
|
484 ShenandoahVerifier::VerifyOptions _options; |
|
485 ShenandoahHeap *_heap; |
|
486 MarkBitMap* _bitmap; |
|
487 ShenandoahLivenessData* _ld; |
|
488 volatile size_t _claimed; |
|
489 volatile size_t _processed; |
|
490 |
|
491 public: |
|
492 ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, |
|
493 ShenandoahLivenessData* ld, |
|
494 const char* label, |
|
495 ShenandoahVerifier::VerifyOptions options) : |
|
496 AbstractGangTask("Shenandoah Parallel Verifier Marked Region"), |
|
497 _label(label), |
|
498 _options(options), |
|
499 _heap(ShenandoahHeap::heap()), |
|
500 _bitmap(bitmap), |
|
501 _ld(ld), |
|
502 _claimed(0), |
|
503 _processed(0) {}; |
|
504 |
|
505 size_t processed() { |
|
506 return _processed; |
|
507 } |
|
508 |
|
509 virtual void work(uint worker_id) { |
|
510 ShenandoahVerifierStack stack; |
|
511 ShenandoahVerifyOopClosure cl(&stack, _bitmap, _ld, |
|
512 ShenandoahMessageBuffer("%s, Marked", _label), |
|
513 _options); |
|
514 |
|
515 while (true) { |
|
516 size_t v = Atomic::add(1u, &_claimed) - 1; |
|
517 if (v < _heap->num_regions()) { |
|
518 ShenandoahHeapRegion* r = _heap->get_region(v); |
|
519 if (!r->is_humongous() && !r->is_trash()) { |
|
520 work_regular(r, stack, cl); |
|
521 } else if (r->is_humongous_start()) { |
|
522 work_humongous(r, stack, cl); |
|
523 } |
|
524 } else { |
|
525 break; |
|
526 } |
|
527 } |
|
528 } |
|
529 |
|
530 virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { |
|
531 size_t processed = 0; |
|
532 HeapWord* obj = r->bottom() + ShenandoahBrooksPointer::word_size(); |
|
533 if (_heap->complete_marking_context()->is_marked((oop)obj)) { |
|
534 verify_and_follow(obj, stack, cl, &processed); |
|
535 } |
|
536 Atomic::add(processed, &_processed); |
|
537 } |
|
538 |
|
539 virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { |
|
540 size_t processed = 0; |
|
541 MarkBitMap* mark_bit_map = _heap->complete_marking_context()->mark_bit_map(); |
|
542 HeapWord* tams = _heap->complete_marking_context()->top_at_mark_start(r); |
|
543 |
|
544 // Bitmaps, before TAMS |
|
545 if (tams > r->bottom()) { |
|
546 HeapWord* start = r->bottom() + ShenandoahBrooksPointer::word_size(); |
|
547 HeapWord* addr = mark_bit_map->get_next_marked_addr(start, tams); |
|
548 |
|
549 while (addr < tams) { |
|
550 verify_and_follow(addr, stack, cl, &processed); |
|
551 addr += ShenandoahBrooksPointer::word_size(); |
|
552 if (addr < tams) { |
|
553 addr = mark_bit_map->get_next_marked_addr(addr, tams); |
|
554 } |
|
555 } |
|
556 } |
|
557 |
|
558 // Size-based, after TAMS |
|
559 { |
|
560 HeapWord* limit = r->top(); |
|
561 HeapWord* addr = tams + ShenandoahBrooksPointer::word_size(); |
|
562 |
|
563 while (addr < limit) { |
|
564 verify_and_follow(addr, stack, cl, &processed); |
|
565 addr += oop(addr)->size() + ShenandoahBrooksPointer::word_size(); |
|
566 } |
|
567 } |
|
568 |
|
569 Atomic::add(processed, &_processed); |
|
570 } |
|
571 |
|
572 void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { |
|
573 if (!_bitmap->par_mark(addr)) return; |
|
574 |
|
575 // Verify the object itself: |
|
576 oop obj = oop(addr); |
|
577 cl.verify_oop_standalone(obj); |
|
578 |
|
579 // Verify everything reachable from that object too, hopefully realizing |
|
580 // everything was already marked, and never touching further: |
|
581 cl.verify_oops_from(obj); |
|
582 (*processed)++; |
|
583 |
|
584 while (!stack.is_empty()) { |
|
585 ShenandoahVerifierTask task = stack.pop(); |
|
586 cl.verify_oops_from(task.obj()); |
|
587 (*processed)++; |
|
588 } |
|
589 } |
|
590 }; |
|
591 |
|
592 class VerifyThreadGCState : public ThreadClosure { |
|
593 private: |
|
594 const char* _label; |
|
595 char _expected; |
|
596 |
|
597 public: |
|
598 VerifyThreadGCState(const char* label, char expected) : _expected(expected) {} |
|
599 void do_thread(Thread* t) { |
|
600 char actual = ShenandoahThreadLocalData::gc_state(t); |
|
601 if (actual != _expected) { |
|
602 fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual); |
|
603 } |
|
604 } |
|
605 }; |
|
606 |
|
607 void ShenandoahVerifier::verify_at_safepoint(const char *label, |
|
608 VerifyForwarded forwarded, VerifyMarked marked, |
|
609 VerifyCollectionSet cset, |
|
610 VerifyLiveness liveness, VerifyRegions regions, |
|
611 VerifyGCState gcstate) { |
|
612 guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); |
|
613 guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); |
|
614 |
|
615 // Avoid side-effect of changing workers' active thread count, but bypass concurrent/parallel protocol check |
|
616 ShenandoahPushWorkerScope verify_worker_scope(_heap->workers(), _heap->max_workers(), false /*bypass check*/); |
|
617 |
|
618 log_info(gc,start)("Verify %s, Level " INTX_FORMAT, label, ShenandoahVerifyLevel); |
|
619 |
|
620 // GC state checks |
|
621 { |
|
622 char expected = -1; |
|
623 bool enabled; |
|
624 switch (gcstate) { |
|
625 case _verify_gcstate_disable: |
|
626 enabled = false; |
|
627 break; |
|
628 case _verify_gcstate_forwarded: |
|
629 enabled = true; |
|
630 expected = ShenandoahHeap::HAS_FORWARDED; |
|
631 break; |
|
632 case _verify_gcstate_stable: |
|
633 enabled = true; |
|
634 expected = ShenandoahHeap::STABLE; |
|
635 break; |
|
636 default: |
|
637 enabled = false; |
|
638 assert(false, "Unhandled gc-state verification"); |
|
639 } |
|
640 |
|
641 if (enabled) { |
|
642 char actual = _heap->gc_state(); |
|
643 if (actual != expected) { |
|
644 fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual); |
|
645 } |
|
646 |
|
647 VerifyThreadGCState vtgcs(label, expected); |
|
648 Threads::java_threads_do(&vtgcs); |
|
649 } |
|
650 } |
|
651 |
|
652 // Heap size checks |
|
653 { |
|
654 ShenandoahHeapLocker lock(_heap->lock()); |
|
655 |
|
656 ShenandoahCalculateRegionStatsClosure cl; |
|
657 _heap->heap_region_iterate(&cl); |
|
658 size_t heap_used = _heap->used(); |
|
659 guarantee(cl.used() == heap_used, |
|
660 "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "K, regions-used = " SIZE_FORMAT "K", |
|
661 label, heap_used/K, cl.used()/K); |
|
662 |
|
663 size_t heap_committed = _heap->committed(); |
|
664 guarantee(cl.committed() == heap_committed, |
|
665 "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "K, regions-committed = " SIZE_FORMAT "K", |
|
666 label, heap_committed/K, cl.committed()/K); |
|
667 } |
|
668 |
|
669 // Internal heap region checks |
|
670 if (ShenandoahVerifyLevel >= 1) { |
|
671 ShenandoahVerifyHeapRegionClosure cl(label, regions); |
|
672 _heap->heap_region_iterate(&cl); |
|
673 } |
|
674 |
|
675 OrderAccess::fence(); |
|
676 _heap->make_parsable(false); |
|
677 |
|
678 // Allocate temporary bitmap for storing marking wavefront: |
|
679 _verification_bit_map->clear(); |
|
680 |
|
681 // Allocate temporary array for storing liveness data |
|
682 ShenandoahLivenessData* ld = NEW_C_HEAP_ARRAY(ShenandoahLivenessData, _heap->num_regions(), mtGC); |
|
683 Copy::fill_to_bytes((void*)ld, _heap->num_regions()*sizeof(ShenandoahLivenessData), 0); |
|
684 |
|
685 const VerifyOptions& options = ShenandoahVerifier::VerifyOptions(forwarded, marked, cset, liveness, regions, gcstate); |
|
686 |
|
687 // Steps 1-2. Scan root set to get initial reachable set. Finish walking the reachable heap. |
|
688 // This verifies what application can see, since it only cares about reachable objects. |
|
689 size_t count_reachable = 0; |
|
690 if (ShenandoahVerifyLevel >= 2) { |
|
691 ShenandoahRootProcessor rp(_heap, _heap->workers()->active_workers(), |
|
692 ShenandoahPhaseTimings::_num_phases); // no need for stats |
|
693 |
|
694 ShenandoahVerifierReachableTask task(_verification_bit_map, ld, &rp, label, options); |
|
695 _heap->workers()->run_task(&task); |
|
696 count_reachable = task.processed(); |
|
697 } |
|
698 |
|
699 // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, |
|
700 // not the application, can see during the region scans. There is no reason to process the objects |
|
701 // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: |
|
702 // before TAMS, we verify the bitmaps, if available; after TAMS, we walk until the top(). It mimics |
|
703 // what marked_object_iterate is doing, without calling into that optimized (and possibly incorrect) |
|
704 // version |
|
705 |
|
706 size_t count_marked = 0; |
|
707 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete) { |
|
708 guarantee(_heap->marking_context()->is_complete(), "Marking context should be complete"); |
|
709 ShenandoahVerifierMarkedRegionTask task(_verification_bit_map, ld, label, options); |
|
710 _heap->workers()->run_task(&task); |
|
711 count_marked = task.processed(); |
|
712 } else { |
|
713 guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be"); |
|
714 } |
|
715 |
|
716 // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes |
|
717 // marked objects. |
|
718 |
|
719 if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { |
|
720 for (size_t i = 0; i < _heap->num_regions(); i++) { |
|
721 ShenandoahHeapRegion* r = _heap->get_region(i); |
|
722 |
|
723 juint verf_live = 0; |
|
724 if (r->is_humongous()) { |
|
725 // For humongous objects, test if start region is marked live, and if so, |
|
726 // all humongous regions in that chain have live data equal to their "used". |
|
727 juint start_live = OrderAccess::load_acquire(&ld[r->humongous_start_region()->region_number()]); |
|
728 if (start_live > 0) { |
|
729 verf_live = (juint)(r->used() / HeapWordSize); |
|
730 } |
|
731 } else { |
|
732 verf_live = OrderAccess::load_acquire(&ld[r->region_number()]); |
|
733 } |
|
734 |
|
735 size_t reg_live = r->get_live_data_words(); |
|
736 if (reg_live != verf_live) { |
|
737 ResourceMark rm; |
|
738 stringStream ss; |
|
739 r->print_on(&ss); |
|
740 fatal("%s: Live data should match: region-live = " SIZE_FORMAT ", verifier-live = " UINT32_FORMAT "\n%s", |
|
741 label, reg_live, verf_live, ss.as_string()); |
|
742 } |
|
743 } |
|
744 } |
|
745 |
|
746 log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", |
|
747 label, ShenandoahVerifyLevel, count_reachable, count_marked); |
|
748 |
|
749 FREE_C_HEAP_ARRAY(ShenandoahLivenessData, ld); |
|
750 } |
|
751 |
|
752 void ShenandoahVerifier::verify_generic(VerifyOption vo) { |
|
753 verify_at_safepoint( |
|
754 "Generic Verification", |
|
755 _verify_forwarded_allow, // conservatively allow forwarded |
|
756 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations |
|
757 _verify_cset_disable, // cset may be inconsistent |
|
758 _verify_liveness_disable, // no reliable liveness data |
|
759 _verify_regions_disable, // no reliable region data |
|
760 _verify_gcstate_disable // no data about gcstate |
|
761 ); |
|
762 } |
|
763 |
|
764 void ShenandoahVerifier::verify_before_concmark() { |
|
765 if (_heap->has_forwarded_objects()) { |
|
766 verify_at_safepoint( |
|
767 "Before Mark", |
|
768 _verify_forwarded_allow, // may have forwarded references |
|
769 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations |
|
770 _verify_cset_forwarded, // allow forwarded references to cset |
|
771 _verify_liveness_disable, // no reliable liveness data |
|
772 _verify_regions_notrash, // no trash regions |
|
773 _verify_gcstate_forwarded // there are forwarded objects |
|
774 ); |
|
775 } else { |
|
776 verify_at_safepoint( |
|
777 "Before Mark", |
|
778 _verify_forwarded_none, // UR should have fixed up |
|
779 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations |
|
780 _verify_cset_none, // UR should have fixed this |
|
781 _verify_liveness_disable, // no reliable liveness data |
|
782 _verify_regions_notrash, // no trash regions |
|
783 _verify_gcstate_stable // there are no forwarded objects |
|
784 ); |
|
785 } |
|
786 } |
|
787 |
|
788 void ShenandoahVerifier::verify_after_concmark() { |
|
789 verify_at_safepoint( |
|
790 "After Mark", |
|
791 _verify_forwarded_none, // no forwarded references |
|
792 _verify_marked_complete, // bitmaps as precise as we can get |
|
793 _verify_cset_none, // no references to cset anymore |
|
794 _verify_liveness_complete, // liveness data must be complete here |
|
795 _verify_regions_disable, // trash regions not yet recycled |
|
796 _verify_gcstate_stable // mark should have stabilized the heap |
|
797 ); |
|
798 } |
|
799 |
|
800 void ShenandoahVerifier::verify_before_evacuation() { |
|
801 verify_at_safepoint( |
|
802 "Before Evacuation", |
|
803 _verify_forwarded_none, // no forwarded references |
|
804 _verify_marked_complete, // walk over marked objects too |
|
805 _verify_cset_disable, // non-forwarded references to cset expected |
|
806 _verify_liveness_complete, // liveness data must be complete here |
|
807 _verify_regions_disable, // trash regions not yet recycled |
|
808 _verify_gcstate_stable // mark should have stabilized the heap |
|
809 ); |
|
810 } |
|
811 |
|
812 void ShenandoahVerifier::verify_after_evacuation() { |
|
813 verify_at_safepoint( |
|
814 "After Evacuation", |
|
815 _verify_forwarded_allow, // objects are still forwarded |
|
816 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well |
|
817 _verify_cset_forwarded, // all cset refs are fully forwarded |
|
818 _verify_liveness_disable, // no reliable liveness data anymore |
|
819 _verify_regions_notrash, // trash regions have been recycled already |
|
820 _verify_gcstate_forwarded // evacuation produced some forwarded objects |
|
821 ); |
|
822 } |
|
823 |
|
824 void ShenandoahVerifier::verify_before_updaterefs() { |
|
825 verify_at_safepoint( |
|
826 "Before Updating References", |
|
827 _verify_forwarded_allow, // forwarded references allowed |
|
828 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well |
|
829 _verify_cset_forwarded, // all cset refs are fully forwarded |
|
830 _verify_liveness_disable, // no reliable liveness data anymore |
|
831 _verify_regions_notrash, // trash regions have been recycled already |
|
832 _verify_gcstate_forwarded // evacuation should have produced some forwarded objects |
|
833 ); |
|
834 } |
|
835 |
|
836 void ShenandoahVerifier::verify_after_updaterefs() { |
|
837 verify_at_safepoint( |
|
838 "After Updating References", |
|
839 _verify_forwarded_none, // no forwarded references |
|
840 _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well |
|
841 _verify_cset_none, // no cset references, all updated |
|
842 _verify_liveness_disable, // no reliable liveness data anymore |
|
843 _verify_regions_nocset, // no cset regions, trash regions have appeared |
|
844 _verify_gcstate_stable // update refs had cleaned up forwarded objects |
|
845 ); |
|
846 } |
|
847 |
|
848 void ShenandoahVerifier::verify_after_degenerated() { |
|
849 verify_at_safepoint( |
|
850 "After Degenerated GC", |
|
851 _verify_forwarded_none, // all objects are non-forwarded |
|
852 _verify_marked_complete, // all objects are marked in complete bitmap |
|
853 _verify_cset_none, // no cset references |
|
854 _verify_liveness_disable, // no reliable liveness data anymore |
|
855 _verify_regions_notrash_nocset, // no trash, no cset |
|
856 _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects |
|
857 ); |
|
858 } |
|
859 |
|
860 void ShenandoahVerifier::verify_before_traversal() { |
|
861 verify_at_safepoint( |
|
862 "Before Traversal", |
|
863 _verify_forwarded_none, // cannot have forwarded objects |
|
864 _verify_marked_disable, // bitmaps are not relevant before traversal |
|
865 _verify_cset_none, // no cset references before traversal |
|
866 _verify_liveness_disable, // no reliable liveness data anymore |
|
867 _verify_regions_notrash_nocset, // no trash and no cset regions |
|
868 _verify_gcstate_stable // nothing forwarded before traversal |
|
869 ); |
|
870 } |
|
871 |
|
872 void ShenandoahVerifier::verify_after_traversal() { |
|
873 verify_at_safepoint( |
|
874 "After Traversal", |
|
875 _verify_forwarded_none, // cannot have forwarded objects |
|
876 _verify_marked_complete, // should have complete marking after traversal |
|
877 _verify_cset_none, // no cset references left after traversal |
|
878 _verify_liveness_disable, // liveness data is not collected for new allocations |
|
879 _verify_regions_nocset, // no cset regions, trash regions allowed |
|
880 _verify_gcstate_stable // nothing forwarded after traversal |
|
881 ); |
|
882 } |
|
883 |
|
884 void ShenandoahVerifier::verify_before_fullgc() { |
|
885 verify_at_safepoint( |
|
886 "Before Full GC", |
|
887 _verify_forwarded_allow, // can have forwarded objects |
|
888 _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations |
|
889 _verify_cset_disable, // cset might be foobared |
|
890 _verify_liveness_disable, // no reliable liveness data anymore |
|
891 _verify_regions_disable, // no reliable region data here |
|
892 _verify_gcstate_disable // no reliable gcstate data |
|
893 ); |
|
894 } |
|
895 |
|
896 void ShenandoahVerifier::verify_after_fullgc() { |
|
897 verify_at_safepoint( |
|
898 "After Full GC", |
|
899 _verify_forwarded_none, // all objects are non-forwarded |
|
900 _verify_marked_complete, // all objects are marked in complete bitmap |
|
901 _verify_cset_none, // no cset references |
|
902 _verify_liveness_disable, // no reliable liveness data anymore |
|
903 _verify_regions_notrash_nocset, // no trash, no cset |
|
904 _verify_gcstate_stable // full gc cleaned up everything |
|
905 ); |
|
906 } |