|
1 /* |
|
2 * Copyright (c) 2019, Red Hat, Inc. All rights reserved. |
|
3 * |
|
4 * This code is free software; you can redistribute it and/or modify it |
|
5 * under the terms of the GNU General Public License version 2 only, as |
|
6 * published by the Free Software Foundation. |
|
7 * |
|
8 * This code is distributed in the hope that it will be useful, but WITHOUT |
|
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
11 * version 2 for more details (a copy is included in the LICENSE file that |
|
12 * accompanied this code). |
|
13 * |
|
14 * You should have received a copy of the GNU General Public License version |
|
15 * 2 along with this work; if not, write to the Free Software Foundation, |
|
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
17 * |
|
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
19 * or visit www.oracle.com if you need additional information or have any |
|
20 * questions. |
|
21 * |
|
22 */ |
|
23 |
|
24 #include "precompiled.hpp" |
|
25 |
|
26 #include "gc/shenandoah/shenandoahClosures.inline.hpp" |
|
27 #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" |
|
28 #include "gc/shenandoah/shenandoahHeap.inline.hpp" |
|
29 #include "gc/shenandoah/shenandoahNMethod.inline.hpp" |
|
30 #include "memory/resourceArea.hpp" |
|
31 |
|
32 ShenandoahNMethod::ShenandoahNMethod(nmethod* nm, GrowableArray<oop*>& oops, bool non_immediate_oops) : |
|
33 _nm(nm), _oops(NULL), _oops_count(0), _unregistered(false) { |
|
34 |
|
35 if (!oops.is_empty()) { |
|
36 _oops_count = oops.length(); |
|
37 _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); |
|
38 for (int c = 0; c < _oops_count; c++) { |
|
39 _oops[c] = oops.at(c); |
|
40 } |
|
41 } |
|
42 _has_non_immed_oops = non_immediate_oops; |
|
43 |
|
44 assert_same_oops(); |
|
45 } |
|
46 |
|
47 ShenandoahNMethod::~ShenandoahNMethod() { |
|
48 if (_oops != NULL) { |
|
49 FREE_C_HEAP_ARRAY(oop*, _oops); |
|
50 } |
|
51 } |
|
52 |
|
53 class ShenandoahHasCSetOopClosure : public OopClosure { |
|
54 private: |
|
55 ShenandoahHeap* const _heap; |
|
56 bool _has_cset_oops; |
|
57 |
|
58 public: |
|
59 ShenandoahHasCSetOopClosure() : |
|
60 _heap(ShenandoahHeap::heap()), |
|
61 _has_cset_oops(false) { |
|
62 } |
|
63 |
|
64 bool has_cset_oops() const { |
|
65 return _has_cset_oops; |
|
66 } |
|
67 |
|
68 void do_oop(oop* p) { |
|
69 oop value = RawAccess<>::oop_load(p); |
|
70 if (!_has_cset_oops && _heap->in_collection_set(value)) { |
|
71 _has_cset_oops = true; |
|
72 } |
|
73 } |
|
74 |
|
75 void do_oop(narrowOop* p) { |
|
76 ShouldNotReachHere(); |
|
77 } |
|
78 }; |
|
79 |
|
80 bool ShenandoahNMethod::has_cset_oops(ShenandoahHeap *heap) { |
|
81 ShenandoahHasCSetOopClosure cl; |
|
82 oops_do(&cl); |
|
83 return cl.has_cset_oops(); |
|
84 } |
|
85 |
|
86 void ShenandoahNMethod::update() { |
|
87 ResourceMark rm; |
|
88 bool non_immediate_oops = false; |
|
89 GrowableArray<oop*> oops; |
|
90 |
|
91 detect_reloc_oops(nm(), oops, non_immediate_oops); |
|
92 if (oops.length() != _oops_count) { |
|
93 if (_oops != NULL) { |
|
94 FREE_C_HEAP_ARRAY(oop*, _oops); |
|
95 _oops = NULL; |
|
96 } |
|
97 |
|
98 _oops_count = oops.length(); |
|
99 if (_oops_count > 0) { |
|
100 _oops = NEW_C_HEAP_ARRAY(oop*, _oops_count, mtGC); |
|
101 } |
|
102 } |
|
103 |
|
104 for (int index = 0; index < _oops_count; index ++) { |
|
105 _oops[index] = oops.at(index); |
|
106 } |
|
107 _has_non_immed_oops = non_immediate_oops; |
|
108 |
|
109 assert_same_oops(); |
|
110 } |
|
111 |
|
112 void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) { |
|
113 for (int c = 0; c < _oops_count; c ++) { |
|
114 oops->do_oop(_oops[c]); |
|
115 } |
|
116 |
|
117 oop* const begin = _nm->oops_begin(); |
|
118 oop* const end = _nm->oops_end(); |
|
119 for (oop* p = begin; p < end; p++) { |
|
120 if (*p != Universe::non_oop_word()) { |
|
121 oops->do_oop(p); |
|
122 } |
|
123 } |
|
124 |
|
125 if (fix_relocations && _has_non_immed_oops) { |
|
126 _nm->fix_oop_relocations(); |
|
127 } |
|
128 } |
|
129 |
|
130 void ShenandoahNMethod::detect_reloc_oops(nmethod* nm, GrowableArray<oop*>& oops, bool& has_non_immed_oops) { |
|
131 has_non_immed_oops = false; |
|
132 // Find all oops relocations |
|
133 RelocIterator iter(nm); |
|
134 while (iter.next()) { |
|
135 if (iter.type() != relocInfo::oop_type) { |
|
136 // Not an oop |
|
137 continue; |
|
138 } |
|
139 |
|
140 oop_Relocation* r = iter.oop_reloc(); |
|
141 if (!r->oop_is_immediate()) { |
|
142 // Non-immediate oop found |
|
143 has_non_immed_oops = true; |
|
144 continue; |
|
145 } |
|
146 |
|
147 if (r->oop_value() != NULL) { |
|
148 // Non-NULL immediate oop found. NULL oops can safely be |
|
149 // ignored since the method will be re-registered if they |
|
150 // are later patched to be non-NULL. |
|
151 oops.push(r->oop_addr()); |
|
152 } |
|
153 } |
|
154 } |
|
155 |
|
156 ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) { |
|
157 ResourceMark rm; |
|
158 bool non_immediate_oops = false; |
|
159 GrowableArray<oop*> oops; |
|
160 |
|
161 detect_reloc_oops(nm, oops, non_immediate_oops); |
|
162 |
|
163 // No embedded oops |
|
164 if(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading() && |
|
165 oops.is_empty() && nm->oops_begin() >= nm->oops_end()) { |
|
166 return NULL; |
|
167 } |
|
168 |
|
169 return new ShenandoahNMethod(nm, oops, non_immediate_oops); |
|
170 } |
|
171 |
|
172 void ShenandoahNMethod::heal_nmethod(nmethod* nm) { |
|
173 ShenandoahNMethod* data = gc_data(nm); |
|
174 assert(data != NULL, "Sanity"); |
|
175 assert(data->lock()->owned_by_self(), "Must hold the lock"); |
|
176 |
|
177 ShenandoahEvacuateUpdateRootsClosure cl; |
|
178 data->oops_do(&cl, true /*fix relocation*/); |
|
179 } |
|
180 |
|
181 #ifdef ASSERT |
|
182 void ShenandoahNMethod::assert_alive_and_correct() { |
|
183 assert(_nm->is_alive(), "only alive nmethods here"); |
|
184 ShenandoahHeap* heap = ShenandoahHeap::heap(); |
|
185 for (int c = 0; c < _oops_count; c++) { |
|
186 oop *loc = _oops[c]; |
|
187 assert(_nm->code_contains((address) loc) || _nm->oops_contains(loc), "nmethod should contain the oop*"); |
|
188 oop o = RawAccess<>::oop_load(loc); |
|
189 shenandoah_assert_correct_except(loc, o, o == NULL || heap->is_full_gc_move_in_progress()); |
|
190 } |
|
191 |
|
192 oop* const begin = _nm->oops_begin(); |
|
193 oop* const end = _nm->oops_end(); |
|
194 for (oop* p = begin; p < end; p++) { |
|
195 if (*p != Universe::non_oop_word()) { |
|
196 oop o = RawAccess<>::oop_load(p); |
|
197 shenandoah_assert_correct_except(p, o, o == NULL || heap->is_full_gc_move_in_progress()); |
|
198 } |
|
199 } |
|
200 } |
|
201 |
|
202 class ShenandoahNMethodOopDetector : public OopClosure { |
|
203 private: |
|
204 ResourceMark rm; // For growable array allocation below. |
|
205 GrowableArray<oop*> _oops; |
|
206 |
|
207 public: |
|
208 ShenandoahNMethodOopDetector() : _oops(10) {}; |
|
209 |
|
210 void do_oop(oop* o) { |
|
211 _oops.append(o); |
|
212 } |
|
213 void do_oop(narrowOop* o) { |
|
214 fatal("NMethods should not have compressed oops embedded."); |
|
215 } |
|
216 |
|
217 GrowableArray<oop*>* oops() { |
|
218 return &_oops; |
|
219 } |
|
220 |
|
221 bool has_oops() { |
|
222 return !_oops.is_empty(); |
|
223 } |
|
224 }; |
|
225 |
|
226 void ShenandoahNMethod::assert_same_oops(bool allow_dead) { |
|
227 ShenandoahNMethodOopDetector detector; |
|
228 nm()->oops_do(&detector, allow_dead); |
|
229 |
|
230 GrowableArray<oop*>* oops = detector.oops(); |
|
231 |
|
232 assert(oops->length() == oop_count(), "Must match"); |
|
233 |
|
234 for (int index = 0; index < _oops_count; index ++) { |
|
235 assert(oops->contains(_oops[index]), "Must contain this oop"); |
|
236 } |
|
237 |
|
238 for (oop* p = nm()->oops_begin(); p < nm()->oops_end(); p ++) { |
|
239 assert(oops->contains(p), "Must contain this oop"); |
|
240 } |
|
241 } |
|
242 |
|
243 void ShenandoahNMethod::assert_no_oops(nmethod* nm, bool allow_dead) { |
|
244 ShenandoahNMethodOopDetector detector; |
|
245 nm->oops_do(&detector, allow_dead); |
|
246 assert(detector.oops()->length() == 0, "Should not have oops"); |
|
247 } |
|
248 #endif |
|
249 |
|
250 ShenandoahNMethodTable::ShenandoahNMethodTable() : |
|
251 _heap(ShenandoahHeap::heap()), |
|
252 _size(minSize), |
|
253 _index(0), |
|
254 _iteration_in_progress(false) { |
|
255 _array = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, _size, mtGC); |
|
256 } |
|
257 |
|
258 ShenandoahNMethodTable::~ShenandoahNMethodTable() { |
|
259 assert(_array != NULL, "Sanity"); |
|
260 FREE_C_HEAP_ARRAY(ShenandoahNMethod*, _array); |
|
261 } |
|
262 |
|
263 void ShenandoahNMethodTable::register_nmethod(nmethod* nm) { |
|
264 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); |
|
265 assert(_index >= 0 && _index <= _size, "Sanity"); |
|
266 |
|
267 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); |
|
268 ShenandoahReentrantLocker data_locker(data != NULL ? data->lock() : NULL); |
|
269 |
|
270 if (data != NULL) { |
|
271 assert(contain(nm), "Must have been registered"); |
|
272 data->update(); |
|
273 } else { |
|
274 data = ShenandoahNMethod::for_nmethod(nm); |
|
275 if (data == NULL) { |
|
276 assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), |
|
277 "Only possible when concurrent class unloading is off"); |
|
278 return; |
|
279 } |
|
280 ShenandoahNMethod::attach_gc_data(nm, data); |
|
281 ShenandoahLocker locker(&_lock); |
|
282 log_register_nmethod(nm); |
|
283 append(data); |
|
284 } |
|
285 // Disarm new nmethod |
|
286 ShenandoahNMethod::disarm_nmethod(nm); |
|
287 } |
|
288 |
|
289 void ShenandoahNMethodTable::unregister_nmethod(nmethod* nm) { |
|
290 assert_locked_or_safepoint(CodeCache_lock); |
|
291 |
|
292 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); |
|
293 if (data == NULL) { |
|
294 assert(!ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), |
|
295 "Only possible when concurrent class unloading is off"); |
|
296 ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); |
|
297 return; |
|
298 } |
|
299 |
|
300 if (Thread::current()->is_Code_cache_sweeper_thread()) { |
|
301 wait_until_concurrent_iteration_done(); |
|
302 } |
|
303 log_unregister_nmethod(nm); |
|
304 ShenandoahLocker locker(&_lock); |
|
305 assert(contain(nm), "Must have been registered"); |
|
306 |
|
307 ShenandoahReentrantLocker data_locker(data->lock()); |
|
308 data->mark_unregistered(); |
|
309 } |
|
310 |
|
311 void ShenandoahNMethodTable::flush_nmethod(nmethod* nm) { |
|
312 assert(CodeCache_lock->owned_by_self(), "Must have CodeCache_lock held"); |
|
313 assert(Thread::current()->is_Code_cache_sweeper_thread(), "Must from Sweep thread"); |
|
314 ShenandoahNMethod* data = ShenandoahNMethod::gc_data(nm); |
|
315 assert(data != NULL || !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), |
|
316 "Only possible when concurrent class unloading is off"); |
|
317 if (data == NULL) { |
|
318 ShenandoahNMethod::assert_no_oops(nm, true /*allow_dead*/); |
|
319 return; |
|
320 } |
|
321 |
|
322 // Can not alter the array when iteration is in progress |
|
323 wait_until_concurrent_iteration_done(); |
|
324 log_flush_nmethod(nm); |
|
325 |
|
326 ShenandoahLocker locker(&_lock); |
|
327 int idx = index_of(nm); |
|
328 assert(idx >= 0 && idx < _index, "Invalid index"); |
|
329 ShenandoahNMethod::attach_gc_data(nm, NULL); |
|
330 remove(idx); |
|
331 } |
|
332 |
|
333 bool ShenandoahNMethodTable::contain(nmethod* nm) const { |
|
334 return index_of(nm) != -1; |
|
335 } |
|
336 |
|
337 ShenandoahNMethod* ShenandoahNMethodTable::at(int index) const { |
|
338 assert(index >= 0 && index < _index, "Out of bound"); |
|
339 return _array[index]; |
|
340 } |
|
341 |
|
342 int ShenandoahNMethodTable::index_of(nmethod* nm) const { |
|
343 for (int index = 0; index < length(); index ++) { |
|
344 if (_array[index]->nm() == nm) { |
|
345 return index; |
|
346 } |
|
347 } |
|
348 return -1; |
|
349 } |
|
350 |
|
351 void ShenandoahNMethodTable::remove(int idx) { |
|
352 shenandoah_assert_locked_or_safepoint(CodeCache_lock); |
|
353 assert(!_iteration_in_progress, "Can not happen"); |
|
354 assert(_index >= 0 && _index <= _size, "Sanity"); |
|
355 |
|
356 assert(idx >= 0 && idx < _index, "Out of bound"); |
|
357 ShenandoahNMethod* snm = _array[idx]; |
|
358 |
|
359 _index --; |
|
360 _array[idx] = _array[_index]; |
|
361 |
|
362 delete snm; |
|
363 } |
|
364 |
|
365 void ShenandoahNMethodTable::wait_until_concurrent_iteration_done() { |
|
366 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); |
|
367 while (iteration_in_progress()) { |
|
368 CodeCache_lock->wait_without_safepoint_check(); |
|
369 } |
|
370 } |
|
371 |
|
372 void ShenandoahNMethodTable::append(ShenandoahNMethod* snm) { |
|
373 if (is_full()) { |
|
374 int new_size = 2 * _size; |
|
375 ShenandoahNMethod** old_table = _array; |
|
376 |
|
377 // Rebuild table and replace current one |
|
378 rebuild(new_size); |
|
379 |
|
380 // An iteration is in progress over early snapshot, |
|
381 // can not release the array until iteration is completed |
|
382 if (!iteration_in_progress()) { |
|
383 FREE_C_HEAP_ARRAY(ShenandoahNMethod*, old_table); |
|
384 } |
|
385 } |
|
386 |
|
387 _array[_index ++] = snm; |
|
388 assert(_index >= 0 && _index <= _size, "Sanity"); |
|
389 } |
|
390 |
|
391 void ShenandoahNMethodTable::rebuild(int size) { |
|
392 ShenandoahNMethod** arr = NEW_C_HEAP_ARRAY(ShenandoahNMethod*, size, mtGC); |
|
393 for (int index = 0; index < _index; index ++) { |
|
394 arr[index] = _array[index]; |
|
395 } |
|
396 _array = arr; |
|
397 _size = size; |
|
398 } |
|
399 |
|
400 ShenandoahNMethodTableSnapshot* ShenandoahNMethodTable::snapshot_for_iteration() { |
|
401 assert(!iteration_in_progress(), "Already in progress"); |
|
402 _iteration_in_progress = true; |
|
403 |
|
404 return new ShenandoahNMethodTableSnapshot(this); |
|
405 } |
|
406 |
|
407 void ShenandoahNMethodTable::finish_iteration(ShenandoahNMethodTableSnapshot* snapshot) { |
|
408 assert(iteration_in_progress(), "Why we here?"); |
|
409 assert(snapshot != NULL, "No snapshot"); |
|
410 _iteration_in_progress = false; |
|
411 |
|
412 // Table has been rebuilt during iteration, free old table |
|
413 if (snapshot->_array != _array) { |
|
414 FREE_C_HEAP_ARRAY(ShenandoahNMethod*, snapshot->_array); |
|
415 } |
|
416 delete snapshot; |
|
417 } |
|
418 |
|
419 void ShenandoahNMethodTable::log_register_nmethod(nmethod* nm) { |
|
420 LogTarget(Debug, gc, nmethod) log; |
|
421 if (!log.is_enabled()) { |
|
422 return; |
|
423 } |
|
424 |
|
425 ResourceMark rm; |
|
426 log.print("Register NMethod: %s.%s [" PTR_FORMAT "] (%s)", |
|
427 nm->method()->method_holder()->external_name(), |
|
428 nm->method()->name()->as_C_string(), |
|
429 p2i(nm), |
|
430 nm->compiler_name()); |
|
431 } |
|
432 |
|
433 void ShenandoahNMethodTable::log_unregister_nmethod(nmethod* nm) { |
|
434 LogTarget(Debug, gc, nmethod) log; |
|
435 if (!log.is_enabled()) { |
|
436 return; |
|
437 } |
|
438 |
|
439 ResourceMark rm; |
|
440 log.print("Unregister NMethod: %s.%s [" PTR_FORMAT "]", |
|
441 nm->method()->method_holder()->external_name(), |
|
442 nm->method()->name()->as_C_string(), |
|
443 p2i(nm)); |
|
444 } |
|
445 |
|
446 void ShenandoahNMethodTable::log_flush_nmethod(nmethod* nm) { |
|
447 LogTarget(Debug, gc, nmethod) log; |
|
448 if (!log.is_enabled()) { |
|
449 return; |
|
450 } |
|
451 |
|
452 ResourceMark rm; |
|
453 log.print("Flush NMethod: (" PTR_FORMAT ")", p2i(nm)); |
|
454 } |
|
455 |
|
456 #ifdef ASSERT |
|
457 void ShenandoahNMethodTable::assert_nmethods_alive_and_correct() { |
|
458 assert_locked_or_safepoint(CodeCache_lock); |
|
459 |
|
460 for (int index = 0; index < length(); index ++) { |
|
461 ShenandoahNMethod* m = _array[index]; |
|
462 // Concurrent unloading may have dead nmethods to be cleaned by sweeper |
|
463 if (m->is_unregistered()) continue; |
|
464 m->assert_alive_and_correct(); |
|
465 } |
|
466 } |
|
467 #endif |
|
468 |
|
469 ShenandoahNMethodTableSnapshot::ShenandoahNMethodTableSnapshot(ShenandoahNMethodTable* table) : |
|
470 _heap(ShenandoahHeap::heap()), _table(table), _array(table->_array), _length(table->_index), _claimed(0) { |
|
471 } |
|
472 |
|
473 void ShenandoahNMethodTableSnapshot::concurrent_nmethods_do(NMethodClosure* cl) { |
|
474 size_t stride = 256; // educated guess |
|
475 |
|
476 ShenandoahNMethod** list = _array; |
|
477 size_t max = (size_t)_length; |
|
478 while (_claimed < max) { |
|
479 size_t cur = Atomic::add(&_claimed, stride) - stride; |
|
480 size_t start = cur; |
|
481 size_t end = MIN2(cur + stride, max); |
|
482 if (start >= max) break; |
|
483 |
|
484 for (size_t idx = start; idx < end; idx++) { |
|
485 ShenandoahNMethod* data = list[idx]; |
|
486 assert(data != NULL, "Should not be NULL"); |
|
487 if (!data->is_unregistered()) { |
|
488 cl->do_nmethod(data->nm()); |
|
489 } |
|
490 } |
|
491 } |
|
492 } |
|
493 |
|
494 ShenandoahConcurrentNMethodIterator::ShenandoahConcurrentNMethodIterator(ShenandoahNMethodTable* table) : |
|
495 _table(table), _table_snapshot(NULL) { |
|
496 } |
|
497 |
|
498 void ShenandoahConcurrentNMethodIterator::nmethods_do_begin() { |
|
499 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); |
|
500 assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), |
|
501 "Only for concurrent class unloading"); |
|
502 _table_snapshot = _table->snapshot_for_iteration(); |
|
503 } |
|
504 |
|
505 void ShenandoahConcurrentNMethodIterator::nmethods_do(NMethodClosure* cl) { |
|
506 assert(_table_snapshot != NULL, "Must first call nmethod_do_begin()"); |
|
507 _table_snapshot->concurrent_nmethods_do(cl); |
|
508 } |
|
509 |
|
510 void ShenandoahConcurrentNMethodIterator::nmethods_do_end() { |
|
511 assert(CodeCache_lock->owned_by_self(), "Lock must be held"); |
|
512 assert(ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(), |
|
513 "Only for concurrent class unloading"); |
|
514 _table->finish_iteration(_table_snapshot); |
|
515 CodeCache_lock->notify_all(); |
|
516 } |