44 |
44 |
45 void MutableNUMASpace::mangle_unused_area() { |
45 void MutableNUMASpace::mangle_unused_area() { |
46 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
46 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
47 LGRPSpace *ls = lgrp_spaces()->at(i); |
47 LGRPSpace *ls = lgrp_spaces()->at(i); |
48 MutableSpace *s = ls->space(); |
48 MutableSpace *s = ls->space(); |
49 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
49 if (!os::numa_has_static_binding()) { |
50 if (top < s->end()) { |
50 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
51 ls->add_invalid_region(MemRegion(top, s->end())); |
51 if (top < s->end()) { |
|
52 ls->add_invalid_region(MemRegion(top, s->end())); |
|
53 } |
52 } |
54 } |
53 s->mangle_unused_area(); |
55 s->mangle_unused_area(); |
54 } |
56 } |
55 } |
57 } |
56 |
58 |
68 if (!ZapUnusedHeapArea) { |
70 if (!ZapUnusedHeapArea) { |
69 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), |
71 area_touched_words = MIN2((size_t)align_object_size(typeArrayOopDesc::header_size(T_INT)), |
70 area_touched_words); |
72 area_touched_words); |
71 } |
73 } |
72 #endif |
74 #endif |
73 MemRegion invalid; |
75 if (!os::numa_has_static_binding()) { |
74 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size()); |
76 MemRegion invalid; |
75 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), |
77 HeapWord *crossing_start = (HeapWord*)round_to((intptr_t)s->top(), os::vm_page_size()); |
76 os::vm_page_size()); |
78 HeapWord *crossing_end = (HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), |
77 if (crossing_start != crossing_end) { |
79 os::vm_page_size()); |
78 // If object header crossed a small page boundary we mark the area |
80 if (crossing_start != crossing_end) { |
79 // as invalid rounding it to a page_size(). |
81 // If object header crossed a small page boundary we mark the area |
80 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
82 // as invalid rounding it to a page_size(). |
81 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()), |
83 HeapWord *start = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
82 s->end()); |
84 HeapWord *end = MIN2((HeapWord*)round_to((intptr_t)(s->top() + area_touched_words), page_size()), |
83 invalid = MemRegion(start, end); |
85 s->end()); |
84 } |
86 invalid = MemRegion(start, end); |
85 |
87 } |
86 ls->add_invalid_region(invalid); |
88 |
|
89 ls->add_invalid_region(invalid); |
|
90 } |
87 s->set_top(s->end()); |
91 s->set_top(s->end()); |
88 } |
92 } |
89 } else { |
93 } else { |
|
94 if (!os::numa_has_static_binding()) { |
90 #ifdef ASSERT |
95 #ifdef ASSERT |
91 MemRegion invalid(s->top(), s->end()); |
|
92 ls->add_invalid_region(invalid); |
|
93 #else |
|
94 if (ZapUnusedHeapArea) { |
|
95 MemRegion invalid(s->top(), s->end()); |
96 MemRegion invalid(s->top(), s->end()); |
96 ls->add_invalid_region(invalid); |
97 ls->add_invalid_region(invalid); |
97 } else break; |
98 #else |
|
99 if (ZapUnusedHeapArea) { |
|
100 MemRegion invalid(s->top(), s->end()); |
|
101 ls->add_invalid_region(invalid); |
|
102 } else break; |
98 #endif |
103 #endif |
|
104 } |
99 } |
105 } |
100 } |
106 } |
101 } |
107 } |
102 |
108 |
103 size_t MutableNUMASpace::used_in_words() const { |
109 size_t MutableNUMASpace::used_in_words() const { |
192 } |
198 } |
193 return false; |
199 return false; |
194 } |
200 } |
195 |
201 |
196 // Bias region towards the first-touching lgrp. Set the right page sizes. |
202 // Bias region towards the first-touching lgrp. Set the right page sizes. |
197 void MutableNUMASpace::bias_region(MemRegion mr) { |
203 void MutableNUMASpace::bias_region(MemRegion mr, int lgrp_id) { |
198 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); |
204 HeapWord *start = (HeapWord*)round_to((intptr_t)mr.start(), page_size()); |
199 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); |
205 HeapWord *end = (HeapWord*)round_down((intptr_t)mr.end(), page_size()); |
200 if (end > start) { |
206 if (end > start) { |
201 MemRegion aligned_region(start, end); |
207 MemRegion aligned_region(start, end); |
202 assert((intptr_t)aligned_region.start() % page_size() == 0 && |
208 assert((intptr_t)aligned_region.start() % page_size() == 0 && |
203 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); |
209 (intptr_t)aligned_region.byte_size() % page_size() == 0, "Bad alignment"); |
204 assert(region().contains(aligned_region), "Sanity"); |
210 assert(region().contains(aligned_region), "Sanity"); |
|
211 // First we tell the OS which page size we want in the given range. The underlying |
|
212 // large page can be broken down if we require small pages. |
|
213 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
|
214 // Then we uncommit the pages in the range. |
205 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); |
215 os::free_memory((char*)aligned_region.start(), aligned_region.byte_size()); |
206 os::realign_memory((char*)aligned_region.start(), aligned_region.byte_size(), page_size()); |
216 // And make them local/first-touch biased. |
207 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size()); |
217 os::numa_make_local((char*)aligned_region.start(), aligned_region.byte_size(), lgrp_id); |
208 } |
218 } |
209 } |
219 } |
210 |
220 |
211 // Free all pages in the region. |
221 // Free all pages in the region. |
212 void MutableNUMASpace::free_region(MemRegion mr) { |
222 void MutableNUMASpace::free_region(MemRegion mr) { |
231 s->set_top(s->bottom()); |
241 s->set_top(s->bottom()); |
232 } |
242 } |
233 initialize(region(), true); |
243 initialize(region(), true); |
234 } else { |
244 } else { |
235 bool should_initialize = false; |
245 bool should_initialize = false; |
236 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
246 if (!os::numa_has_static_binding()) { |
237 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { |
247 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
238 should_initialize = true; |
248 if (!lgrp_spaces()->at(i)->invalid_region().is_empty()) { |
239 break; |
249 should_initialize = true; |
|
250 break; |
|
251 } |
240 } |
252 } |
241 } |
253 } |
242 |
254 |
243 if (should_initialize || |
255 if (should_initialize || |
244 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { |
256 (UseAdaptiveNUMAChunkSizing && adaptation_cycles() < samples_count())) { |
470 prev_page_size > page_size()) { // If the page size got smaller we have to change |
482 prev_page_size > page_size()) { // If the page size got smaller we have to change |
471 // the page size preference for the whole space. |
483 // the page size preference for the whole space. |
472 intersection = MemRegion(new_region.start(), new_region.start()); |
484 intersection = MemRegion(new_region.start(), new_region.start()); |
473 } |
485 } |
474 select_tails(new_region, intersection, &bottom_region, &top_region); |
486 select_tails(new_region, intersection, &bottom_region, &top_region); |
475 bias_region(bottom_region); |
487 bias_region(bottom_region, lgrp_spaces()->at(0)->lgrp_id()); |
476 bias_region(top_region); |
488 bias_region(top_region, lgrp_spaces()->at(lgrp_spaces()->length() - 1)->lgrp_id()); |
477 } |
489 } |
478 |
490 |
479 // Check if the space layout has changed significantly? |
491 // Check if the space layout has changed significantly? |
480 // This happens when the space has been resized so that either head or tail |
492 // This happens when the space has been resized so that either head or tail |
481 // chunk became less than a page. |
493 // chunk became less than a page. |
543 |
555 |
544 if (intersection.start() == NULL || intersection.end() == NULL) { |
556 if (intersection.start() == NULL || intersection.end() == NULL) { |
545 intersection = MemRegion(new_region.start(), new_region.start()); |
557 intersection = MemRegion(new_region.start(), new_region.start()); |
546 } |
558 } |
547 |
559 |
548 MemRegion invalid_region = ls->invalid_region().intersection(new_region); |
560 if (!os::numa_has_static_binding()) { |
549 if (!invalid_region.is_empty()) { |
561 MemRegion invalid_region = ls->invalid_region().intersection(new_region); |
550 merge_regions(new_region, &intersection, &invalid_region); |
562 // Invalid region is a range of memory that could've possibly |
551 free_region(invalid_region); |
563 // been allocated on the other node. That's relevant only on Solaris where |
552 } |
564 // there is no static memory binding. |
|
565 if (!invalid_region.is_empty()) { |
|
566 merge_regions(new_region, &intersection, &invalid_region); |
|
567 free_region(invalid_region); |
|
568 ls->set_invalid_region(MemRegion()); |
|
569 } |
|
570 } |
|
571 |
553 select_tails(new_region, intersection, &bottom_region, &top_region); |
572 select_tails(new_region, intersection, &bottom_region, &top_region); |
554 free_region(bottom_region); |
573 |
555 free_region(top_region); |
574 if (!os::numa_has_static_binding()) { |
|
575 // If that's a system with the first-touch policy then it's enough |
|
576 // to free the pages. |
|
577 free_region(bottom_region); |
|
578 free_region(top_region); |
|
579 } else { |
|
580 // In a system with static binding we have to change the bias whenever |
|
581 // we reshape the heap. |
|
582 bias_region(bottom_region, ls->lgrp_id()); |
|
583 bias_region(top_region, ls->lgrp_id()); |
|
584 } |
556 |
585 |
557 // If we clear the region, we would mangle it in debug. That would cause page |
586 // If we clear the region, we would mangle it in debug. That would cause page |
558 // allocation in a different place. Hence setting the top directly. |
587 // allocation in a different place. Hence setting the top directly. |
559 s->initialize(new_region, false); |
588 s->initialize(new_region, false); |
560 s->set_top(s->bottom()); |
589 s->set_top(s->bottom()); |
561 |
590 |
562 ls->set_invalid_region(MemRegion()); |
|
563 |
|
564 set_adaptation_cycles(samples_count()); |
591 set_adaptation_cycles(samples_count()); |
565 } |
592 } |
566 } |
593 } |
567 |
594 |
568 // Set the top of the whole space. |
595 // Set the top of the whole space. |
573 LGRPSpace *ls = lgrp_spaces()->at(i); |
600 LGRPSpace *ls = lgrp_spaces()->at(i); |
574 MutableSpace *s = ls->space(); |
601 MutableSpace *s = ls->space(); |
575 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
602 HeapWord *top = MAX2((HeapWord*)round_down((intptr_t)s->top(), page_size()), s->bottom()); |
576 |
603 |
577 if (s->contains(value)) { |
604 if (s->contains(value)) { |
578 if (top < value && top < s->end()) { |
605 if (!os::numa_has_static_binding() && top < value && top < s->end()) { |
579 ls->add_invalid_region(MemRegion(top, value)); |
606 ls->add_invalid_region(MemRegion(top, value)); |
580 } |
607 } |
581 s->set_top(value); |
608 s->set_top(value); |
582 found_top = true; |
609 found_top = true; |
583 } else { |
610 } else { |
584 if (found_top) { |
611 if (found_top) { |
585 s->set_top(s->bottom()); |
612 s->set_top(s->bottom()); |
586 } else { |
613 } else { |
587 if (top < s->end()) { |
614 if (!os::numa_has_static_binding() && top < s->end()) { |
588 ls->add_invalid_region(MemRegion(top, s->end())); |
615 ls->add_invalid_region(MemRegion(top, s->end())); |
589 } |
616 } |
590 s->set_top(s->end()); |
617 s->set_top(s->end()); |
591 } |
618 } |
592 } |
619 } |
593 } |
620 } |
594 MutableSpace::set_top(value); |
621 MutableSpace::set_top(value); |
595 } |
622 } |
599 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
626 for (int i = 0; i < lgrp_spaces()->length(); i++) { |
600 lgrp_spaces()->at(i)->space()->clear(); |
627 lgrp_spaces()->at(i)->space()->clear(); |
601 } |
628 } |
602 } |
629 } |
603 |
630 |
|
631 /* |
|
632 Linux supports static memory binding, therefore the most part of the |
|
633 logic dealing with the possible invalid page allocation is effectively |
|
634 disabled. Besides there is no notion of the home node in Linux. A |
|
635 thread is allowed to migrate freely. Although the scheduler is rather |
|
636 reluctant to move threads between the nodes. We check for the current |
|
637 node every allocation. And with a high probability a thread stays on |
|
638 the same node for some time allowing local access to recently allocated |
|
639 objects. |
|
640 */ |
|
641 |
604 HeapWord* MutableNUMASpace::allocate(size_t size) { |
642 HeapWord* MutableNUMASpace::allocate(size_t size) { |
605 int lgrp_id = Thread::current()->lgrp_id(); |
643 Thread* thr = Thread::current(); |
606 if (lgrp_id == -1) { |
644 int lgrp_id = thr->lgrp_id(); |
|
645 if (lgrp_id == -1 || !os::numa_has_group_homing()) { |
607 lgrp_id = os::numa_get_group_id(); |
646 lgrp_id = os::numa_get_group_id(); |
608 Thread::current()->set_lgrp_id(lgrp_id); |
647 thr->set_lgrp_id(lgrp_id); |
609 } |
648 } |
610 |
649 |
611 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
650 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
612 |
651 |
613 // It is possible that a new CPU has been hotplugged and |
652 // It is possible that a new CPU has been hotplugged and |
626 if (p != NULL) { |
665 if (p != NULL) { |
627 if (top() < s->top()) { // Keep _top updated. |
666 if (top() < s->top()) { // Keep _top updated. |
628 MutableSpace::set_top(s->top()); |
667 MutableSpace::set_top(s->top()); |
629 } |
668 } |
630 } |
669 } |
631 // Make the page allocation happen here. |
670 // Make the page allocation happen here if there is no static binding.. |
632 if (p != NULL) { |
671 if (p != NULL && !os::numa_has_static_binding()) { |
633 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
672 for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) { |
634 *(int*)i = 0; |
673 *(int*)i = 0; |
635 } |
674 } |
636 } |
675 } |
637 |
|
638 return p; |
676 return p; |
639 } |
677 } |
640 |
678 |
641 // This version is lock-free. |
679 // This version is lock-free. |
642 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { |
680 HeapWord* MutableNUMASpace::cas_allocate(size_t size) { |
643 int lgrp_id = Thread::current()->lgrp_id(); |
681 Thread* thr = Thread::current(); |
644 if (lgrp_id == -1) { |
682 int lgrp_id = thr->lgrp_id(); |
|
683 if (lgrp_id == -1 || !os::numa_has_group_homing()) { |
645 lgrp_id = os::numa_get_group_id(); |
684 lgrp_id = os::numa_get_group_id(); |
646 Thread::current()->set_lgrp_id(lgrp_id); |
685 thr->set_lgrp_id(lgrp_id); |
647 } |
686 } |
648 |
687 |
649 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
688 int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals); |
650 // It is possible that a new CPU has been hotplugged and |
689 // It is possible that a new CPU has been hotplugged and |
651 // we haven't reshaped the space accordingly. |
690 // we haven't reshaped the space accordingly. |