|
1 /* |
|
2 * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. |
|
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
|
4 * |
|
5 * This code is free software; you can redistribute it and/or modify it |
|
6 * under the terms of the GNU General Public License version 2 only, as |
|
7 * published by the Free Software Foundation. |
|
8 * |
|
9 * This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 * version 2 for more details (a copy is included in the LICENSE file that |
|
13 * accompanied this code). |
|
14 * |
|
15 * You should have received a copy of the GNU General Public License version |
|
16 * 2 along with this work; if not, write to the Free Software Foundation, |
|
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 * |
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 * or visit www.oracle.com if you need additional information or have any |
|
21 * questions. |
|
22 */ |
|
23 |
|
24 #include "precompiled.hpp" |
|
25 #include "gc/z/zLock.inline.hpp" |
|
26 #include "gc/z/zMarkStack.inline.hpp" |
|
27 #include "gc/z/zMarkStackAllocator.hpp" |
|
28 #include "logging/log.hpp" |
|
29 #include "runtime/atomic.hpp" |
|
30 #include "runtime/os.hpp" |
|
31 #include "utilities/debug.hpp" |
|
32 |
|
33 uintptr_t ZMarkStackSpaceStart; |
|
34 |
|
35 ZMarkStackSpace::ZMarkStackSpace() : |
|
36 _expand_lock(), |
|
37 _start(0), |
|
38 _top(0), |
|
39 _end(0) { |
|
40 assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); |
|
41 |
|
42 // Reserve address space |
|
43 const size_t size = ZMarkStackSpaceLimit; |
|
44 const size_t alignment = (size_t)os::vm_allocation_granularity(); |
|
45 const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); |
|
46 if (addr == 0) { |
|
47 log_error(gc, marking)("Failed to reserve address space for mark stacks"); |
|
48 return; |
|
49 } |
|
50 |
|
51 // Successfully initialized |
|
52 _start = _top = _end = addr; |
|
53 |
|
54 // Register mark stack space start |
|
55 ZMarkStackSpaceStart = _start; |
|
56 } |
|
57 |
|
58 bool ZMarkStackSpace::is_initialized() const { |
|
59 return _start != 0; |
|
60 } |
|
61 |
|
62 uintptr_t ZMarkStackSpace::alloc_space(size_t size) { |
|
63 uintptr_t top = Atomic::load(&_top); |
|
64 |
|
65 for (;;) { |
|
66 const uintptr_t end = Atomic::load(&_end); |
|
67 const uintptr_t new_top = top + size; |
|
68 if (new_top > end) { |
|
69 // Not enough space left |
|
70 return 0; |
|
71 } |
|
72 |
|
73 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top); |
|
74 if (prev_top == top) { |
|
75 // Success |
|
76 return top; |
|
77 } |
|
78 |
|
79 // Retry |
|
80 top = prev_top; |
|
81 } |
|
82 } |
|
83 |
|
84 uintptr_t ZMarkStackSpace::expand_and_alloc_space(size_t size) { |
|
85 ZLocker locker(&_expand_lock); |
|
86 |
|
87 // Retry allocation before expanding |
|
88 uintptr_t addr = alloc_space(size); |
|
89 if (addr != 0) { |
|
90 return addr; |
|
91 } |
|
92 |
|
93 // Check expansion limit |
|
94 const size_t expand_size = ZMarkStackSpaceExpandSize; |
|
95 const size_t old_size = _end - _start; |
|
96 const size_t new_size = old_size + expand_size; |
|
97 if (new_size > ZMarkStackSpaceLimit) { |
|
98 // Expansion limit reached. This is a fatal error since we |
|
99 // currently can't recover from running out of mark stack space. |
|
100 fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the " |
|
101 "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", |
|
102 ZMarkStackSpaceLimit / M); |
|
103 } |
|
104 |
|
105 log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", |
|
106 old_size / M, new_size / M); |
|
107 |
|
108 // Expand |
|
109 os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); |
|
110 |
|
111 // Increment top before end to make sure another |
|
112 // thread can't steal out newly expanded space. |
|
113 addr = Atomic::add(size, &_top) - size; |
|
114 Atomic::add(expand_size, &_end); |
|
115 |
|
116 return addr; |
|
117 } |
|
118 |
|
119 uintptr_t ZMarkStackSpace::alloc(size_t size) { |
|
120 const uintptr_t addr = alloc_space(size); |
|
121 if (addr != 0) { |
|
122 return addr; |
|
123 } |
|
124 |
|
125 return expand_and_alloc_space(size); |
|
126 } |
|
127 |
|
128 ZMarkStackAllocator::ZMarkStackAllocator() : |
|
129 _freelist(), |
|
130 _space() { |
|
131 guarantee(sizeof(ZMarkStack) == ZMarkStackSize, "Size mismatch"); |
|
132 guarantee(sizeof(ZMarkStackMagazine) <= ZMarkStackSize, "Size mismatch"); |
|
133 |
|
134 // Prime free list to avoid an immediate space |
|
135 // expansion when marking starts. |
|
136 if (_space.is_initialized()) { |
|
137 prime_freelist(); |
|
138 } |
|
139 } |
|
140 |
|
141 bool ZMarkStackAllocator::is_initialized() const { |
|
142 return _space.is_initialized(); |
|
143 } |
|
144 |
|
145 void ZMarkStackAllocator::prime_freelist() { |
|
146 for (size_t size = 0; size < ZMarkStackSpaceExpandSize; size += ZMarkStackMagazineSize) { |
|
147 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize); |
|
148 ZMarkStackMagazine* const magazine = create_magazine_from_space(addr, ZMarkStackMagazineSize); |
|
149 free_magazine(magazine); |
|
150 } |
|
151 } |
|
152 |
|
153 ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) { |
|
154 assert(is_aligned(size, ZMarkStackSize), "Invalid size"); |
|
155 |
|
156 // Use first stack as magazine |
|
157 ZMarkStackMagazine* const magazine = new ((void*)addr) ZMarkStackMagazine(); |
|
158 for (size_t i = ZMarkStackSize; i < size; i += ZMarkStackSize) { |
|
159 ZMarkStack* const stack = new ((void*)(addr + i)) ZMarkStack(); |
|
160 const bool success = magazine->push(stack); |
|
161 assert(success, "Magazine should never get full"); |
|
162 } |
|
163 |
|
164 return magazine; |
|
165 } |
|
166 |
|
167 ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() { |
|
168 // Try allocating from the free list first |
|
169 ZMarkStackMagazine* const magazine = _freelist.pop_atomic(); |
|
170 if (magazine != NULL) { |
|
171 return magazine; |
|
172 } |
|
173 |
|
174 // Allocate new magazine |
|
175 const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize); |
|
176 if (addr == 0) { |
|
177 return NULL; |
|
178 } |
|
179 |
|
180 return create_magazine_from_space(addr, ZMarkStackMagazineSize); |
|
181 } |
|
182 |
|
183 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) { |
|
184 _freelist.push_atomic(magazine); |
|
185 } |