26 #include "gc/z/zGlobals.hpp" |
26 #include "gc/z/zGlobals.hpp" |
27 #include "gc/z/zLock.inline.hpp" |
27 #include "gc/z/zLock.inline.hpp" |
28 #include "gc/z/zMarkStack.inline.hpp" |
28 #include "gc/z/zMarkStack.inline.hpp" |
29 #include "logging/log.hpp" |
29 #include "logging/log.hpp" |
30 #include "runtime/atomic.hpp" |
30 #include "runtime/atomic.hpp" |
|
31 #include "runtime/os.hpp" |
31 #include "utilities/debug.hpp" |
32 #include "utilities/debug.hpp" |
32 |
33 |
33 #include <sys/mman.h> |
34 uintptr_t ZMarkStackSpaceStart; |
34 #include <sys/types.h> |
|
35 |
35 |
36 ZMarkStackSpace::ZMarkStackSpace() : |
36 ZMarkStackSpace::ZMarkStackSpace() : |
37 _expand_lock(), |
37 _expand_lock(), |
|
38 _start(0), |
38 _top(0), |
39 _top(0), |
39 _end(0) { |
40 _end(0) { |
40 assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small"); |
41 assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small"); |
41 assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large"); |
|
42 |
42 |
43 // Reserve address space |
43 // Reserve address space |
44 const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize, |
44 const size_t size = ZMarkStackSpaceLimit; |
45 PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); |
45 const size_t alignment = (size_t)os::vm_allocation_granularity(); |
46 if (res != (void*)ZMarkStackSpaceStart) { |
46 const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC); |
47 log_error(gc, marking)("Failed to reserve address space for marking stacks"); |
47 if (addr == 0) { |
|
48 log_error(gc, marking)("Failed to reserve address space for mark stacks"); |
48 return; |
49 return; |
49 } |
50 } |
50 |
51 |
51 // Successfully initialized |
52 // Successfully initialized |
52 _top = _end = ZMarkStackSpaceStart; |
53 _start = _top = _end = addr; |
|
54 |
|
55 // Register mark stack space start |
|
56 ZMarkStackSpaceStart = _start; |
53 } |
57 } |
54 |
58 |
55 bool ZMarkStackSpace::is_initialized() const { |
59 bool ZMarkStackSpace::is_initialized() const { |
56 return _top != 0; |
60 return _start != 0; |
57 } |
|
58 |
|
59 void ZMarkStackSpace::expand() { |
|
60 const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax; |
|
61 if (_end + ZMarkStackSpaceExpandSize > max) { |
|
62 // Expansion limit reached. This is a fatal error since we |
|
63 // currently can't recover from running out of mark stack space. |
|
64 fatal("Mark stack overflow (current size " SIZE_FORMAT "M, max size " SIZE_FORMAT "M)," |
|
65 " use -XX:ZMarkStacksMax=<size> to increase this limit", |
|
66 (_end - ZMarkStackSpaceStart) / M, ZMarkStacksMax / M); |
|
67 } |
|
68 |
|
69 void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize, |
|
70 PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0); |
|
71 if (res == MAP_FAILED) { |
|
72 // Failed to map memory. This is a fatal error since we |
|
73 // currently can't recover from running out of mark stack space. |
|
74 ZErrno err; |
|
75 fatal("Failed to map memory for marking stacks (%s)", err.to_string()); |
|
76 } |
|
77 } |
61 } |
78 |
62 |
79 uintptr_t ZMarkStackSpace::alloc_space(size_t size) { |
63 uintptr_t ZMarkStackSpace::alloc_space(size_t size) { |
80 uintptr_t top = _top; |
64 uintptr_t top = Atomic::load(&_top); |
81 |
65 |
82 for (;;) { |
66 for (;;) { |
|
67 const uintptr_t end = Atomic::load(&_end); |
83 const uintptr_t new_top = top + size; |
68 const uintptr_t new_top = top + size; |
84 if (new_top > _end) { |
69 if (new_top > end) { |
85 // Not enough space left |
70 // Not enough space left |
86 return 0; |
71 return 0; |
87 } |
72 } |
88 |
73 |
89 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top); |
74 const uintptr_t prev_top = Atomic::cmpxchg(new_top, &_top, top); |
104 uintptr_t addr = alloc_space(size); |
89 uintptr_t addr = alloc_space(size); |
105 if (addr != 0) { |
90 if (addr != 0) { |
106 return addr; |
91 return addr; |
107 } |
92 } |
108 |
93 |
109 // Expand stack space |
94 // Check expansion limit |
110 expand(); |
95 const size_t expand_size = ZMarkStackSpaceExpandSize; |
|
96 const size_t old_size = _end - _start; |
|
97 const size_t new_size = old_size + expand_size; |
|
98 if (new_size > ZMarkStackSpaceLimit) { |
|
99 // Expansion limit reached. This is a fatal error since we |
|
100 // currently can't recover from running out of mark stack space. |
|
101 fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the " |
|
102 "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.", |
|
103 ZMarkStackSpaceLimit / M); |
|
104 } |
111 |
105 |
112 log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", |
106 log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M", |
113 (_end - ZMarkStackSpaceStart) / M, |
107 old_size / M, new_size / M); |
114 (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M); |
108 |
|
109 // Expand |
|
110 os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space"); |
115 |
111 |
116 // Increment top before end to make sure another |
112 // Increment top before end to make sure another |
117 // thread can't steal out newly expanded space. |
113 // thread can't steal out newly expanded space. |
118 addr = Atomic::add(size, &_top) - size; |
114 addr = Atomic::add(size, &_top) - size; |
119 _end += ZMarkStackSpaceExpandSize; |
115 Atomic::add(expand_size, &_end); |
120 |
116 |
121 return addr; |
117 return addr; |
122 } |
118 } |
123 |
119 |
124 uintptr_t ZMarkStackSpace::alloc(size_t size) { |
120 uintptr_t ZMarkStackSpace::alloc(size_t size) { |