8209375: ZGC: Use dynamic base address for mark stack space
Reviewed-by: eosterlund, kbarrett
--- a/src/hotspot/share/gc/z/zGlobals.hpp Tue Aug 14 00:15:56 2018 -0400
+++ b/src/hotspot/share/gc/z/zGlobals.hpp Mon Aug 13 14:04:42 2018 +0200
@@ -117,11 +117,8 @@
// Marked state
extern uintptr_t ZAddressMetadataMarked;
-// Address space for mark stack allocations
-const size_t ZMarkStackSpaceSizeShift = 40; // 1TB
-const size_t ZMarkStackSpaceSize = (size_t)1 << ZMarkStackSpaceSizeShift;
-const uintptr_t ZMarkStackSpaceStart = ZAddressSpaceEnd + ZMarkStackSpaceSize;
-const uintptr_t ZMarkStackSpaceEnd = ZMarkStackSpaceStart + ZMarkStackSpaceSize;
+// Mark stack space
+extern uintptr_t ZMarkStackSpaceStart;
const size_t ZMarkStackSpaceExpandSize = (size_t)1 << 25; // 32M
// Mark stack and magazine sizes
--- a/src/hotspot/share/gc/z/zMarkStack.cpp Tue Aug 14 00:15:56 2018 -0400
+++ b/src/hotspot/share/gc/z/zMarkStack.cpp Mon Aug 13 14:04:42 2018 +0200
@@ -28,60 +28,45 @@
#include "gc/z/zMarkStack.inline.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
#include "utilities/debug.hpp"
-#include <sys/mman.h>
-#include <sys/types.h>
+uintptr_t ZMarkStackSpaceStart;
ZMarkStackSpace::ZMarkStackSpace() :
_expand_lock(),
+ _start(0),
_top(0),
_end(0) {
- assert(ZMarkStacksMax >= ZMarkStackSpaceExpandSize, "ZMarkStacksMax too small");
- assert(ZMarkStacksMax <= ZMarkStackSpaceSize, "ZMarkStacksMax too large");
+ assert(ZMarkStackSpaceLimit >= ZMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
// Reserve address space
- const void* res = mmap((void*)ZMarkStackSpaceStart, ZMarkStackSpaceSize,
- PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
- if (res != (void*)ZMarkStackSpaceStart) {
- log_error(gc, marking)("Failed to reserve address space for marking stacks");
+ const size_t size = ZMarkStackSpaceLimit;
+ const size_t alignment = (size_t)os::vm_allocation_granularity();
+ const uintptr_t addr = (uintptr_t)os::reserve_memory(size, NULL, alignment, mtGC);
+ if (addr == 0) {
+ log_error(gc, marking)("Failed to reserve address space for mark stacks");
return;
}
// Successfully initialized
- _top = _end = ZMarkStackSpaceStart;
+ _start = _top = _end = addr;
+
+ // Register mark stack space start
+ ZMarkStackSpaceStart = _start;
}
bool ZMarkStackSpace::is_initialized() const {
- return _top != 0;
-}
-
-void ZMarkStackSpace::expand() {
- const size_t max = ZMarkStackSpaceStart + ZMarkStacksMax;
- if (_end + ZMarkStackSpaceExpandSize > max) {
- // Expansion limit reached. This is a fatal error since we
- // currently can't recover from running out of mark stack space.
- fatal("Mark stack overflow (current size " SIZE_FORMAT "M, max size " SIZE_FORMAT "M),"
- " use -XX:ZMarkStacksMax=<size> to increase this limit",
- (_end - ZMarkStackSpaceStart) / M, ZMarkStacksMax / M);
- }
-
- void* const res = mmap((void*)_end, ZMarkStackSpaceExpandSize,
- PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_FIXED, -1, 0);
- if (res == MAP_FAILED) {
- // Failed to map memory. This is a fatal error since we
- // currently can't recover from running out of mark stack space.
- ZErrno err;
- fatal("Failed to map memory for marking stacks (%s)", err.to_string());
- }
+ return _start != 0;
}
uintptr_t ZMarkStackSpace::alloc_space(size_t size) {
- uintptr_t top = _top;
+ uintptr_t top = Atomic::load(&_top);
for (;;) {
+ const uintptr_t end = Atomic::load(&_end);
const uintptr_t new_top = top + size;
- if (new_top > _end) {
+ if (new_top > end) {
// Not enough space left
return 0;
}
@@ -106,17 +91,28 @@
return addr;
}
- // Expand stack space
- expand();
+ // Check expansion limit
+ const size_t expand_size = ZMarkStackSpaceExpandSize;
+ const size_t old_size = _end - _start;
+ const size_t new_size = old_size + expand_size;
+ if (new_size > ZMarkStackSpaceLimit) {
+ // Expansion limit reached. This is a fatal error since we
+ // currently can't recover from running out of mark stack space.
+ fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
+ "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
+ ZMarkStackSpaceLimit / M);
+ }
log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
- (_end - ZMarkStackSpaceStart) / M,
- (_end - ZMarkStackSpaceStart + ZMarkStackSpaceExpandSize) / M);
+ old_size / M, new_size / M);
+
+ // Expand
+ os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
// Increment top before end to make sure another
// thread can't steal out newly expanded space.
addr = Atomic::add(size, &_top) - size;
- _end += ZMarkStackSpaceExpandSize;
+ Atomic::add(expand_size, &_end);
return addr;
}
--- a/src/hotspot/share/gc/z/zMarkStack.hpp Tue Aug 14 00:15:56 2018 -0400
+++ b/src/hotspot/share/gc/z/zMarkStack.hpp Mon Aug 13 14:04:42 2018 +0200
@@ -76,6 +76,7 @@
class ZMarkStackSpace {
private:
ZLock _expand_lock;
+ uintptr_t _start;
volatile uintptr_t _top;
volatile uintptr_t _end;
--- a/src/hotspot/share/gc/z/z_globals.hpp Tue Aug 14 00:15:56 2018 -0400
+++ b/src/hotspot/share/gc/z/z_globals.hpp Mon Aug 13 14:04:42 2018 +0200
@@ -53,9 +53,9 @@
"Allow Java threads to stall and wait for GC to complete " \
"instead of immediately throwing an OutOfMemoryError") \
\
- product(size_t, ZMarkStacksMax, NOT_LP64(512*M) LP64_ONLY(8*G), \
- "Maximum number of bytes allocated for marking stacks") \
- range(32*M, NOT_LP64(512*M) LP64_ONLY(1024*G)) \
+ product(size_t, ZMarkStackSpaceLimit, 8*G, \
+ "Maximum number of bytes allocated for mark stacks") \
+ range(32*M, 1024*G) \
\
product(uint, ZCollectionInterval, 0, \
"Force GC at a fixed time interval (in seconds)") \