--- a/src/hotspot/share/gc/z/zGlobals.cpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zGlobals.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -24,27 +24,35 @@
#include "precompiled.hpp"
#include "gc/z/zGlobals.hpp"
-uint32_t ZGlobalPhase = ZPhaseRelocate;
-uint32_t ZGlobalSeqNum = 1;
+uint32_t ZGlobalPhase = ZPhaseRelocate;
+uint32_t ZGlobalSeqNum = 1;
-const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes;
-const int& ZObjectAlignmentSmall = MinObjAlignmentInBytes;
+size_t ZPageSizeMediumShift;
+size_t ZPageSizeMedium;
-uintptr_t ZAddressGoodMask;
-uintptr_t ZAddressBadMask;
-uintptr_t ZAddressWeakBadMask;
+size_t ZObjectSizeLimitMedium;
-uintptr_t ZAddressBase;
+const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes;
+int ZObjectAlignmentMediumShift;
+
+const int& ZObjectAlignmentSmall = MinObjAlignmentInBytes;
+int ZObjectAlignmentMedium;
-size_t ZAddressOffsetBits;
-uintptr_t ZAddressOffsetMask;
-size_t ZAddressOffsetMax;
+uintptr_t ZAddressGoodMask;
+uintptr_t ZAddressBadMask;
+uintptr_t ZAddressWeakBadMask;
+
+uintptr_t ZAddressBase;
-size_t ZAddressMetadataShift;
-uintptr_t ZAddressMetadataMask;
+size_t ZAddressOffsetBits;
+uintptr_t ZAddressOffsetMask;
+size_t ZAddressOffsetMax;
-uintptr_t ZAddressMetadataMarked;
-uintptr_t ZAddressMetadataMarked0;
-uintptr_t ZAddressMetadataMarked1;
-uintptr_t ZAddressMetadataRemapped;
-uintptr_t ZAddressMetadataFinalizable;
+size_t ZAddressMetadataShift;
+uintptr_t ZAddressMetadataMask;
+
+uintptr_t ZAddressMetadataMarked;
+uintptr_t ZAddressMetadataMarked0;
+uintptr_t ZAddressMetadataMarked1;
+uintptr_t ZAddressMetadataRemapped;
+uintptr_t ZAddressMetadataFinalizable;
--- a/src/hotspot/share/gc/z/zGlobals.hpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zGlobals.hpp Wed Nov 20 10:37:46 2019 +0100
@@ -50,17 +50,6 @@
// Virtual memory to physical memory ratio
const size_t ZVirtualToPhysicalRatio = 16; // 16:1
-//
-// Page Tiers (assuming ZGranuleSize=2M)
-// -------------------------------------
-//
-// Page Size Object Size Object Alignment
-// --------------------------------------------------
-// Small 2M <= 265K MinObjAlignmentInBytes
-// Medium 32M <= 4M 4K
-// Large N x 2M > 4M 2M
-//
-
// Page types
const uint8_t ZPageTypeSmall = 0;
const uint8_t ZPageTypeMedium = 1;
@@ -68,24 +57,24 @@
// Page size shifts
const size_t ZPageSizeSmallShift = ZGranuleSizeShift;
-const size_t ZPageSizeMediumShift = ZPageSizeSmallShift + 4;
+extern size_t ZPageSizeMediumShift;
// Page sizes
const size_t ZPageSizeSmall = (size_t)1 << ZPageSizeSmallShift;
-const size_t ZPageSizeMedium = (size_t)1 << ZPageSizeMediumShift;
+extern size_t ZPageSizeMedium;
// Object size limits
-const size_t ZObjectSizeLimitSmall = (ZPageSizeSmall / 8); // Allow 12.5% waste
-const size_t ZObjectSizeLimitMedium = (ZPageSizeMedium / 8); // Allow 12.5% waste
+const size_t ZObjectSizeLimitSmall = ZPageSizeSmall / 8; // 12.5% max waste
+extern size_t ZObjectSizeLimitMedium;
// Object alignment shifts
extern const int& ZObjectAlignmentSmallShift;
-const int ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13; // 8192 objects per page
-const int ZObjectAlignmentLargeShift = ZPageSizeSmallShift;
+extern int ZObjectAlignmentMediumShift;
+const int ZObjectAlignmentLargeShift = ZGranuleSizeShift;
// Object alignments
extern const int& ZObjectAlignmentSmall;
-const int ZObjectAlignmentMedium = 1 << ZObjectAlignmentMediumShift;
+extern int ZObjectAlignmentMedium;
const int ZObjectAlignmentLarge = 1 << ZObjectAlignmentLargeShift;
//
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zHeuristics.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zCPU.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zHeuristics.hpp"
+#include "gc/z/zUtils.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+
+void ZHeuristics::set_medium_page_size() {
+ // Set ZPageSizeMedium so that a medium page occupies at most 3.125% of the
+ // max heap size. ZPageSizeMedium is initially set to 0, which means medium
+ // pages are effectively disabled. It is adjusted only if ZPageSizeMedium
+ // becomes larger than ZPageSizeSmall.
+ const size_t min = ZGranuleSize;
+ const size_t max = ZGranuleSize * 16;
+ const size_t unclamped = MaxHeapSize * 0.03125;
+ const size_t clamped = MIN2(MAX2(min, unclamped), max);
+ const size_t size = ZUtils::round_down_power_of_2(clamped);
+
+ if (size > ZPageSizeSmall) {
+ // Enable medium pages
+ ZPageSizeMedium = size;
+ ZPageSizeMediumShift = log2_intptr(ZPageSizeMedium);
+ ZObjectSizeLimitMedium = ZPageSizeMedium / 8;
+ ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13;
+ ZObjectAlignmentMedium = 1 << ZObjectAlignmentMediumShift;
+
+ log_info(gc, init)("Medium Page Size: " SIZE_FORMAT "M", ZPageSizeMedium / M);
+ } else {
+ log_info(gc, init)("Medium Page Size: N/A");
+ }
+}
+
+bool ZHeuristics::use_per_cpu_shared_small_pages() {
+ // Use per-CPU shared small pages only if these pages occupy at most 3.125%
+ // of the max heap size. Otherwise fall back to using a single shared small
+ // page. This is useful when using small heaps on large machines.
+ const size_t per_cpu_share = (MaxHeapSize * 0.03125) / ZCPU::count();
+ return per_cpu_share >= ZPageSizeSmall;
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/src/hotspot/share/gc/z/zHeuristics.hpp Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZHEURISTICS_HPP
+#define SHARE_GC_Z_ZHEURISTICS_HPP
+
+#include "memory/allocation.hpp"
+
+class ZHeuristics : public AllStatic {
+public:
+ static void set_medium_page_size();
+
+ static bool use_per_cpu_shared_small_pages();
+};
+
+#endif // SHARE_GC_Z_ZHEURISTICS_HPP
--- a/src/hotspot/share/gc/z/zInitialize.cpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zInitialize.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -26,6 +26,7 @@
#include "gc/z/zBarrierSet.hpp"
#include "gc/z/zCPU.hpp"
#include "gc/z/zGlobals.hpp"
+#include "gc/z/zHeuristics.hpp"
#include "gc/z/zInitialize.hpp"
#include "gc/z/zLargePages.hpp"
#include "gc/z/zNUMA.hpp"
@@ -49,6 +50,7 @@
ZThreadLocalAllocBuffer::initialize();
ZTracer::initialize();
ZLargePages::initialize();
+ ZHeuristics::set_medium_page_size();
ZBarrierSet::set_barrier_set(barrier_set);
initialize_os();
--- a/src/hotspot/share/gc/z/zObjectAllocator.cpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
#include "gc/z/zCollectedHeap.hpp"
#include "gc/z/zGlobals.hpp"
#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zHeuristics.hpp"
#include "gc/z/zObjectAllocator.hpp"
#include "gc/z/zPage.inline.hpp"
#include "gc/z/zStat.hpp"
@@ -43,12 +44,21 @@
static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);
ZObjectAllocator::ZObjectAllocator() :
+ _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
_used(0),
_undone(0),
_shared_medium_page(NULL),
_shared_small_page(NULL),
_worker_small_page(NULL) {}
+ZPage** ZObjectAllocator::shared_small_page_addr() {
+ return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
+ZPage* const* ZObjectAllocator::shared_small_page_addr() const {
+ return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
if (page != NULL) {
@@ -72,7 +82,7 @@
size_t size,
ZAllocationFlags flags) {
uintptr_t addr = 0;
- ZPage* page = *shared_page;
+ ZPage* page = OrderAccess::load_acquire(shared_page);
if (page != NULL) {
addr = page->alloc_object_atomic(size);
@@ -142,7 +152,7 @@
// Non-worker small page allocation can never use the reserve
flags.set_no_reserve();
- return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
+ return alloc_object_in_shared_page(shared_small_page_addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
}
uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) {
@@ -294,7 +304,7 @@
size_t ZObjectAllocator::remaining() const {
assert(ZThread::is_java(), "Should be a Java thread");
- ZPage* page = _shared_small_page.get();
+ const ZPage* const page = OrderAccess::load_acquire(shared_small_page_addr());
if (page != NULL) {
return page->remaining();
}
--- a/src/hotspot/share/gc/z/zObjectAllocator.hpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp Wed Nov 20 10:37:46 2019 +0100
@@ -31,12 +31,16 @@
class ZObjectAllocator {
private:
+ const bool _use_per_cpu_shared_small_pages;
ZPerCPU<size_t> _used;
ZPerCPU<size_t> _undone;
ZContended<ZPage*> _shared_medium_page;
ZPerCPU<ZPage*> _shared_small_page;
ZPerWorker<ZPage*> _worker_small_page;
+ ZPage** shared_small_page_addr();
+ ZPage* const* shared_small_page_addr() const;
+
ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
void undo_alloc_page(ZPage* page);
--- a/src/hotspot/share/gc/z/zPage.inline.hpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zPage.inline.hpp Wed Nov 20 10:37:46 2019 +0100
@@ -39,14 +39,11 @@
#include "utilities/debug.hpp"
inline uint8_t ZPage::type_from_size(size_t size) const {
- switch (size) {
- case ZPageSizeSmall:
+ if (size == ZPageSizeSmall) {
return ZPageTypeSmall;
-
- case ZPageSizeMedium:
+ } else if (size == ZPageSizeMedium) {
return ZPageTypeMedium;
-
- default:
+ } else {
return ZPageTypeLarge;
}
}
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -97,6 +97,11 @@
}
void ZRelocationSetSelectorGroup::select() {
+ if (_page_size == 0) {
+ // Page type disabled
+ return;
+ }
+
// Calculate the number of pages to relocate by successively including pages in
// a candidate relocation set and calculate the maximum space requirement for
// their live objects.
--- a/src/hotspot/share/gc/z/zWorkers.cpp Wed Nov 20 10:37:46 2019 +0100
+++ b/src/hotspot/share/gc/z/zWorkers.cpp Wed Nov 20 10:37:46 2019 +0100
@@ -35,15 +35,16 @@
}
static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) {
- const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall;
+ const int nworkers = (MaxHeapSize * (reserve_share_in_percent / 100.0)) / ZPageSizeSmall;
return MAX2(nworkers, 1);
}
static uint calculate_nworkers(double cpu_share_in_percent) {
- // Cap number of workers so that we never use more than 10% of the max heap
- // for the reserve. This is useful when using small heaps on large machines.
+ // Cap number of workers so that we don't use more than 2% of the max heap
+ // for the small page reserve. This is useful when using small heaps on
+ // large machines.
return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent),
- calculate_nworkers_based_on_heap_size(10.0));
+ calculate_nworkers_based_on_heap_size(2.0));
}
uint ZWorkers::calculate_nparallel() {
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/gc/z/TestSmallHeap.java Wed Nov 20 10:37:46 2019 +0100
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.z;
+
+/*
+ * @test TestSmallHeap
+ * @requires vm.gc.Z & !vm.graal.enabled
+ * @summary Test ZGC with small heaps
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx8M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx16M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx32M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx64M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx128M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx256M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx512M gc.z.TestSmallHeap
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx1024M gc.z.TestSmallHeap
+ */
+
+import java.lang.ref.Reference;
+
+public class TestSmallHeap {
+ public static void main(String[] args) throws Exception {
+ final long maxCapacity = Runtime.getRuntime().maxMemory();
+ System.out.println("Max Capacity " + maxCapacity + " bytes");
+
+ // Allocate byte arrays of increasing length, so that
+ // all allocaion paths (small/medium/large) are tested.
+ for (int length = 16; length <= maxCapacity / 16; length *= 2) {
+ System.out.println("Allocating " + length + " bytes");
+ Reference.reachabilityFence(new byte[length]);
+ }
+
+ System.out.println("Success");
+ }
+}