# HG changeset patch # User pliden # Date 1574242666 -3600 # Node ID 3b998574be4b61e87dbaa10612cf09c41531c1a3 # Parent 877c000fd688a68c18e7af806bf201eb44152227 8234338: ZGC: Improve small heap usage Reviewed-by: eosterlund, stefank diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zGlobals.cpp --- a/src/hotspot/share/gc/z/zGlobals.cpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zGlobals.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -24,27 +24,35 @@ #include "precompiled.hpp" #include "gc/z/zGlobals.hpp" -uint32_t ZGlobalPhase = ZPhaseRelocate; -uint32_t ZGlobalSeqNum = 1; +uint32_t ZGlobalPhase = ZPhaseRelocate; +uint32_t ZGlobalSeqNum = 1; -const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes; -const int& ZObjectAlignmentSmall = MinObjAlignmentInBytes; +size_t ZPageSizeMediumShift; +size_t ZPageSizeMedium; -uintptr_t ZAddressGoodMask; -uintptr_t ZAddressBadMask; -uintptr_t ZAddressWeakBadMask; +size_t ZObjectSizeLimitMedium; -uintptr_t ZAddressBase; +const int& ZObjectAlignmentSmallShift = LogMinObjAlignmentInBytes; +int ZObjectAlignmentMediumShift; + +const int& ZObjectAlignmentSmall = MinObjAlignmentInBytes; +int ZObjectAlignmentMedium; -size_t ZAddressOffsetBits; -uintptr_t ZAddressOffsetMask; -size_t ZAddressOffsetMax; +uintptr_t ZAddressGoodMask; +uintptr_t ZAddressBadMask; +uintptr_t ZAddressWeakBadMask; + +uintptr_t ZAddressBase; -size_t ZAddressMetadataShift; -uintptr_t ZAddressMetadataMask; +size_t ZAddressOffsetBits; +uintptr_t ZAddressOffsetMask; +size_t ZAddressOffsetMax; -uintptr_t ZAddressMetadataMarked; -uintptr_t ZAddressMetadataMarked0; -uintptr_t ZAddressMetadataMarked1; -uintptr_t ZAddressMetadataRemapped; -uintptr_t ZAddressMetadataFinalizable; +size_t ZAddressMetadataShift; +uintptr_t ZAddressMetadataMask; + +uintptr_t ZAddressMetadataMarked; +uintptr_t ZAddressMetadataMarked0; +uintptr_t ZAddressMetadataMarked1; +uintptr_t ZAddressMetadataRemapped; +uintptr_t ZAddressMetadataFinalizable; diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zGlobals.hpp --- a/src/hotspot/share/gc/z/zGlobals.hpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zGlobals.hpp Wed Nov 20 10:37:46 2019 +0100 @@ -50,17 +50,6 @@ // Virtual memory to physical memory ratio const size_t ZVirtualToPhysicalRatio = 16; // 16:1 -// -// Page Tiers (assuming ZGranuleSize=2M) -// ------------------------------------- -// -// Page Size Object Size Object Alignment -// -------------------------------------------------- -// Small 2M <= 265K MinObjAlignmentInBytes -// Medium 32M <= 4M 4K -// Large N x 2M > 4M 2M -// - // Page types const uint8_t ZPageTypeSmall = 0; const uint8_t ZPageTypeMedium = 1; @@ -68,24 +57,24 @@ // Page size shifts const size_t ZPageSizeSmallShift = ZGranuleSizeShift; -const size_t ZPageSizeMediumShift = ZPageSizeSmallShift + 4; +extern size_t ZPageSizeMediumShift; // Page sizes const size_t ZPageSizeSmall = (size_t)1 << ZPageSizeSmallShift; -const size_t ZPageSizeMedium = (size_t)1 << ZPageSizeMediumShift; +extern size_t ZPageSizeMedium; // Object size limits -const size_t ZObjectSizeLimitSmall = (ZPageSizeSmall / 8); // Allow 12.5% waste -const size_t ZObjectSizeLimitMedium = (ZPageSizeMedium / 8); // Allow 12.5% waste +const size_t ZObjectSizeLimitSmall = ZPageSizeSmall / 8; // 12.5% max waste +extern size_t ZObjectSizeLimitMedium; // Object alignment shifts extern const int& ZObjectAlignmentSmallShift; -const int ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13; // 8192 objects per page -const int ZObjectAlignmentLargeShift = ZPageSizeSmallShift; +extern int ZObjectAlignmentMediumShift; +const int ZObjectAlignmentLargeShift = ZGranuleSizeShift; // Object alignments extern const int& ZObjectAlignmentSmall; -const int ZObjectAlignmentMedium = 1 << ZObjectAlignmentMediumShift; +extern int ZObjectAlignmentMedium; const int ZObjectAlignmentLarge = 1 << ZObjectAlignmentLargeShift; // diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zHeuristics.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zHeuristics.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "precompiled.hpp" +#include "gc/z/zCPU.inline.hpp" +#include "gc/z/zGlobals.hpp" +#include "gc/z/zHeuristics.hpp" +#include "gc/z/zUtils.inline.hpp" +#include "logging/log.hpp" +#include "runtime/globals.hpp" +#include "runtime/os.hpp" + +void ZHeuristics::set_medium_page_size() { + // Set ZPageSizeMedium so that a medium page occupies at most 3.125% of the + // max heap size. ZPageSizeMedium is initially set to 0, which means medium + // pages are effectively disabled. It is adjusted only if ZPageSizeMedium + // becomes larger than ZPageSizeSmall. + const size_t min = ZGranuleSize; + const size_t max = ZGranuleSize * 16; + const size_t unclamped = MaxHeapSize * 0.03125; + const size_t clamped = MIN2(MAX2(min, unclamped), max); + const size_t size = ZUtils::round_down_power_of_2(clamped); + + if (size > ZPageSizeSmall) { + // Enable medium pages + ZPageSizeMedium = size; + ZPageSizeMediumShift = log2_intptr(ZPageSizeMedium); + ZObjectSizeLimitMedium = ZPageSizeMedium / 8; + ZObjectAlignmentMediumShift = ZPageSizeMediumShift - 13; + ZObjectAlignmentMedium = 1 << ZObjectAlignmentMediumShift; + + log_info(gc, init)("Medium Page Size: " SIZE_FORMAT "M", ZPageSizeMedium / M); + } else { + log_info(gc, init)("Medium Page Size: N/A"); + } +} + +bool ZHeuristics::use_per_cpu_shared_small_pages() { + // Use per-CPU shared small pages only if these pages occupy at most 3.125% + // of the max heap size. Otherwise fall back to using a single shared small + // page. This is useful when using small heaps on large machines. + const size_t per_cpu_share = (MaxHeapSize * 0.03125) / ZCPU::count(); + return per_cpu_share >= ZPageSizeSmall; +} diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zHeuristics.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zHeuristics.hpp Wed Nov 20 10:37:46 2019 +0100 @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZHEURISTICS_HPP +#define SHARE_GC_Z_ZHEURISTICS_HPP + +#include "memory/allocation.hpp" + +class ZHeuristics : public AllStatic { +public: + static void set_medium_page_size(); + + static bool use_per_cpu_shared_small_pages(); +}; + +#endif // SHARE_GC_Z_ZHEURISTICS_HPP diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zInitialize.cpp --- a/src/hotspot/share/gc/z/zInitialize.cpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zInitialize.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -26,6 +26,7 @@ #include "gc/z/zBarrierSet.hpp" #include "gc/z/zCPU.hpp" #include "gc/z/zGlobals.hpp" +#include "gc/z/zHeuristics.hpp" #include "gc/z/zInitialize.hpp" #include "gc/z/zLargePages.hpp" #include "gc/z/zNUMA.hpp" @@ -49,6 +50,7 @@ ZThreadLocalAllocBuffer::initialize(); ZTracer::initialize(); ZLargePages::initialize(); + ZHeuristics::set_medium_page_size(); ZBarrierSet::set_barrier_set(barrier_set); initialize_os(); diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zObjectAllocator.cpp --- a/src/hotspot/share/gc/z/zObjectAllocator.cpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHeap.inline.hpp" +#include "gc/z/zHeuristics.hpp" #include "gc/z/zObjectAllocator.hpp" #include "gc/z/zPage.inline.hpp" #include "gc/z/zStat.hpp" @@ -43,12 +44,21 @@ static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond); ZObjectAllocator::ZObjectAllocator() : + _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()), _used(0), _undone(0), _shared_medium_page(NULL), _shared_small_page(NULL), _worker_small_page(NULL) {} +ZPage** ZObjectAllocator::shared_small_page_addr() { + return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0); +} + +ZPage* const* ZObjectAllocator::shared_small_page_addr() const { + return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0); +} + ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) { ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags); if (page != NULL) { @@ -72,7 +82,7 @@ size_t size, ZAllocationFlags flags) { uintptr_t addr = 0; - ZPage* page = *shared_page; + ZPage* page = OrderAccess::load_acquire(shared_page); if (page != NULL) { addr = page->alloc_object_atomic(size); @@ -142,7 +152,7 @@ // Non-worker small page allocation can never use the reserve flags.set_no_reserve(); - return alloc_object_in_shared_page(_shared_small_page.addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags); + return alloc_object_in_shared_page(shared_small_page_addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags); } uintptr_t ZObjectAllocator::alloc_small_object_from_worker(size_t size, ZAllocationFlags flags) { @@ -294,7 +304,7 @@ size_t ZObjectAllocator::remaining() const { assert(ZThread::is_java(), "Should be a Java thread"); - ZPage* page = _shared_small_page.get(); + const ZPage* const page = OrderAccess::load_acquire(shared_small_page_addr()); if (page != NULL) { return page->remaining(); } diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zObjectAllocator.hpp --- a/src/hotspot/share/gc/z/zObjectAllocator.hpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp Wed Nov 20 10:37:46 2019 +0100 @@ -31,12 +31,16 @@ class ZObjectAllocator { private: + const bool _use_per_cpu_shared_small_pages; ZPerCPU _used; ZPerCPU _undone; ZContended _shared_medium_page; ZPerCPU _shared_small_page; ZPerWorker _worker_small_page; + ZPage** shared_small_page_addr(); + ZPage* const* shared_small_page_addr() const; + ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags); void undo_alloc_page(ZPage* page); diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zPage.inline.hpp --- a/src/hotspot/share/gc/z/zPage.inline.hpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zPage.inline.hpp Wed Nov 20 10:37:46 2019 +0100 @@ -39,14 +39,11 @@ #include "utilities/debug.hpp" inline uint8_t ZPage::type_from_size(size_t size) const { - switch (size) { - case ZPageSizeSmall: + if (size == ZPageSizeSmall) { return ZPageTypeSmall; - - case ZPageSizeMedium: + } else if (size == ZPageSizeMedium) { return ZPageTypeMedium; - - default: + } else { return ZPageTypeLarge; } } diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zRelocationSetSelector.cpp --- a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -97,6 +97,11 @@ } void ZRelocationSetSelectorGroup::select() { + if (_page_size == 0) { + // Page type disabled + return; + } + // Calculate the number of pages to relocate by successively including pages in // a candidate relocation set and calculate the maximum space requirement for // their live objects. diff -r 877c000fd688 -r 3b998574be4b src/hotspot/share/gc/z/zWorkers.cpp --- a/src/hotspot/share/gc/z/zWorkers.cpp Wed Nov 20 10:37:46 2019 +0100 +++ b/src/hotspot/share/gc/z/zWorkers.cpp Wed Nov 20 10:37:46 2019 +0100 @@ -35,15 +35,16 @@ } static uint calculate_nworkers_based_on_heap_size(double reserve_share_in_percent) { - const int nworkers = ((MaxHeapSize * (reserve_share_in_percent / 100.0)) - ZPageSizeMedium) / ZPageSizeSmall; + const int nworkers = (MaxHeapSize * (reserve_share_in_percent / 100.0)) / ZPageSizeSmall; return MAX2(nworkers, 1); } static uint calculate_nworkers(double cpu_share_in_percent) { - // Cap number of workers so that we never use more than 10% of the max heap - // for the reserve. This is useful when using small heaps on large machines. + // Cap number of workers so that we don't use more than 2% of the max heap + // for the small page reserve. This is useful when using small heaps on + // large machines. return MIN2(calculate_nworkers_based_on_ncpus(cpu_share_in_percent), - calculate_nworkers_based_on_heap_size(10.0)); + calculate_nworkers_based_on_heap_size(2.0)); } uint ZWorkers::calculate_nparallel() { diff -r 877c000fd688 -r 3b998574be4b test/hotspot/jtreg/gc/z/TestSmallHeap.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/hotspot/jtreg/gc/z/TestSmallHeap.java Wed Nov 20 10:37:46 2019 +0100 @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package gc.z; + +/* + * @test TestSmallHeap + * @requires vm.gc.Z & !vm.graal.enabled + * @summary Test ZGC with small heaps + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx8M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx16M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx32M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx64M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx128M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx256M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx512M gc.z.TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseZGC -Xlog:gc,gc+init,gc+heap -Xmx1024M gc.z.TestSmallHeap + */ + +import java.lang.ref.Reference; + +public class TestSmallHeap { + public static void main(String[] args) throws Exception { + final long maxCapacity = Runtime.getRuntime().maxMemory(); + System.out.println("Max Capacity " + maxCapacity + " bytes"); + + // Allocate byte arrays of increasing length, so that + // all allocaion paths (small/medium/large) are tested. + for (int length = 16; length <= maxCapacity / 16; length *= 2) { + System.out.println("Allocating " + length + " bytes"); + Reference.reachabilityFence(new byte[length]); + } + + System.out.println("Success"); + } +}