# HG changeset patch # User pliden # Date 1537445084 -7200 # Node ID 75e4ce0fa1baef05027d2e4069cffb1930edb92f # Parent 46eac084082d81dc545d3128cb157336d4be52d7 8210883: ZGC: Parallel retire/resize/remap of TLABs Reviewed-by: eosterlund diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zCollectedHeap.cpp --- a/src/hotspot/share/gc/z/zCollectedHeap.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -117,6 +117,10 @@ return is_in(p); } +void ZCollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) { + // Does nothing, not a parsable heap +} + HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(requested_size)); const uintptr_t addr = _heap.alloc_tlab(size_in_bytes); diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zCollectedHeap.hpp --- a/src/hotspot/share/gc/z/zCollectedHeap.hpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp Thu Sep 20 14:04:44 2018 +0200 @@ -56,9 +56,6 @@ public: static ZCollectedHeap* heap(); - using CollectedHeap::ensure_parsability; - using CollectedHeap::resize_all_tlabs; - ZCollectedHeap(ZCollectorPolicy* policy); virtual Name kind() const; virtual const char* name() const; @@ -78,6 +75,8 @@ virtual bool is_in(const void* p) const; virtual bool is_in_closed_subset(const void* p) const; + virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap); + virtual HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded); virtual MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, size_t size, diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zHeap.cpp --- a/src/hotspot/share/gc/z/zHeap.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zHeap.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -273,13 +273,13 @@ // Update statistics ZStatSample(ZSamplerHeapUsedBeforeMark, used()); - // Retire TLABs - _object_allocator.retire_tlabs(); - // Flip address view ZAddressMasks::flip_to_marked(); flip_views(); + // Retire allocating pages + _object_allocator.retire_pages(); + // Reset allocated/reclaimed/used statistics _page_allocator.reset_statistics(); @@ -475,9 +475,6 @@ ZAddressMasks::flip_to_remapped(); flip_views(); - // Remap TLABs - _object_allocator.remap_tlabs(); - // Enter relocate phase ZGlobalPhase = ZPhaseRelocate; diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zInitialize.cpp --- a/src/hotspot/share/gc/z/zInitialize.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zInitialize.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,6 +30,7 @@ #include "gc/z/zLargePages.hpp" #include "gc/z/zNUMA.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zStatTLAB.hpp" #include "gc/z/zTracer.hpp" #include "logging/log.hpp" #include "runtime/vm_version.hpp" @@ -45,6 +46,7 @@ ZNUMA::initialize(); ZCPU::initialize(); ZStatValue::initialize(); + ZStatTLAB::initialize(); ZTracer::initialize(); ZLargePages::initialize(); ZBarrierSet::set_barrier_set(barrier_set); diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zMark.cpp --- a/src/hotspot/share/gc/z/zMark.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zMark.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -32,6 +32,7 @@ #include "gc/z/zPageTable.inline.hpp" #include "gc/z/zRootsIterator.hpp" #include "gc/z/zStat.hpp" +#include "gc/z/zStatTLAB.hpp" #include "gc/z/zTask.hpp" #include "gc/z/zThread.hpp" #include "gc/z/zUtils.inline.hpp" @@ -119,11 +120,25 @@ class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure { public: + ZMarkRootsIteratorClosure() { + ZStatTLAB::reset(); + } + + ~ZMarkRootsIteratorClosure() { + ZStatTLAB::publish(); + } + virtual void do_thread(Thread* thread) { ZRootsIteratorClosure::do_thread(thread); // Update thread local address bad mask ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); + + // Retire TLAB + if (UseTLAB && thread->is_Java_thread()) { + thread->tlab().retire(ZStatTLAB::get()); + thread->tlab().resize(); + } } virtual void do_oop(oop* p) { @@ -137,8 +152,9 @@ class ZMarkRootsTask : public ZTask { private: - ZMark* const _mark; - ZRootsIterator _roots; + ZMark* const _mark; + ZRootsIterator _roots; + ZMarkRootsIteratorClosure _cl; public: ZMarkRootsTask(ZMark* mark) : @@ -147,8 +163,7 @@ _roots() {} virtual void work() { - ZMarkRootsIteratorClosure cl; - _roots.oops_do(&cl); + _roots.oops_do(&_cl); // Flush and free worker stacks. Needed here since // the set of workers executing during root scanning diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zObjectAllocator.cpp --- a/src/hotspot/share/gc/z/zObjectAllocator.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -22,7 +22,6 @@ */ #include "precompiled.hpp" -#include "gc/shared/threadLocalAllocBuffer.inline.hpp" #include "gc/z/zCollectedHeap.hpp" #include "gc/z/zGlobals.hpp" #include "gc/z/zHeap.inline.hpp" @@ -41,8 +40,6 @@ static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond); static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond); -static const ZStatSubPhase ZSubPhasePauseRetireTLABS("Pause Retire TLABS"); -static const ZStatSubPhase ZSubPhasePauseRemapTLABS("Pause Remap TLABS"); ZObjectAllocator::ZObjectAllocator(uint nworkers) : _nworkers(nworkers), @@ -293,17 +290,9 @@ return 0; } -void ZObjectAllocator::retire_tlabs() { - ZStatTimer timer(ZSubPhasePauseRetireTLABS); +void ZObjectAllocator::retire_pages() { assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); - // Retire TLABs - if (UseTLAB) { - ZCollectedHeap* heap = ZCollectedHeap::heap(); - heap->ensure_parsability(true /* retire_tlabs */); - heap->resize_all_tlabs(); - } - // Reset used _used.set_all(0); @@ -312,18 +301,3 @@ _shared_small_page.set_all(NULL); _worker_small_page.set_all(NULL); } - -static void remap_tlab_address(HeapWord** p) { - *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p); -} - -void ZObjectAllocator::remap_tlabs() { - ZStatTimer timer(ZSubPhasePauseRemapTLABS); - assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint"); - - if (UseTLAB) { - for (JavaThreadIteratorWithHandle iter; JavaThread* thread = iter.next(); ) { - thread->tlab().addresses_do(remap_tlab_address); - } - } -} diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zObjectAllocator.hpp --- a/src/hotspot/share/gc/z/zObjectAllocator.hpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp Thu Sep 20 14:04:44 2018 +0200 @@ -72,8 +72,7 @@ size_t used() const; size_t remaining() const; - void retire_tlabs(); - void remap_tlabs(); + void retire_pages(); }; #endif // SHARE_GC_Z_ZOBJECTALLOCATOR_HPP diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zRelocate.cpp --- a/src/hotspot/share/gc/z/zRelocate.cpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zRelocate.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -22,6 +22,8 @@ */ #include "precompiled.hpp" +#include "gc/z/zAddress.inline.hpp" +#include "gc/z/zBarrier.inline.hpp" #include "gc/z/zHeap.hpp" #include "gc/z/zOopClosures.inline.hpp" #include "gc/z/zPage.hpp" @@ -35,12 +37,22 @@ _workers(workers) {} class ZRelocateRootsIteratorClosure : public ZRootsIteratorClosure { +private: + static void remap_address(HeapWord** p) { + *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p); + } + public: virtual void do_thread(Thread* thread) { ZRootsIteratorClosure::do_thread(thread); // Update thread local address bad mask ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask); + + // Remap TLAB + if (UseTLAB && thread->is_Java_thread()) { + thread->tlab().addresses_do(remap_address); + } } virtual void do_oop(oop* p) { @@ -54,7 +66,8 @@ class ZRelocateRootsTask : public ZTask { private: - ZRootsIterator _roots; + ZRootsIterator _roots; + ZRelocateRootsIteratorClosure _cl; public: ZRelocateRootsTask() : @@ -64,8 +77,7 @@ virtual void work() { // During relocation we need to visit the JVMTI // export weak roots to rehash the JVMTI tag map - ZRelocateRootsIteratorClosure cl; - _roots.oops_do(&cl, true /* visit_jvmti_weak_export */); + _roots.oops_do(&_cl, true /* visit_jvmti_weak_export */); } }; diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zStatTLAB.cpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zStatTLAB.cpp Thu Sep 20 14:04:44 2018 +0200 @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#include "gc/z/zStatTLAB.hpp" + +ZPerWorker* ZStatTLAB::_stats = NULL; + +void ZStatTLAB::initialize() { + if (UseTLAB) { + assert(_stats == NULL, "Already initialized"); + _stats = new ZPerWorker(); + reset(); + } +} + +void ZStatTLAB::reset() { + if (UseTLAB) { + ZPerWorkerIterator iter(_stats); + for (ThreadLocalAllocStats* stats; iter.next(&stats);) { + stats->reset(); + } + } +} + +ThreadLocalAllocStats* ZStatTLAB::get() { + if (UseTLAB) { + return _stats->addr(); + } + + return NULL; +} + +void ZStatTLAB::publish() { + if (UseTLAB) { + ThreadLocalAllocStats total; + + ZPerWorkerIterator iter(_stats); + for (ThreadLocalAllocStats* stats; iter.next(&stats);) { + total.update(*stats); + } + + total.publish(); + } +} diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zStatTLAB.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/hotspot/share/gc/z/zStatTLAB.hpp Thu Sep 20 14:04:44 2018 +0200 @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +#ifndef SHARE_GC_Z_ZSTATTLAB_HPP +#define SHARE_GC_Z_ZSTATTLAB_HPP + +#include "gc/shared/threadLocalAllocBuffer.hpp" +#include "gc/z/zValue.hpp" +#include "memory/allocation.hpp" + +class ZStatTLAB : public AllStatic { +private: + static ZPerWorker* _stats; + +public: + static void initialize(); + static void reset(); + static ThreadLocalAllocStats* get(); + static void publish(); +}; + +#endif // SHARE_GC_Z_ZSTATTLAB_HPP diff -r 46eac084082d -r 75e4ce0fa1ba src/hotspot/share/gc/z/zValue.hpp --- a/src/hotspot/share/gc/z/zValue.hpp Thu Sep 20 14:04:43 2018 +0200 +++ b/src/hotspot/share/gc/z/zValue.hpp Thu Sep 20 14:04:44 2018 +0200 @@ -131,7 +131,7 @@ class ZValueIterator; template -class ZValue { +class ZValue : public CHeapObj { private: const uintptr_t _addr;