# HG changeset patch # User shade # Date 1526846590 -7200 # Node ID 338a117fb65cbd0c3f826f4f2b2a143649313d25 # Parent 92316ff16d5c1141ce4e426ac4518fe7d8c8aa15 Elastic TLAB machinery should handle decays diff -r 92316ff16d5c -r 338a117fb65c src/hotspot/share/gc/epsilon/epsilonArguments.cpp --- a/src/hotspot/share/gc/epsilon/epsilonArguments.cpp Sat May 19 23:51:09 2018 +0200 +++ b/src/hotspot/share/gc/epsilon/epsilonArguments.cpp Sun May 20 22:03:10 2018 +0200 @@ -50,6 +50,11 @@ warning("EpsilonMaxTLABSize < MinTLABSize, adjusting it to " SIZE_FORMAT, MinTLABSize); EpsilonMaxTLABSize = MinTLABSize; } + + if (!EpsilonElasticTLAB && EpsilonElasticTLABDecay) { + warning("Disabling EpsilonElasticTLABDecay because EpsilonElasticTLAB is disabled"); + FLAG_SET_DEFAULT(EpsilonElasticTLABDecay, false); + } #endif #ifdef COMPILER2 diff -r 92316ff16d5c -r 338a117fb65c src/hotspot/share/gc/epsilon/epsilonHeap.cpp --- a/src/hotspot/share/gc/epsilon/epsilonHeap.cpp Sat May 19 23:51:09 2018 +0200 +++ b/src/hotspot/share/gc/epsilon/epsilonHeap.cpp Sun May 20 22:03:10 2018 +0200 @@ -56,6 +56,8 @@ _step_counter_update = MIN2(max_byte_size / 16, EpsilonUpdateCountersStep); _step_heap_print = (EpsilonPrintHeapStep == 0) ? SIZE_MAX : (max_byte_size / EpsilonPrintHeapStep); + _decay_time_ns = (int64_t) EpsilonTLABDecayTime * 1000 * 1000; + if (init_byte_size != max_byte_size) { log_info(gc)("Initialized with " SIZE_FORMAT "M heap, resizeable to up to " SIZE_FORMAT "M heap with " SIZE_FORMAT "M steps", init_byte_size / M, max_byte_size / M, EpsilonMinHeapExpand / M); @@ -63,9 +65,13 @@ log_info(gc)("Initialized with " SIZE_FORMAT "M non-resizeable heap", init_byte_size / M); } if (UseTLAB) { - log_info(gc)("Using TLAB allocation; max: " SIZE_FORMAT "K, elasticity %.2f", - _max_tlab_size * HeapWordSize / K, - EpsilonTLABElasticity); + log_info(gc)("Using TLAB allocation; max: " SIZE_FORMAT "K", _max_tlab_size * HeapWordSize / K); + if (EpsilonElasticTLAB) { + log_info(gc)("Elastic TLABs with elasticity = %.2fx", EpsilonTLABElasticity); + } + if (EpsilonElasticTLABDecay) { + log_info(gc)("Elastic TLABs with decay = " INT64_FORMAT " ms", EpsilonTLABDecayTime); + } } else { log_info(gc)("Not using TLAB allocation"); } @@ -153,41 +159,71 @@ HeapWord* EpsilonHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { - size_t ergo_tlab = EpsilonThreadLocalData::ergo_tlab_size(Thread::current()); + Thread* thread = Thread::current(); + + // Defaults in case elastic paths are not taken + bool fits = true; + size_t size = requested_size; + size_t ergo_tlab = requested_size; + int64_t time = 0; - bool fits = (requested_size <= ergo_tlab); + if (EpsilonElasticTLAB) { + ergo_tlab = EpsilonThreadLocalData::ergo_tlab_size(thread); + + if (EpsilonElasticTLABDecay) { + int64_t last_alloc = EpsilonThreadLocalData::last_alloc_time(thread); + time = (int64_t) os::javaTimeNanos(); + + assert(last_alloc <= time, "time should be monotonic"); - // If we can fit the allocation under current TLAB size, do so. - // Otherwise, we want to elastically increase the TLAB size. - size_t size = fits ? requested_size : (size_t)(ergo_tlab * EpsilonTLABElasticity); + // If the thread had not allocated recently, retract the ergonomic size. + // This conserves memory when the thread had initial burst of allocations, + // and then started allocating only sporadically. + if (last_alloc != 0 && (time - last_alloc > _decay_time_ns)) { + ergo_tlab = 0; + EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); + } + } + + // If we can fit the allocation under current TLAB size, do so. + // Otherwise, we want to elastically increase the TLAB size. + fits = (requested_size <= ergo_tlab); + if (!fits) { + size = (size_t) (ergo_tlab * EpsilonTLABElasticity); + } + } // Honor boundaries size = MAX2(min_size, MIN2(_max_tlab_size, size)); if (log_is_enabled(Trace, gc)) { ResourceMark rm; - log_trace(gc)( - "TLAB size for \"%s\" (Requested: " SIZE_FORMAT "K, Min: " SIZE_FORMAT - "K, Max: " SIZE_FORMAT "K, Ergo: " SIZE_FORMAT "K) -> " SIZE_FORMAT "K", - Thread::current()->name(), - requested_size * HeapWordSize / K, - min_size * HeapWordSize / K, - _max_tlab_size * HeapWordSize / K, - ergo_tlab * HeapWordSize / K, - size * HeapWordSize / K); + log_trace(gc)("TLAB size for \"%s\" (Requested: " SIZE_FORMAT "K, Min: " SIZE_FORMAT + "K, Max: " SIZE_FORMAT "K, Ergo: " SIZE_FORMAT "K) -> " SIZE_FORMAT "K", + thread->name(), + requested_size * HeapWordSize / K, + min_size * HeapWordSize / K, + _max_tlab_size * HeapWordSize / K, + ergo_tlab * HeapWordSize / K, + size * HeapWordSize / K); } HeapWord* res = allocate_work(size); if (res != NULL) { + // Allocation successful *actual_size = size; - - // Allocation successful, this our new TLAB size, if we requested expansion - if (!fits) { - EpsilonThreadLocalData::set_ergo_tlab_size(Thread::current(), size); + if (EpsilonElasticTLABDecay) { + EpsilonThreadLocalData::set_last_alloc_time(thread, time); + } + if (EpsilonElasticTLAB && !fits) { + // If we requested expansion, this is our new ergonomic TLAB size + EpsilonThreadLocalData::set_ergo_tlab_size(thread, size); } } else { - // Allocation failed, reset ergonomics to try an fit smaller TLABs - EpsilonThreadLocalData::set_ergo_tlab_size(Thread::current(), 0); + // Allocation failed, reset ergonomics to try and fit smaller TLABs + if (EpsilonElasticTLAB) { + EpsilonThreadLocalData::set_ergo_tlab_size(thread, 0); + } } return res; diff -r 92316ff16d5c -r 338a117fb65c src/hotspot/share/gc/epsilon/epsilonHeap.hpp --- a/src/hotspot/share/gc/epsilon/epsilonHeap.hpp Sat May 19 23:51:09 2018 +0200 +++ b/src/hotspot/share/gc/epsilon/epsilonHeap.hpp Sun May 20 22:03:10 2018 +0200 @@ -48,6 +48,7 @@ size_t _last_heap_print; size_t _step_counter_update; size_t _step_heap_print; + int64_t _decay_time_ns; public: static EpsilonHeap* heap(); diff -r 92316ff16d5c -r 338a117fb65c src/hotspot/share/gc/epsilon/epsilonThreadLocalData.hpp --- a/src/hotspot/share/gc/epsilon/epsilonThreadLocalData.hpp Sat May 19 23:51:09 2018 +0200 +++ b/src/hotspot/share/gc/epsilon/epsilonThreadLocalData.hpp Sun May 20 22:03:10 2018 +0200 @@ -27,9 +27,11 @@ class EpsilonThreadLocalData { private: size_t _ergo_tlab_size; + int64_t _last_alloc_time; EpsilonThreadLocalData() : - _ergo_tlab_size(0) {} + _ergo_tlab_size(0), + _last_alloc_time(0) {} static EpsilonThreadLocalData* data(Thread* thread) { assert(UseEpsilonGC, "Sanity"); @@ -49,9 +51,18 @@ return data(thread)->_ergo_tlab_size; } + static int64_t last_alloc_time(Thread* thread) { + return data(thread)->_last_alloc_time; + } + static void set_ergo_tlab_size(Thread *thread, size_t val) { data(thread)->_ergo_tlab_size = val; } + + static void set_last_alloc_time(Thread* thread, int64_t time) { + data(thread)->_last_alloc_time = time; + } + }; #endif // SHARE_VM_GC_EPSILON_EPSILONTHREADLOCALDATA_HPP diff -r 92316ff16d5c -r 338a117fb65c src/hotspot/share/gc/epsilon/epsilon_globals.hpp --- a/src/hotspot/share/gc/epsilon/epsilon_globals.hpp Sat May 19 23:51:09 2018 +0200 +++ b/src/hotspot/share/gc/epsilon/epsilon_globals.hpp Sun May 20 22:03:10 2018 +0200 @@ -62,13 +62,32 @@ "asks TLAB machinery to cap TLAB sizes at this value.") \ range(1, max_intx) \ \ + experimental(bool, EpsilonElasticTLAB, true, \ + "Use elastic policy to manage TLAB sizes. This conserves memory " \ + "for non-actively allocating threads, even when they request " \ + "large TLABs for themselves. Active threads would experience " \ + "smaller TLABs until policy catches up.") \ + \ + experimental(bool, EpsilonElasticTLABDecay, true, \ + "Use timed decays to shrik TLAB sizes. This conserves memory " \ + "for the threads that allocate in bursts of different sizes, " \ + "for example the small/rare allocations coming after the initial "\ + "large burst.") \ + \ experimental(double, EpsilonTLABElasticity, 1.1, \ "Multiplier to use when deciding on next TLAB size. Larger value "\ - "improves performance at the expense of per-thread memory waste." \ - "Lower value improves memory footprint, especially for rarely " \ + "improves performance at the expense of per-thread memory waste. "\ + "Lower value improves memory footprint, but penalizes actively " \ "allocating threads.") \ range(1, max_intx) \ \ + experimental(size_t, EpsilonTLABDecayTime, 1000, \ + "TLAB sizing policy decays to initial size after thread had not " \ + "allocated for this long. Time is in milliseconds. Lower value " \ + "improves memory footprint, but penalizes actively allocating " \ + "threads.") \ + range(1, max_intx) \ + \ experimental(size_t, EpsilonMinHeapExpand, 128 * M, \ "Min expansion step for heap. Larger value improves performance " \ "at the potential expense of memory waste.") \ diff -r 92316ff16d5c -r 338a117fb65c test/hotspot/jtreg/gc/epsilon/TestElasticTLAB.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/hotspot/jtreg/gc/epsilon/TestElasticTLAB.java Sun May 20 22:03:10 2018 +0200 @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test TestElasticTLAB + * @key gc + * @requires vm.gc.Epsilon + * @summary Epsilon is able to work with/without elastic TLABs + * + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:-EpsilonElasticTLAB TestElasticTLAB + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:EpsilonTLABElasticity=1 TestElasticTLAB + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:EpsilonTLABElasticity=1.1 TestElasticTLAB + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:EpsilonTLABElasticity=2.0 TestElasticTLAB + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:EpsilonTLABElasticity=42 TestElasticTLAB + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:EpsilonTLABElasticity=100 TestElasticTLAB + */ + +import java.util.Random; + +public class TestElasticTLAB { + + static long SEED = Long.getLong("seed", System.nanoTime()); + static int COUNT = Integer.getInteger("count", 3000); // ~500 MB allocation + + static byte[][] arr; + + public static void main(String[] args) throws Exception { + Random r = new Random(SEED); + + arr = new byte[COUNT * 100][]; + for (int c = 0; c < COUNT; c++) { + arr[c] = new byte[c * 100]; + for (int v = 0; v < c; v++) { + arr[c][v] = (byte)(r.nextInt(255) & 0xFF); + } + } + + r = new Random(SEED); + for (int c = 0; c < COUNT; c++) { + byte[] b = arr[c]; + if (b.length != (c * 100)) { + throw new IllegalStateException("Failure: length = " + b.length + ", need = " + (c*100)); + } + for (int v = 0; v < c; v++) { + byte actual = b[v]; + byte expected = (byte)(r.nextInt(255) & 0xFF); + if (actual != expected) { + throw new IllegalStateException("Failure: expected = " + expected + ", actual = " + actual); + } + } + } + } +} diff -r 92316ff16d5c -r 338a117fb65c test/hotspot/jtreg/gc/epsilon/TestElasticTLABDecay.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/hotspot/jtreg/gc/epsilon/TestElasticTLABDecay.java Sun May 20 22:03:10 2018 +0200 @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test TestElasticTLABDecay + * @key gc + * @requires vm.gc.Epsilon + * @summary Epsilon is able to work with/without elastic TLABs + * + * @run main/othervm -Xmx1g -Xlog:gc -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:-EpsilonElasticTLABDecay TestElasticTLABDecay + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:+EpsilonElasticTLABDecay -XX:EpsilonTLABDecayTime=1 TestElasticTLABDecay + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:+EpsilonElasticTLABDecay -XX:EpsilonTLABDecayTime=10 TestElasticTLABDecay + * @run main/othervm -Xmx1g -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -XX:+EpsilonElasticTLAB -XX:+EpsilonElasticTLABDecay -XX:EpsilonTLABDecayTime=100 TestElasticTLABDecay + */ + +import java.util.Random; + +public class TestElasticTLABDecay { + + static long SEED = Long.getLong("seed", System.nanoTime()); + static int COUNT = Integer.getInteger("count", 3000); // ~500 MB allocation + + static byte[][] arr; + + public static void main(String[] args) throws Exception { + Random r = new Random(SEED); + + arr = new byte[COUNT * 100][]; + for (int c = 0; c < COUNT; c++) { + arr[c] = new byte[c * 100]; + for (int v = 0; v < c; v++) { + arr[c][v] = (byte)(r.nextInt(255) & 0xFF); + } + //Thread.sleep(10); + } + + r = new Random(SEED); + for (int c = 0; c < COUNT; c++) { + byte[] b = arr[c]; + if (b.length != (c * 100)) { + throw new IllegalStateException("Failure: length = " + b.length + ", need = " + (c*100)); + } + for (int v = 0; v < c; v++) { + byte actual = b[v]; + byte expected = (byte)(r.nextInt(255) & 0xFF); + if (actual != expected) { + throw new IllegalStateException("Failure: expected = " + expected + ", actual = " + actual); + } + } + } + } +}