author | phh |
Sat, 30 Nov 2019 14:33:05 -0800 | |
changeset 59330 | 5b96c12f909d |
parent 59290 | 97d13893ec3c |
permissions | -rw-r--r-- |
50113 | 1 |
/* |
54623
1126f0607c70
8222811: Consolidate MutexLockerEx and MutexLocker
coleenp
parents:
50429
diff
changeset
|
2 |
* Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved. |
50113 | 3 |
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 |
* |
|
5 |
* This code is free software; you can redistribute it and/or modify it |
|
6 |
* under the terms of the GNU General Public License version 2 only, as |
|
7 |
* published by the Free Software Foundation. |
|
8 |
* |
|
9 |
* This code is distributed in the hope that it will be useful, but WITHOUT |
|
10 |
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
|
11 |
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
|
12 |
* version 2 for more details (a copy is included in the LICENSE file that |
|
13 |
* accompanied this code). |
|
14 |
* |
|
15 |
* You should have received a copy of the GNU General Public License version |
|
16 |
* 2 along with this work; if not, write to the Free Software Foundation, |
|
17 |
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
|
18 |
* |
|
19 |
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
20 |
* or visit www.oracle.com if you need additional information or have any |
|
21 |
* questions. |
|
22 |
* |
|
23 |
*/ |
|
24 |
||
25 |
#include "precompiled.hpp" |
|
26 |
#include "jfr/jfrEvents.hpp" |
|
27 |
#include "jfr/jni/jfrJavaSupport.hpp" |
|
28 |
#include "jfr/recorder/jfrRecorder.hpp" |
|
58863 | 29 |
#include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" |
50113 | 30 |
#include "jfr/recorder/repository/jfrChunkWriter.hpp" |
31 |
#include "jfr/recorder/service/jfrOptionSet.hpp" |
|
32 |
#include "jfr/recorder/service/jfrPostBox.hpp" |
|
33 |
#include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" |
|
34 |
#include "jfr/recorder/storage/jfrStorage.hpp" |
|
35 |
#include "jfr/recorder/storage/jfrStorageControl.hpp" |
|
36 |
#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" |
|
37 |
#include "jfr/utilities/jfrIterator.hpp" |
|
38 |
#include "jfr/utilities/jfrTime.hpp" |
|
39 |
#include "jfr/writers/jfrNativeEventWriter.hpp" |
|
40 |
#include "logging/log.hpp" |
|
41 |
#include "runtime/mutexLocker.hpp" |
|
50117
fb66b2959eaf
8203251: Non-PCH build failed after JDK-8199712 (Flight Recorder)
shade
parents:
50113
diff
changeset
|
42 |
#include "runtime/os.inline.hpp" |
50113 | 43 |
#include "runtime/safepoint.hpp" |
44 |
#include "runtime/thread.hpp" |
|
45 |
||
46 |
typedef JfrStorage::Buffer* BufferPtr; |
|
47 |
||
48 |
static JfrStorage* _instance = NULL; |
|
49 |
static JfrStorageControl* _control; |
|
50 |
||
51 |
JfrStorage& JfrStorage::instance() { |
|
52 |
return *_instance; |
|
53 |
} |
|
54 |
||
55 |
JfrStorage* JfrStorage::create(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) { |
|
56 |
assert(_instance == NULL, "invariant"); |
|
57 |
_instance = new JfrStorage(chunkwriter, post_box); |
|
58 |
return _instance; |
|
59 |
} |
|
60 |
||
61 |
void JfrStorage::destroy() { |
|
62 |
if (_instance != NULL) { |
|
63 |
delete _instance; |
|
64 |
_instance = NULL; |
|
65 |
} |
|
66 |
} |
|
67 |
||
68 |
JfrStorage::JfrStorage(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) : |
|
69 |
_control(NULL), |
|
70 |
_global_mspace(NULL), |
|
71 |
_thread_local_mspace(NULL), |
|
72 |
_transient_mspace(NULL), |
|
73 |
_age_mspace(NULL), |
|
74 |
_chunkwriter(chunkwriter), |
|
75 |
_post_box(post_box) {} |
|
76 |
||
77 |
JfrStorage::~JfrStorage() { |
|
78 |
if (_control != NULL) { |
|
79 |
delete _control; |
|
80 |
} |
|
81 |
if (_global_mspace != NULL) { |
|
82 |
delete _global_mspace; |
|
83 |
} |
|
84 |
if (_thread_local_mspace != NULL) { |
|
85 |
delete _thread_local_mspace; |
|
86 |
} |
|
87 |
if (_transient_mspace != NULL) { |
|
88 |
delete _transient_mspace; |
|
89 |
} |
|
90 |
if (_age_mspace != NULL) { |
|
91 |
delete _age_mspace; |
|
92 |
} |
|
93 |
_instance = NULL; |
|
94 |
} |
|
95 |
||
96 |
static const size_t in_memory_discard_threshold_delta = 2; // start to discard data when the only this number of free buffers are left |
|
97 |
static const size_t unlimited_mspace_size = 0; |
|
98 |
static const size_t thread_local_cache_count = 8; |
|
99 |
static const size_t thread_local_scavenge_threshold = thread_local_cache_count / 2; |
|
100 |
static const size_t transient_buffer_size_multiplier = 8; // against thread local buffer size |
|
101 |
||
102 |
template <typename Mspace> |
|
103 |
static Mspace* create_mspace(size_t buffer_size, size_t limit, size_t cache_count, JfrStorage* storage_instance) { |
|
104 |
Mspace* mspace = new Mspace(buffer_size, limit, cache_count, storage_instance); |
|
105 |
if (mspace != NULL) { |
|
106 |
mspace->initialize(); |
|
107 |
} |
|
108 |
return mspace; |
|
109 |
} |
|
110 |
||
111 |
bool JfrStorage::initialize() { |
|
112 |
assert(_control == NULL, "invariant"); |
|
113 |
assert(_global_mspace == NULL, "invariant"); |
|
114 |
assert(_thread_local_mspace == NULL, "invariant"); |
|
115 |
assert(_transient_mspace == NULL, "invariant"); |
|
116 |
assert(_age_mspace == NULL, "invariant"); |
|
117 |
||
118 |
const size_t num_global_buffers = (size_t)JfrOptionSet::num_global_buffers(); |
|
119 |
assert(num_global_buffers >= in_memory_discard_threshold_delta, "invariant"); |
|
120 |
const size_t memory_size = (size_t)JfrOptionSet::memory_size(); |
|
121 |
const size_t global_buffer_size = (size_t)JfrOptionSet::global_buffer_size(); |
|
122 |
const size_t thread_buffer_size = (size_t)JfrOptionSet::thread_buffer_size(); |
|
123 |
||
124 |
_control = new JfrStorageControl(num_global_buffers, num_global_buffers - in_memory_discard_threshold_delta); |
|
125 |
if (_control == NULL) { |
|
126 |
return false; |
|
127 |
} |
|
128 |
_global_mspace = create_mspace<JfrStorageMspace>(global_buffer_size, memory_size, num_global_buffers, this); |
|
129 |
if (_global_mspace == NULL) { |
|
130 |
return false; |
|
131 |
} |
|
132 |
_thread_local_mspace = create_mspace<JfrThreadLocalMspace>(thread_buffer_size, unlimited_mspace_size, thread_local_cache_count, this); |
|
133 |
if (_thread_local_mspace == NULL) { |
|
134 |
return false; |
|
135 |
} |
|
136 |
_transient_mspace = create_mspace<JfrStorageMspace>(thread_buffer_size * transient_buffer_size_multiplier, unlimited_mspace_size, 0, this); |
|
137 |
if (_transient_mspace == NULL) { |
|
138 |
return false; |
|
139 |
} |
|
140 |
_age_mspace = create_mspace<JfrStorageAgeMspace>(0 /* no extra size except header */, unlimited_mspace_size, num_global_buffers, this); |
|
141 |
if (_age_mspace == NULL) { |
|
142 |
return false; |
|
143 |
} |
|
144 |
control().set_scavenge_threshold(thread_local_scavenge_threshold); |
|
145 |
return true; |
|
146 |
} |
|
147 |
||
148 |
JfrStorageControl& JfrStorage::control() { |
|
149 |
return *instance()._control; |
|
150 |
} |
|
151 |
||
152 |
static void log_allocation_failure(const char* msg, size_t size) { |
|
153 |
log_warning(jfr)("Unable to allocate " SIZE_FORMAT " bytes of %s.", size, msg); |
|
154 |
} |
|
155 |
||
156 |
BufferPtr JfrStorage::acquire_thread_local(Thread* thread, size_t size /* 0 */) { |
|
157 |
BufferPtr buffer = mspace_get_to_full(size, instance()._thread_local_mspace, thread); |
|
158 |
if (buffer == NULL) { |
|
159 |
log_allocation_failure("thread local_memory", size); |
|
160 |
return NULL; |
|
161 |
} |
|
162 |
assert(buffer->acquired_by_self(), "invariant"); |
|
163 |
return buffer; |
|
164 |
} |
|
165 |
||
166 |
BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) { |
|
167 |
BufferPtr buffer = mspace_allocate_transient_lease_to_full(size, instance()._transient_mspace, thread); |
|
168 |
if (buffer == NULL) { |
|
169 |
log_allocation_failure("transient memory", size); |
|
170 |
return NULL; |
|
171 |
} |
|
172 |
assert(buffer->acquired_by_self(), "invariant"); |
|
173 |
assert(buffer->transient(), "invariant"); |
|
174 |
assert(buffer->lease(), "invariant"); |
|
175 |
return buffer; |
|
176 |
} |
|
177 |
||
178 |
static BufferPtr get_lease(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { |
|
179 |
assert(size <= mspace->min_elem_size(), "invariant"); |
|
180 |
while (true) { |
|
181 |
BufferPtr t = mspace_get_free_lease_with_retry(size, mspace, retry_count, thread); |
|
182 |
if (t == NULL && storage_instance.control().should_discard()) { |
|
183 |
storage_instance.discard_oldest(thread); |
|
184 |
continue; |
|
185 |
} |
|
186 |
return t; |
|
187 |
} |
|
188 |
} |
|
189 |
||
190 |
static BufferPtr get_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { |
|
191 |
assert(size <= mspace->min_elem_size(), "invariant"); |
|
192 |
while (true) { |
|
193 |
BufferPtr t = mspace_get_free_with_retry(size, mspace, retry_count, thread); |
|
194 |
if (t == NULL && storage_instance.control().should_discard()) { |
|
195 |
storage_instance.discard_oldest(thread); |
|
196 |
continue; |
|
197 |
} |
|
198 |
return t; |
|
199 |
} |
|
200 |
} |
|
201 |
||
202 |
static const size_t lease_retry = 10; |
|
203 |
||
204 |
BufferPtr JfrStorage::acquire_large(size_t size, Thread* thread) { |
|
205 |
JfrStorage& storage_instance = instance(); |
|
206 |
const size_t max_elem_size = storage_instance._global_mspace->min_elem_size(); // min is also max |
|
207 |
// if not too large and capacity is still available, ask for a lease from the global system |
|
208 |
if (size < max_elem_size && storage_instance.control().is_global_lease_allowed()) { |
|
209 |
BufferPtr const buffer = get_lease(size, storage_instance._global_mspace, storage_instance, lease_retry, thread); |
|
210 |
if (buffer != NULL) { |
|
211 |
assert(buffer->acquired_by_self(), "invariant"); |
|
212 |
assert(!buffer->transient(), "invariant"); |
|
213 |
assert(buffer->lease(), "invariant"); |
|
214 |
storage_instance.control().increment_leased(); |
|
215 |
return buffer; |
|
216 |
} |
|
217 |
} |
|
218 |
return acquire_transient(size, thread); |
|
219 |
} |
|
220 |
||
221 |
static void write_data_loss_event(JfrBuffer* buffer, u8 unflushed_size, Thread* thread) { |
|
222 |
assert(buffer != NULL, "invariant"); |
|
223 |
assert(buffer->empty(), "invariant"); |
|
224 |
const u8 total_data_loss = thread->jfr_thread_local()->add_data_lost(unflushed_size); |
|
225 |
if (EventDataLoss::is_enabled()) { |
|
226 |
JfrNativeEventWriter writer(buffer, thread); |
|
227 |
writer.write<u8>(EventDataLoss::eventId); |
|
228 |
writer.write(JfrTicks::now()); |
|
229 |
writer.write(unflushed_size); |
|
230 |
writer.write(total_data_loss); |
|
231 |
} |
|
232 |
} |
|
233 |
||
234 |
static void write_data_loss(BufferPtr buffer, Thread* thread) { |
|
235 |
assert(buffer != NULL, "invariant"); |
|
236 |
const size_t unflushed_size = buffer->unflushed_size(); |
|
237 |
buffer->concurrent_reinitialization(); |
|
238 |
if (unflushed_size == 0) { |
|
239 |
return; |
|
240 |
} |
|
241 |
write_data_loss_event(buffer, unflushed_size, thread); |
|
242 |
} |
|
243 |
||
244 |
static const size_t promotion_retry = 100; |
|
245 |
||
246 |
bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) { |
|
247 |
assert(buffer != NULL, "invariant"); |
|
248 |
assert(!buffer->lease(), "invariant"); |
|
249 |
assert(!buffer->transient(), "invariant"); |
|
250 |
const size_t unflushed_size = buffer->unflushed_size(); |
|
251 |
if (unflushed_size == 0) { |
|
252 |
buffer->concurrent_reinitialization(); |
|
253 |
assert(buffer->empty(), "invariant"); |
|
254 |
return true; |
|
255 |
} |
|
58863 | 256 |
|
257 |
if (buffer->excluded()) { |
|
258 |
const bool thread_is_excluded = thread->jfr_thread_local()->is_excluded(); |
|
259 |
buffer->reinitialize(thread_is_excluded); |
|
260 |
assert(buffer->empty(), "invariant"); |
|
261 |
if (!thread_is_excluded) { |
|
262 |
// state change from exclusion to inclusion requires a thread checkpoint |
|
263 |
JfrCheckpointManager::write_thread_checkpoint(thread); |
|
264 |
} |
|
265 |
return true; |
|
266 |
} |
|
267 |
||
50113 | 268 |
BufferPtr const promotion_buffer = get_promotion_buffer(unflushed_size, _global_mspace, *this, promotion_retry, thread); |
269 |
if (promotion_buffer == NULL) { |
|
270 |
write_data_loss(buffer, thread); |
|
271 |
return false; |
|
272 |
} |
|
273 |
assert(promotion_buffer->acquired_by_self(), "invariant"); |
|
274 |
assert(promotion_buffer->free_size() >= unflushed_size, "invariant"); |
|
275 |
buffer->concurrent_move_and_reinitialize(promotion_buffer, unflushed_size); |
|
276 |
assert(buffer->empty(), "invariant"); |
|
277 |
return true; |
|
278 |
} |
|
279 |
||
280 |
/* |
|
281 |
* 1. If the buffer was a "lease" from the global system, release back. |
|
282 |
* 2. If the buffer is transient (temporal dynamically allocated), retire and register full. |
|
283 |
* |
|
284 |
* The buffer is effectively invalidated for the thread post-return, |
|
285 |
* and the caller should take means to ensure that it is not referenced any longer. |
|
286 |
*/ |
|
287 |
void JfrStorage::release_large(BufferPtr buffer, Thread* thread) { |
|
288 |
assert(buffer != NULL, "invariant"); |
|
289 |
assert(buffer->lease(), "invariant"); |
|
290 |
assert(buffer->acquired_by_self(), "invariant"); |
|
291 |
buffer->clear_lease(); |
|
292 |
if (buffer->transient()) { |
|
293 |
buffer->set_retired(); |
|
294 |
register_full(buffer, thread); |
|
295 |
} else { |
|
296 |
buffer->release(); |
|
297 |
control().decrement_leased(); |
|
298 |
} |
|
299 |
} |
|
300 |
||
301 |
static JfrAgeNode* new_age_node(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, Thread* thread) { |
|
302 |
assert(buffer != NULL, "invariant"); |
|
303 |
assert(age_mspace != NULL, "invariant"); |
|
304 |
return mspace_allocate_transient(0, age_mspace, thread); |
|
305 |
} |
|
306 |
||
307 |
static void log_registration_failure(size_t unflushed_size) { |
|
308 |
log_warning(jfr)("Unable to register a full buffer of " SIZE_FORMAT " bytes.", unflushed_size); |
|
309 |
log_debug(jfr, system)("Cleared 1 full buffer of " SIZE_FORMAT " bytes.", unflushed_size); |
|
310 |
} |
|
311 |
||
312 |
static void handle_registration_failure(BufferPtr buffer) { |
|
313 |
assert(buffer != NULL, "invariant"); |
|
314 |
assert(buffer->retired(), "invariant"); |
|
315 |
const size_t unflushed_size = buffer->unflushed_size(); |
|
58863 | 316 |
buffer->concurrent_reinitialization(); |
50113 | 317 |
log_registration_failure(unflushed_size); |
318 |
} |
|
319 |
||
320 |
static JfrAgeNode* get_free_age_node(JfrStorageAgeMspace* age_mspace, Thread* thread) { |
|
321 |
assert(JfrBuffer_lock->owned_by_self(), "invariant"); |
|
322 |
return mspace_get_free_with_detach(0, age_mspace, thread); |
|
323 |
} |
|
324 |
||
325 |
static bool insert_full_age_node(JfrAgeNode* age_node, JfrStorageAgeMspace* age_mspace, Thread* thread) { |
|
326 |
assert(JfrBuffer_lock->owned_by_self(), "invariant"); |
|
55053
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
327 |
assert(age_node != NULL, "invariant"); |
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
328 |
assert(age_node->acquired_by_self(), "invariant"); |
50113 | 329 |
assert(age_node->retired_buffer()->retired(), "invariant"); |
55053
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
330 |
age_node->release(); // drop identity claim on age node when inserting to full list |
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
331 |
assert(age_node->identity() == NULL, "invariant"); |
50113 | 332 |
age_mspace->insert_full_head(age_node); |
333 |
return true; |
|
334 |
} |
|
335 |
||
336 |
static bool full_buffer_registration(BufferPtr buffer, JfrStorageAgeMspace* age_mspace, JfrStorageControl& control, Thread* thread) { |
|
337 |
assert(buffer != NULL, "invariant"); |
|
338 |
assert(buffer->retired(), "invariant"); |
|
339 |
assert(age_mspace != NULL, "invariant"); |
|
54623
1126f0607c70
8222811: Consolidate MutexLockerEx and MutexLocker
coleenp
parents:
50429
diff
changeset
|
340 |
MutexLocker lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag); |
50113 | 341 |
JfrAgeNode* age_node = get_free_age_node(age_mspace, thread); |
342 |
if (age_node == NULL) { |
|
343 |
age_node = new_age_node(buffer, age_mspace, thread); |
|
344 |
if (age_node == NULL) { |
|
345 |
return false; |
|
346 |
} |
|
347 |
} |
|
55053
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
348 |
assert(age_node != NULL, "invariant"); |
50113 | 349 |
assert(age_node->acquired_by_self(), "invariant"); |
350 |
age_node->set_retired_buffer(buffer); |
|
50234
6ba3e32a9882
8203457: Add back missing full buffer notification
mgronlun
parents:
50117
diff
changeset
|
351 |
control.increment_full(); |
50113 | 352 |
return insert_full_age_node(age_node, age_mspace, thread); |
353 |
} |
|
354 |
||
355 |
void JfrStorage::register_full(BufferPtr buffer, Thread* thread) { |
|
356 |
assert(buffer != NULL, "invariant"); |
|
357 |
assert(buffer->retired(), "invariant"); |
|
54964 | 358 |
assert(buffer->acquired_by(thread), "invariant"); |
50113 | 359 |
if (!full_buffer_registration(buffer, _age_mspace, control(), thread)) { |
360 |
handle_registration_failure(buffer); |
|
361 |
} |
|
362 |
if (control().should_post_buffer_full_message()) { |
|
363 |
_post_box.post(MSG_FULLBUFFER); |
|
364 |
} |
|
365 |
} |
|
366 |
||
367 |
void JfrStorage::lock() { |
|
368 |
assert(!JfrBuffer_lock->owned_by_self(), "invariant"); |
|
369 |
JfrBuffer_lock->lock_without_safepoint_check(); |
|
370 |
} |
|
371 |
||
372 |
void JfrStorage::unlock() { |
|
373 |
assert(JfrBuffer_lock->owned_by_self(), "invariant"); |
|
374 |
JfrBuffer_lock->unlock(); |
|
375 |
} |
|
376 |
||
377 |
#ifdef ASSERT |
|
378 |
bool JfrStorage::is_locked() const { |
|
379 |
return JfrBuffer_lock->owned_by_self(); |
|
380 |
} |
|
381 |
#endif |
|
382 |
||
383 |
// don't use buffer on return, it is gone |
|
384 |
void JfrStorage::release(BufferPtr buffer, Thread* thread) { |
|
385 |
assert(buffer != NULL, "invariant"); |
|
386 |
assert(!buffer->lease(), "invariant"); |
|
387 |
assert(!buffer->transient(), "invariant"); |
|
388 |
assert(!buffer->retired(), "invariant"); |
|
389 |
if (!buffer->empty()) { |
|
390 |
if (!flush_regular_buffer(buffer, thread)) { |
|
391 |
buffer->concurrent_reinitialization(); |
|
392 |
} |
|
393 |
} |
|
394 |
assert(buffer->empty(), "invariant"); |
|
54964 | 395 |
assert(buffer->identity() != NULL, "invariant"); |
50113 | 396 |
control().increment_dead(); |
397 |
buffer->set_retired(); |
|
398 |
} |
|
399 |
||
400 |
void JfrStorage::release_thread_local(BufferPtr buffer, Thread* thread) { |
|
401 |
assert(buffer != NULL, "invariant"); |
|
402 |
JfrStorage& storage_instance = instance(); |
|
403 |
storage_instance.release(buffer, thread); |
|
404 |
if (storage_instance.control().should_scavenge()) { |
|
405 |
storage_instance._post_box.post(MSG_DEADBUFFER); |
|
406 |
} |
|
407 |
} |
|
408 |
||
409 |
static void log_discard(size_t count, size_t amount, size_t current) { |
|
410 |
if (log_is_enabled(Debug, jfr, system)) { |
|
411 |
assert(count > 0, "invariant"); |
|
412 |
log_debug(jfr, system)("Cleared " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" bytes.", count, amount); |
|
413 |
log_debug(jfr, system)("Current number of full buffers " SIZE_FORMAT "", current); |
|
414 |
} |
|
415 |
} |
|
416 |
||
417 |
void JfrStorage::discard_oldest(Thread* thread) { |
|
418 |
if (JfrBuffer_lock->try_lock()) { |
|
419 |
if (!control().should_discard()) { |
|
420 |
// another thread handled it |
|
421 |
return; |
|
422 |
} |
|
423 |
const size_t num_full_pre_discard = control().full_count(); |
|
424 |
size_t num_full_post_discard = 0; |
|
425 |
size_t discarded_size = 0; |
|
426 |
while (true) { |
|
427 |
JfrAgeNode* const oldest_age_node = _age_mspace->full_tail(); |
|
428 |
if (oldest_age_node == NULL) { |
|
429 |
break; |
|
430 |
} |
|
55053
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
431 |
assert(oldest_age_node->identity() == NULL, "invariant"); |
50113 | 432 |
BufferPtr const buffer = oldest_age_node->retired_buffer(); |
433 |
assert(buffer->retired(), "invariant"); |
|
58967
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
434 |
discarded_size += buffer->discard(); |
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
435 |
assert(buffer->unflushed_size() == 0, "invariant"); |
50113 | 436 |
num_full_post_discard = control().decrement_full(); |
58967
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
437 |
mspace_release_full(oldest_age_node, _age_mspace); |
50113 | 438 |
if (buffer->transient()) { |
439 |
mspace_release_full(buffer, _transient_mspace); |
|
440 |
continue; |
|
441 |
} |
|
58967
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
442 |
buffer->reinitialize(); |
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
443 |
buffer->release(); // publish |
3c2e49d43ba3
8232905: JFR fails with assertion: assert(t->unflushed_size() == 0) failed: invariant
mgronlun
parents:
58863
diff
changeset
|
444 |
break; |
50113 | 445 |
} |
446 |
JfrBuffer_lock->unlock(); |
|
447 |
const size_t number_of_discards = num_full_pre_discard - num_full_post_discard; |
|
448 |
if (number_of_discards > 0) { |
|
449 |
log_discard(number_of_discards, discarded_size, num_full_post_discard); |
|
450 |
} |
|
451 |
} |
|
452 |
} |
|
453 |
||
454 |
#ifdef ASSERT |
|
455 |
typedef const BufferPtr ConstBufferPtr; |
|
456 |
||
457 |
static void assert_flush_precondition(ConstBufferPtr cur, size_t used, bool native, const Thread* t) { |
|
458 |
assert(t != NULL, "invariant"); |
|
459 |
assert(cur != NULL, "invariant"); |
|
460 |
assert(cur->pos() + used <= cur->end(), "invariant"); |
|
461 |
assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant"); |
|
462 |
} |
|
463 |
||
464 |
static void assert_flush_regular_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, const Thread* t) { |
|
465 |
assert(t != NULL, "invariant"); |
|
466 |
assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant"); |
|
467 |
assert(cur != NULL, "invariant"); |
|
468 |
assert(!cur->lease(), "invariant"); |
|
469 |
assert(cur_pos != NULL, "invariant"); |
|
470 |
assert(req >= used, "invariant"); |
|
471 |
} |
|
472 |
||
473 |
static void assert_provision_large_precondition(ConstBufferPtr cur, size_t used, size_t req, const Thread* t) { |
|
474 |
assert(cur != NULL, "invariant"); |
|
475 |
assert(t != NULL, "invariant"); |
|
476 |
assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant"); |
|
477 |
assert(req >= used, "invariant"); |
|
478 |
} |
|
479 |
||
480 |
static void assert_flush_large_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { |
|
481 |
assert(t != NULL, "invariant"); |
|
482 |
assert(cur != NULL, "invariant"); |
|
483 |
assert(cur->lease(), "invariant"); |
|
58863 | 484 |
assert(!cur->excluded(), "invariant"); |
50113 | 485 |
assert(cur_pos != NULL, "invariant"); |
486 |
assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant"); |
|
487 |
assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant"); |
|
488 |
assert(req >= used, "invariant"); |
|
489 |
assert(cur != t->jfr_thread_local()->shelved_buffer(), "invariant"); |
|
490 |
} |
|
491 |
#endif // ASSERT |
|
492 |
||
493 |
BufferPtr JfrStorage::flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* t) { |
|
494 |
debug_only(assert_flush_precondition(cur, used, native, t);) |
|
495 |
const u1* const cur_pos = cur->pos(); |
|
496 |
req += used; |
|
497 |
// requested size now encompass the outstanding used size |
|
498 |
return cur->lease() ? instance().flush_large(cur, cur_pos, used, req, native, t) : |
|
499 |
instance().flush_regular(cur, cur_pos, used, req, native, t); |
|
500 |
} |
|
501 |
||
502 |
BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { |
|
503 |
debug_only(assert_flush_regular_precondition(cur, cur_pos, used, req, t);) |
|
504 |
// A flush is needed before memcpy since a non-large buffer is thread stable |
|
505 |
// (thread local). The flush will not modify memory in addresses above pos() |
|
506 |
// which is where the "used / uncommitted" data resides. It is therefore both |
|
507 |
// possible and valid to migrate data after the flush. This is however only |
|
508 |
// the case for stable thread local buffers; it is not the case for large buffers. |
|
509 |
if (!cur->empty()) { |
|
510 |
flush_regular_buffer(cur, t); |
|
58863 | 511 |
if (cur->excluded()) { |
512 |
return cur; |
|
513 |
} |
|
50113 | 514 |
} |
515 |
assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant"); |
|
516 |
if (cur->free_size() >= req) { |
|
517 |
// simplest case, no switching of buffers |
|
518 |
if (used > 0) { |
|
519 |
memcpy(cur->pos(), (void*)cur_pos, used); |
|
520 |
} |
|
521 |
assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant"); |
|
522 |
return cur; |
|
523 |
} |
|
524 |
// Going for a "larger-than-regular" buffer. |
|
525 |
// Shelve the current buffer to make room for a temporary lease. |
|
526 |
t->jfr_thread_local()->shelve_buffer(cur); |
|
527 |
return provision_large(cur, cur_pos, used, req, native, t); |
|
528 |
} |
|
529 |
||
530 |
static BufferPtr store_buffer_to_thread_local(BufferPtr buffer, JfrThreadLocal* jfr_thread_local, bool native) { |
|
531 |
assert(buffer != NULL, "invariant"); |
|
532 |
if (native) { |
|
533 |
jfr_thread_local->set_native_buffer(buffer); |
|
534 |
} else { |
|
535 |
jfr_thread_local->set_java_buffer(buffer); |
|
536 |
} |
|
537 |
return buffer; |
|
538 |
} |
|
539 |
||
540 |
static BufferPtr restore_shelved_buffer(bool native, Thread* t) { |
|
541 |
JfrThreadLocal* const tl = t->jfr_thread_local(); |
|
542 |
BufferPtr shelved = tl->shelved_buffer(); |
|
543 |
assert(shelved != NULL, "invariant"); |
|
544 |
tl->shelve_buffer(NULL); |
|
545 |
// restore shelved buffer back as primary |
|
546 |
return store_buffer_to_thread_local(shelved, tl, native); |
|
547 |
} |
|
548 |
||
549 |
BufferPtr JfrStorage::flush_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { |
|
550 |
debug_only(assert_flush_large_precondition(cur, cur_pos, used, req, native, t);) |
|
551 |
// Can the "regular" buffer (now shelved) accommodate the requested size? |
|
552 |
BufferPtr shelved = t->jfr_thread_local()->shelved_buffer(); |
|
553 |
assert(shelved != NULL, "invariant"); |
|
554 |
if (shelved->free_size() >= req) { |
|
555 |
if (req > 0) { |
|
556 |
memcpy(shelved->pos(), (void*)cur_pos, (size_t)used); |
|
557 |
} |
|
558 |
// release and invalidate |
|
559 |
release_large(cur, t); |
|
560 |
return restore_shelved_buffer(native, t); |
|
561 |
} |
|
562 |
// regular too small |
|
563 |
return provision_large(cur, cur_pos, used, req, native, t); |
|
564 |
} |
|
565 |
||
566 |
static BufferPtr large_fail(BufferPtr cur, bool native, JfrStorage& storage_instance, Thread* t) { |
|
567 |
assert(cur != NULL, "invariant"); |
|
568 |
assert(t != NULL, "invariant"); |
|
569 |
if (cur->lease()) { |
|
570 |
storage_instance.release_large(cur, t); |
|
571 |
} |
|
572 |
return restore_shelved_buffer(native, t); |
|
573 |
} |
|
574 |
||
575 |
// Always returns a non-null buffer. |
|
576 |
// If accommodating the large request fails, the shelved buffer is returned |
|
577 |
// even though it might be smaller than the requested size. |
|
578 |
// Caller needs to ensure if the size was successfully accommodated. |
|
579 |
BufferPtr JfrStorage::provision_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { |
|
580 |
debug_only(assert_provision_large_precondition(cur, used, req, t);) |
|
581 |
assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant"); |
|
582 |
BufferPtr const buffer = acquire_large(req, t); |
|
583 |
if (buffer == NULL) { |
|
584 |
// unable to allocate and serve the request |
|
585 |
return large_fail(cur, native, *this, t); |
|
586 |
} |
|
587 |
// ok managed to acquire a "large" buffer for the requested size |
|
588 |
assert(buffer->free_size() >= req, "invariant"); |
|
589 |
assert(buffer->lease(), "invariant"); |
|
590 |
// transfer outstanding data |
|
591 |
memcpy(buffer->pos(), (void*)cur_pos, used); |
|
592 |
if (cur->lease()) { |
|
593 |
release_large(cur, t); |
|
594 |
// don't use current anymore, it is gone |
|
595 |
} |
|
596 |
return store_buffer_to_thread_local(buffer, t->jfr_thread_local(), native); |
|
597 |
} |
|
598 |
||
599 |
typedef UnBufferedWriteToChunk<JfrBuffer> WriteOperation; |
|
600 |
typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation; |
|
601 |
typedef ConcurrentWriteOp<WriteOperation> ConcurrentWriteOperation; |
|
58863 | 602 |
|
603 |
typedef Retired<JfrBuffer, true> NonRetired; |
|
604 |
typedef Excluded<JfrBuffer, true> NonExcluded; |
|
605 |
typedef CompositeOperation<NonRetired, NonExcluded> BufferPredicate; |
|
606 |
typedef PredicatedMutexedWriteOp<WriteOperation, BufferPredicate> ThreadLocalMutexedWriteOperation; |
|
607 |
typedef PredicatedConcurrentWriteOp<WriteOperation, BufferPredicate> ThreadLocalConcurrentWriteOperation; |
|
50113 | 608 |
|
609 |
size_t JfrStorage::write() { |
|
58863 | 610 |
const size_t full_elements = write_full(); |
50113 | 611 |
WriteOperation wo(_chunkwriter); |
58863 | 612 |
NonRetired nr; |
613 |
NonExcluded ne; |
|
614 |
BufferPredicate bp(&nr, &ne); |
|
615 |
ThreadLocalConcurrentWriteOperation tlwo(wo, bp); |
|
50113 | 616 |
process_full_list(tlwo, _thread_local_mspace); |
617 |
ConcurrentWriteOperation cwo(wo); |
|
618 |
process_free_list(cwo, _global_mspace); |
|
58863 | 619 |
return full_elements + wo.elements(); |
50113 | 620 |
} |
621 |
||
622 |
size_t JfrStorage::write_at_safepoint() { |
|
623 |
assert(SafepointSynchronize::is_at_safepoint(), "invariant"); |
|
624 |
WriteOperation wo(_chunkwriter); |
|
625 |
MutexedWriteOperation writer(wo); // mutexed write mode |
|
58863 | 626 |
NonRetired nr; |
627 |
NonExcluded ne; |
|
628 |
BufferPredicate bp(&nr, &ne); |
|
629 |
ThreadLocalMutexedWriteOperation tlmwo(wo, bp); |
|
630 |
process_full_list(tlmwo, _thread_local_mspace); |
|
50113 | 631 |
assert(_transient_mspace->is_free_empty(), "invariant"); |
632 |
process_full_list(writer, _transient_mspace); |
|
633 |
assert(_global_mspace->is_full_empty(), "invariant"); |
|
634 |
process_free_list(writer, _global_mspace); |
|
58863 | 635 |
return wo.elements(); |
50113 | 636 |
} |
637 |
||
638 |
typedef DiscardOp<DefaultDiscarder<JfrStorage::Buffer> > DiscardOperation; |
|
639 |
typedef ReleaseOp<JfrStorageMspace> ReleaseOperation; |
|
640 |
typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> FullOperation; |
|
641 |
||
642 |
size_t JfrStorage::clear() { |
|
58863 | 643 |
const size_t full_elements = clear_full(); |
50113 | 644 |
DiscardOperation discarder(concurrent); // concurrent discard mode |
645 |
process_full_list(discarder, _thread_local_mspace); |
|
646 |
assert(_transient_mspace->is_free_empty(), "invariant"); |
|
647 |
process_full_list(discarder, _transient_mspace); |
|
648 |
assert(_global_mspace->is_full_empty(), "invariant"); |
|
649 |
process_free_list(discarder, _global_mspace); |
|
58863 | 650 |
return full_elements + discarder.elements(); |
50113 | 651 |
} |
652 |
||
653 |
static void insert_free_age_nodes(JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, JfrAgeNode* tail, size_t count) { |
|
654 |
if (tail != NULL) { |
|
655 |
assert(tail->next() == NULL, "invariant"); |
|
656 |
assert(head != NULL, "invariant"); |
|
657 |
assert(head->prev() == NULL, "invariant"); |
|
54623
1126f0607c70
8222811: Consolidate MutexLockerEx and MutexLocker
coleenp
parents:
50429
diff
changeset
|
658 |
MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag); |
50113 | 659 |
age_mspace->insert_free_tail(head, tail, count); |
660 |
} |
|
661 |
} |
|
662 |
||
663 |
template <typename Processor> |
|
664 |
static void process_age_list(Processor& processor, JfrStorageAgeMspace* age_mspace, JfrAgeNode* head, size_t count) { |
|
665 |
assert(age_mspace != NULL, "invariant"); |
|
666 |
assert(head != NULL, "invariant"); |
|
50234
6ba3e32a9882
8203457: Add back missing full buffer notification
mgronlun
parents:
50117
diff
changeset
|
667 |
assert(count > 0, "invariant"); |
50113 | 668 |
JfrAgeNode* node = head; |
669 |
JfrAgeNode* last = NULL; |
|
670 |
while (node != NULL) { |
|
671 |
last = node; |
|
55053
d58e1a447d2b
8221121: applications/microbenchmarks are encountering crashes in tier5
mgronlun
parents:
54964
diff
changeset
|
672 |
assert(node->identity() == NULL, "invariant"); |
50113 | 673 |
BufferPtr const buffer = node->retired_buffer(); |
674 |
assert(buffer != NULL, "invariant"); |
|
675 |
assert(buffer->retired(), "invariant"); |
|
676 |
processor.process(buffer); |
|
677 |
// at this point, buffer is already live or destroyed |
|
678 |
JfrAgeNode* const next = (JfrAgeNode*)node->next(); |
|
679 |
if (node->transient()) { |
|
680 |
// detach |
|
681 |
last = (JfrAgeNode*)last->prev(); |
|
682 |
if (last != NULL) { |
|
683 |
last->set_next(next); |
|
684 |
} else { |
|
685 |
head = next; |
|
686 |
} |
|
687 |
if (next != NULL) { |
|
688 |
next->set_prev(last); |
|
689 |
} |
|
690 |
--count; |
|
691 |
age_mspace->deallocate(node); |
|
692 |
} |
|
693 |
node = next; |
|
694 |
} |
|
695 |
insert_free_age_nodes(age_mspace, head, last, count); |
|
696 |
} |
|
697 |
||
698 |
template <typename Processor> |
|
699 |
static size_t process_full(Processor& processor, JfrStorageControl& control, JfrStorageAgeMspace* age_mspace) { |
|
700 |
assert(age_mspace != NULL, "invariant"); |
|
701 |
if (age_mspace->is_full_empty()) { |
|
702 |
// nothing to do |
|
703 |
return 0; |
|
704 |
} |
|
705 |
size_t count; |
|
50234
6ba3e32a9882
8203457: Add back missing full buffer notification
mgronlun
parents:
50117
diff
changeset
|
706 |
JfrAgeNode* head; |
50113 | 707 |
{ |
708 |
// fetch age list |
|
54623
1126f0607c70
8222811: Consolidate MutexLockerEx and MutexLocker
coleenp
parents:
50429
diff
changeset
|
709 |
MutexLocker buffer_lock(JfrBuffer_lock, Mutex::_no_safepoint_check_flag); |
50113 | 710 |
count = age_mspace->full_count(); |
711 |
head = age_mspace->clear_full(); |
|
712 |
control.reset_full(); |
|
713 |
} |
|
714 |
assert(head != NULL, "invariant"); |
|
50234
6ba3e32a9882
8203457: Add back missing full buffer notification
mgronlun
parents:
50117
diff
changeset
|
715 |
assert(count > 0, "invariant"); |
50113 | 716 |
process_age_list(processor, age_mspace, head, count); |
717 |
return count; |
|
718 |
} |
|
719 |
||
720 |
static void log(size_t count, size_t amount, bool clear = false) { |
|
721 |
if (log_is_enabled(Debug, jfr, system)) { |
|
722 |
if (count > 0) { |
|
723 |
log_debug(jfr, system)("%s " SIZE_FORMAT " full buffer(s) of " SIZE_FORMAT" B of data%s", |
|
724 |
clear ? "Discarded" : "Wrote", count, amount, clear ? "." : " to chunk."); |
|
725 |
} |
|
726 |
} |
|
727 |
} |
|
728 |
||
729 |
// full writer |
|
730 |
// Assumption is retired only; exclusive access |
|
731 |
// MutexedWriter -> ReleaseOp |
|
732 |
// |
|
733 |
size_t JfrStorage::write_full() { |
|
734 |
assert(_chunkwriter.is_valid(), "invariant"); |
|
735 |
Thread* const thread = Thread::current(); |
|
736 |
WriteOperation wo(_chunkwriter); |
|
737 |
MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access |
|
738 |
ReleaseOperation ro(_transient_mspace, thread); |
|
739 |
FullOperation cmd(&writer, &ro); |
|
740 |
const size_t count = process_full(cmd, control(), _age_mspace); |
|
58863 | 741 |
if (0 == count) { |
742 |
assert(0 == writer.elements(), "invariant"); |
|
743 |
return 0; |
|
744 |
} |
|
745 |
const size_t size = writer.size(); |
|
746 |
log(count, size); |
|
747 |
return count; |
|
50113 | 748 |
} |
749 |
||
750 |
size_t JfrStorage::clear_full() { |
|
751 |
DiscardOperation discarder(mutexed); // a retired buffer implies mutexed access |
|
752 |
const size_t count = process_full(discarder, control(), _age_mspace); |
|
58863 | 753 |
if (0 == count) { |
754 |
assert(0 == discarder.elements(), "invariant"); |
|
755 |
return 0; |
|
756 |
} |
|
757 |
const size_t size = discarder.size(); |
|
758 |
log(count, size, true); |
|
759 |
return count; |
|
50113 | 760 |
} |
761 |
||
762 |
static void scavenge_log(size_t count, size_t amount, size_t current) { |
|
763 |
if (count > 0) { |
|
764 |
if (log_is_enabled(Debug, jfr, system)) { |
|
765 |
log_debug(jfr, system)("Released " SIZE_FORMAT " dead buffer(s) of " SIZE_FORMAT" B of data.", count, amount); |
|
766 |
log_debug(jfr, system)("Current number of dead buffers " SIZE_FORMAT "", current); |
|
767 |
} |
|
768 |
} |
|
769 |
} |
|
770 |
||
771 |
template <typename Mspace> |
|
772 |
class Scavenger { |
|
773 |
private: |
|
774 |
JfrStorageControl& _control; |
|
775 |
Mspace* _mspace; |
|
776 |
size_t _count; |
|
777 |
size_t _amount; |
|
778 |
public: |
|
779 |
typedef typename Mspace::Type Type; |
|
780 |
Scavenger(JfrStorageControl& control, Mspace* mspace) : _control(control), _mspace(mspace), _count(0), _amount(0) {} |
|
781 |
bool process(Type* t) { |
|
782 |
if (t->retired()) { |
|
54964 | 783 |
assert(t->identity() != NULL, "invariant"); |
784 |
assert(t->empty(), "invariant"); |
|
50113 | 785 |
assert(!t->transient(), "invariant"); |
786 |
assert(!t->lease(), "invariant"); |
|
787 |
++_count; |
|
788 |
_amount += t->total_size(); |
|
58863 | 789 |
if (t->excluded()) { |
790 |
t->clear_excluded(); |
|
791 |
} |
|
792 |
assert(!t->excluded(), "invariant"); |
|
50113 | 793 |
t->clear_retired(); |
54964 | 794 |
t->release(); |
50113 | 795 |
_control.decrement_dead(); |
796 |
mspace_release_full_critical(t, _mspace); |
|
797 |
} |
|
798 |
return true; |
|
799 |
} |
|
800 |
size_t processed() const { return _count; } |
|
801 |
size_t amount() const { return _amount; } |
|
802 |
}; |
|
803 |
||
804 |
size_t JfrStorage::scavenge() { |
|
805 |
JfrStorageControl& ctrl = control(); |
|
806 |
if (ctrl.dead_count() == 0) { |
|
807 |
return 0; |
|
808 |
} |
|
809 |
Scavenger<JfrThreadLocalMspace> scavenger(ctrl, _thread_local_mspace); |
|
810 |
process_full_list(scavenger, _thread_local_mspace); |
|
58863 | 811 |
const size_t count = scavenger.processed(); |
812 |
if (0 == count) { |
|
813 |
assert(0 == scavenger.amount(), "invariant"); |
|
814 |
return 0; |
|
815 |
} |
|
816 |
scavenge_log(count, scavenger.amount(), ctrl.dead_count()); |
|
817 |
return count; |
|
50113 | 818 |
} |