58 // set data iff *dest == NULL

    59 // incremented on each flushpoint

    59 static bool try_set(void* const data, void** dest, bool clear) {

    60 static u8 flushpoint_id = 0;

    60   assert(data != NULL, "invariant");

    61 

    61   const void* const current = OrderAccess::load_acquire(dest);

    62 template <typename E, typename Instance, size_t(Instance::*func)()>

    62   if (current != NULL) {

    63 class Content {

    63     if (current != data) {

    64       // already set

    65       return false;

    66     }

    67     assert(current == data, "invariant");

    68     if (!clear) {

    69       // recursion disallowed

    70       return false;

    71     }

    72   }

    73   return Atomic::cmpxchg(clear ? NULL : data, dest, current) == current;

    74 }

    75 

    76 static void* rotation_thread = NULL;

    77 static const int rotation_try_limit = 1000;

    78 static const int rotation_retry_sleep_millis = 10;

    79 

    80 class RotationLock : public StackObj {

    82   Thread* const _thread;

    65   Instance& _instance;

    83   bool _acquired;

    66   u4 _elements;

    84 

    85   void log(bool recursion) {

    86     assert(!_acquired, "invariant");

    87     const char* error_msg = NULL;

    88     if (recursion) {

    89       error_msg = "Unable to issue rotation due to recursive calls.";

    90     }

    91     else {

    92       error_msg = "Unable to issue rotation due to wait timeout.";

    93     }

    94     log_info(jfr)( // For user, should not be "jfr, system"

    95       "%s", error_msg);

    96   }

    98   RotationLock(Thread* thread) : _thread(thread), _acquired(false) {

    68   typedef E EventType;

    99     assert(_thread != NULL, "invariant");

    69   Content(Instance& instance) : _instance(instance), _elements(0) {}

   100     if (_thread == rotation_thread) {

    70   bool process() {

   101       // recursion not supported

    71     _elements = (u4)(_instance.*func)();

   102       log(true);

    72     return true;

   103       return;

    73   }

   104     }

    74   u4 elements() const { return _elements; }

   105 

    75 };

   106     // limited to not spin indefinitely

    76 

   107     for (int i = 0; i < rotation_try_limit; ++i) {

    77 template <typename Content>

   108       if (try_set(_thread, &rotation_thread, false)) {

    78 class WriteContent : public StackObj {

   109         _acquired = true;

    79  protected:

   110         assert(_thread == rotation_thread, "invariant");

    80   const JfrTicks _start_time;

   111         return;

    81   JfrTicks _end_time;

   112       }

    82   JfrChunkWriter& _cw;

   113       if (_thread->is_Java_thread()) {

    83   Content& _content;

   114         // in order to allow the system to move to a safepoint

    84   const int64_t _start_offset;

   115         MutexLocker msg_lock(JfrMsg_lock);

    85  public:

   116         JfrMsg_lock->wait(rotation_retry_sleep_millis);

    86   typedef typename Content::EventType EventType;

   117       }

    87 

   118       else {

    88   WriteContent(JfrChunkWriter& cw, Content& content) :

   119         os::naked_short_sleep(rotation_retry_sleep_millis);

    89     _start_time(JfrTicks::now()),

   120       }

    90     _end_time(),

   121     }

    91     _cw(cw),

   122     log(false);

    92     _content(content),

   123   }

    93     _start_offset(_cw.current_offset()) {

   124 

    94     assert(_cw.is_valid(), "invariant");

   125   ~RotationLock() {

    95   }

   126     assert(_thread != NULL, "invariant");

    96 

   127     if (_acquired) {

    97   bool process() {

   128       assert(_thread == rotation_thread, "invariant");

    98     // invocation

   129       while (!try_set(_thread, &rotation_thread, true));

    99     _content.process();

   130     }

   100     _end_time = JfrTicks::now();

   131   }

   101     return 0 != _content.elements();

   132   bool not_acquired() const { return !_acquired; }

   102   }

   103 

   104   const JfrTicks& start_time() const {

   105     return _start_time;

   106   }

   107 

   108   const JfrTicks& end_time() const {

   109     return _end_time;

   110   }

   111 

   112   int64_t start_offset() const {

   113     return _start_offset;

   114   }

   115 

   116   int64_t end_offset() const {

   117     return current_offset();

   118   }

   119 

   120   int64_t current_offset() const {

   121     return _cw.current_offset();

   122   }

   123 

   124   u4 elements() const {

   125     return (u4) _content.elements();

   126   }

   127 

   128   u4 size() const {

   129     return (u4)(end_offset() - start_offset());

   130   }

   131 

   132   static bool is_event_enabled() {

   133     return EventType::is_enabled();

   134   }

   135 

   136   static u8 event_id() {

   137     return EventType::eventId;

   138   }

   139 

   140   void write_elements(int64_t offset) {

   141     _cw.write_padded_at_offset<u4>(elements(), offset);

   142   }

   143 

   144   void write_size() {

   145     _cw.write_padded_at_offset<u4>(size(), start_offset());

   146   }

   147 

   148   void set_last_checkpoint() {

   149     _cw.set_last_checkpoint_offset(start_offset());

   150   }

   151 

   152   void rewind() {

   153     _cw.seek(start_offset());

   154   }

   141   cw.write((int64_t)0); // duration

   163   cw.write<u8>(0); // duration

   143   cw.write<bool>(false); // flushpoint

   165   cw.write<u4>(GENERIC); // checkpoint type

   144   cw.write((u4)1); // nof types in this checkpoint

   166   cw.write<u4>(1); // nof types in this checkpoint

   146   const int64_t number_of_elements_offset = cw.current_offset();

   168   return cw.reserve(sizeof(u4));

   147   cw.reserve(sizeof(u4));

   169 }

   148   return number_of_elements_offset;

   170 

   149 }

   171 template <typename Content>

   150 

   172 class WriteCheckpointEvent : public WriteContent<Content> {

   151 template <typename ContentFunctor>

   173  private:

   152 class WriteCheckpointEvent : public StackObj {

   174   const u8 _type_id;

   175  public:

   176   WriteCheckpointEvent(JfrChunkWriter& cw, Content& content, u8 type_id) :

   177     WriteContent<Content>(cw, content), _type_id(type_id) {}

   178 

   179   bool process() {

   180     const int64_t num_elements_offset = write_checkpoint_event_prologue(this->_cw, _type_id);

   181     if (!WriteContent<Content>::process()) {

   182       // nothing to do, rewind writer to start

   183       this->rewind();

   184       assert(this->current_offset() == this->start_offset(), "invariant");

   185       return false;

   186     }

   187     assert(this->elements() > 0, "invariant");

   188     assert(this->current_offset() > num_elements_offset, "invariant");

   189     this->write_elements(num_elements_offset);

   190     this->write_size();

   191     this->set_last_checkpoint();

   192     return true;

   193   }

   194 };

   195 

   196 template <typename Functor>

   197 static u4 invoke(Functor& f) {

   198   f.process();

   199   return f.elements();

   200 }

   201 

   202 template <typename Functor>

   203 static void write_flush_event(Functor& f) {

   204   if (Functor::is_event_enabled()) {

   205     typename Functor::EventType e(UNTIMED);

   206     e.set_starttime(f.start_time());

   207     e.set_endtime(f.end_time());

   208     e.set_flushId(flushpoint_id);

   209     e.set_elements(f.elements());

   210     e.set_size(f.size());

   211     e.commit();

   212   }

   213 }

   214 

   215 template <typename Functor>

   216 static u4 invoke_with_flush_event(Functor& f) {

   217   const u4 elements = invoke(f);

   218   write_flush_event(f);

   219   return elements;

   220 }

   221 

   222 class StackTraceRepository : public StackObj {

   223  private:

   224   JfrStackTraceRepository& _repo;

   225   JfrChunkWriter& _cw;

   226   size_t _elements;

   227   bool _clear;

   228 

   229  public:

   230   typedef EventFlushStacktrace EventType;

   231   StackTraceRepository(JfrStackTraceRepository& repo, JfrChunkWriter& cw, bool clear) :

   232     _repo(repo), _cw(cw), _elements(0), _clear(clear) {}

   233   bool process() {

   234     _elements = _repo.write(_cw, _clear);

   235     return true;

   236   }

   237   size_t elements() const { return _elements; }

   238   void reset() { _elements = 0; }

   239 };

   240 

   241 typedef WriteCheckpointEvent<StackTraceRepository> WriteStackTrace;

   242 

   243 static u4 flush_stacktrace(JfrStackTraceRepository& stack_trace_repo, JfrChunkWriter& chunkwriter) {

   244   StackTraceRepository str(stack_trace_repo, chunkwriter, false);

   245   WriteStackTrace wst(chunkwriter, str, TYPE_STACKTRACE);

   246   return invoke_with_flush_event(wst);

   247 }

   248 

   249 static u4 write_stacktrace(JfrStackTraceRepository& stack_trace_repo, JfrChunkWriter& chunkwriter, bool clear) {

   250   StackTraceRepository str(stack_trace_repo, chunkwriter, clear);

   251   WriteStackTrace wst(chunkwriter, str, TYPE_STACKTRACE);

   252   return invoke(wst);

   253 }

   254 

   255 typedef Content<EventFlushStorage, JfrStorage, &JfrStorage::write> Storage;

   256 typedef WriteContent<Storage> WriteStorage;

   257 

   258 static size_t flush_storage(JfrStorage& storage, JfrChunkWriter& chunkwriter) {

   259   assert(chunkwriter.is_valid(), "invariant");

   260   Storage fsf(storage);

   261   WriteStorage fs(chunkwriter, fsf);

   262   return invoke_with_flush_event(fs);

   263 }

   264 

   265 static size_t write_storage(JfrStorage& storage, JfrChunkWriter& chunkwriter) {

   266   assert(chunkwriter.is_valid(), "invariant");

   267   Storage fsf(storage);

   268   WriteStorage fs(chunkwriter, fsf);

   269   return invoke(fs);

   270 }

   271 

   272 typedef Content<EventFlushStringPool, JfrStringPool, &JfrStringPool::write> StringPool;

   273 typedef Content<EventFlushStringPool, JfrStringPool, &JfrStringPool::write_at_safepoint> StringPoolSafepoint;

   274 typedef WriteCheckpointEvent<StringPool> WriteStringPool;

   275 typedef WriteCheckpointEvent<StringPoolSafepoint> WriteStringPoolSafepoint;

   276 

   277 static u4 flush_stringpool(JfrStringPool& string_pool, JfrChunkWriter& chunkwriter) {

   278   StringPool sp(string_pool);

   279   WriteStringPool wsp(chunkwriter, sp, TYPE_STRING);

   280   return invoke_with_flush_event(wsp);

   281 }

   282 

   283 static u4 write_stringpool(JfrStringPool& string_pool, JfrChunkWriter& chunkwriter) {

   284   StringPool sp(string_pool);

   285   WriteStringPool wsp(chunkwriter, sp, TYPE_STRING);

   286   return invoke(wsp);

   287 }

   288 

   289 static u4 write_stringpool_safepoint(JfrStringPool& string_pool, JfrChunkWriter& chunkwriter) {

   290   StringPoolSafepoint sps(string_pool);

   291   WriteStringPoolSafepoint wsps(chunkwriter, sps, TYPE_STRING);

   292   return invoke(wsps);

   293 }

   294 

   295 typedef Content<EventFlushTypeSet, JfrCheckpointManager, &JfrCheckpointManager::flush_type_set> FlushTypeSetFunctor;

   296 typedef WriteContent<FlushTypeSetFunctor> FlushTypeSet;

   297 

   298 static u4 flush_typeset(JfrCheckpointManager& checkpoint_manager, JfrChunkWriter& chunkwriter) {

   299   FlushTypeSetFunctor flush_type_set(checkpoint_manager);

   300   FlushTypeSet fts(chunkwriter, flush_type_set);

   301   return invoke_with_flush_event(fts);

   302 }

   303 

   304 class MetadataEvent : public StackObj {

   158   WriteCheckpointEvent(JfrChunkWriter& cw, u8 type_id, ContentFunctor& functor) :

   308   typedef EventFlushMetadata EventType;

   159     _cw(cw),

   309   MetadataEvent(JfrChunkWriter& cw) : _cw(cw) {}

   160     _type_id(type_id),

   161     _content_functor(functor) {

   162     assert(_cw.is_valid(), "invariant");

   163   }

   165     // current_cp_offset is also offset for the event size header field

   311     JfrMetadataEvent::write(_cw);

   166     const int64_t current_cp_offset = _cw.current_offset();

   167     const int64_t num_elements_offset = write_checkpoint_event_prologue(_cw, _type_id);

   168     // invocation

   169     _content_functor.process();

   170     const u4 number_of_elements = (u4)_content_functor.processed();

   171     if (number_of_elements == 0) {

   172       // nothing to do, rewind writer to start

   173       _cw.seek(current_cp_offset);

   174       return true;

   175     }

   176     assert(number_of_elements > 0, "invariant");

   177     assert(_cw.current_offset() > num_elements_offset, "invariant");

   178     _cw.write_padded_at_offset<u4>(number_of_elements, num_elements_offset);

   179     _cw.write_padded_at_offset<u4>((u4)_cw.current_offset() - current_cp_offset, current_cp_offset);

   180     // update writer with last checkpoint position

   181     _cw.set_last_checkpoint_offset(current_cp_offset);

   186 template <typename Instance, size_t(Instance::*func)()>

   317 typedef WriteContent<MetadataEvent> WriteMetadata;

   187 class ServiceFunctor {

   318 

   188  private:

   319 static u4 flush_metadata(JfrChunkWriter& chunkwriter) {

   189   Instance& _instance;

   320   assert(chunkwriter.is_valid(), "invariant");

   190   size_t _processed;

   321   MetadataEvent me(chunkwriter);

   191  public:

   322   WriteMetadata wm(chunkwriter, me);

   192   ServiceFunctor(Instance& instance) : _instance(instance), _processed(0) {}

   323   return invoke_with_flush_event(wm);

   193   bool process() {

   324 }

   194     _processed = (_instance.*func)();

   325 

   195     return true;

   326 static u4 write_metadata(JfrChunkWriter& chunkwriter) {

   196   }

   327   assert(chunkwriter.is_valid(), "invariant");

   197   size_t processed() const { return _processed; }

   328   MetadataEvent me(chunkwriter);

   198 };

   329   WriteMetadata wm(chunkwriter, me);

   330   return invoke(wm);

   331 }

   249   RotationLock rl(Thread::current());

   364   MutexLocker lock(JfrStream_lock);

   250   if (rl.not_acquired()) {

   251     return;

   252   }

   296   _chunkwriter.time_stamp_chunk_now();

   398   _chunkwriter.set_time_stamp();

   399   _stack_trace_repository.clear();

   311   RotationLock rl(Thread::current());

   444   assert(!JfrStream_lock->owned_by_self(), "invariant");

   312   if (rl.not_acquired()) {

   445   MutexLocker lock(JfrStream_lock);

   313     return;

   314   }

   355   assert(!_chunkwriter.is_valid(), "invariant");

   464   assert(JfrStream_lock->owned_by_self(), "invariant");

   360     serialize_storage_from_in_memory_recording();

   470     write_storage(_storage, _chunkwriter);

   361   }

   471   }

   362 }

   363 

   364 void JfrRecorderService::serialize_storage_from_in_memory_recording() {

   365   assert(!JfrStream_lock->owned_by_self(), "not holding stream lock!");

   366   MutexLocker stream_lock(JfrStream_lock, Mutex::_no_safepoint_check_flag);

   367   _storage.write();

   436   _storage.write();

   500   if (_string_pool.is_modified()) {

   501     write_stringpool(_string_pool, _chunkwriter);

   502   }

   503   write_storage(_storage, _chunkwriter);

   504   if (_stack_trace_repository.is_modified()) {

   505     write_stacktrace(_stack_trace_repository, _chunkwriter, false);

   506   }

   459   MutexLocker stream_lock(JfrStream_lock, Mutex::_no_safepoint_check_flag);

   516   if (_string_pool.is_modified()) {

   460   write_stacktrace_checkpoint(_stack_trace_repository, _chunkwriter, true);

   517     write_stringpool_safepoint(_string_pool, _chunkwriter);

   461   write_stringpool_checkpoint_safepoint(_string_pool, _chunkwriter);

   518   }

   462   _checkpoint_manager.write_safepoint_types();

   519   _checkpoint_manager.on_rotation();

   465   _chunkwriter.time_stamp_chunk_now();

   522   _chunkwriter.set_time_stamp();

   466   JfrMetadataEvent::lock();

   523   write_stacktrace(_stack_trace_repository, _chunkwriter, true);

   467 }

   524 }

   468 

   525 

   469 static int64_t write_metadata_event(JfrChunkWriter& chunkwriter) {

   470   assert(chunkwriter.is_valid(), "invariant");

   471   const int64_t metadata_offset = chunkwriter.current_offset();

   472   JfrMetadataEvent::write(chunkwriter, metadata_offset);

   473   return metadata_offset;

   474 }

   475 

   476 //

   477 // post-safepoint write sequence

   478 //

   479 //   write type set ->

   480 //     release object sampler ->

   481 //       lock stream lock ->

   482 //         write checkpoints ->

   483 //           write metadata event ->

   484 //             write chunk header ->

   485 //               close chunk fd ->

   486 //                 release stream lock

   487 //

   504   _repository.close_chunk(write_metadata_event(_chunkwriter));

   539   write_metadata(_chunkwriter);

   505   assert(!_chunkwriter.is_valid(), "invariant");

   540   _repository.close_chunk();

   506 }

   541 }

   507 

   542 

   508 void JfrRecorderService::vm_error_rotation() {

   543 static JfrBuffer* thread_local_buffer(Thread* t) {

   509   if (_chunkwriter.is_valid()) {

   544   assert(t != NULL, "invariant");

   510     finalize_current_chunk_on_vm_error();

   545   return t->jfr_thread_local()->native_buffer();

   511     assert(!_chunkwriter.is_valid(), "invariant");

   546 }

   512     _repository.on_vm_error();

   547 

   513   }

   548 static void reset_buffer(JfrBuffer* buffer, Thread* t) {

   514 }

   549   assert(buffer != NULL, "invariant");

   515 

   550   assert(t != NULL, "invariant");

   516 void JfrRecorderService::finalize_current_chunk_on_vm_error() {

   551   assert(buffer == thread_local_buffer(t), "invariant");

   552   buffer->set_pos(const_cast<u1*>(buffer->top()));

   553 }

   554 

   555 static void reset_thread_local_buffer(Thread* t) {

   556   reset_buffer(thread_local_buffer(t), t);

   557 }

   558 

   559 static void write_thread_local_buffer(JfrChunkWriter& chunkwriter, Thread* t) {

   560   JfrBuffer * const buffer = thread_local_buffer(t);

   561   assert(buffer != NULL, "invariant");

   562   if (!buffer->empty()) {

   563     chunkwriter.write_unbuffered(buffer->top(), buffer->pos() - buffer->top());

   564     reset_buffer(buffer, t);

   565   }

   566 }

   567 

   568 size_t JfrRecorderService::flush() {

   569   assert(JfrStream_lock->owned_by_self(), "invariant");

   570   size_t total_elements = flush_metadata(_chunkwriter);

   571   const size_t storage_elements = flush_storage(_storage, _chunkwriter);

   572   if (0 == storage_elements) {

   573     return total_elements;

   574   }

   575   total_elements += storage_elements;

   576   if (_string_pool.is_modified()) {

   577     total_elements += flush_stringpool(_string_pool, _chunkwriter);

   578   }

   579   if (_stack_trace_repository.is_modified()) {

   580     total_elements += flush_stacktrace(_stack_trace_repository, _chunkwriter);

   581   }

   582   if (_checkpoint_manager.is_type_set_required()) {

   583     total_elements += flush_typeset(_checkpoint_manager, _chunkwriter);

   584   } else if (_checkpoint_manager.is_static_type_set_required()) {

   585     // don't tally this, it is only in order to flush the waiting constants

   586     _checkpoint_manager.flush_static_type_set();

   587   }

   588   return total_elements;

   589 }

   590 

   591 typedef Content<EventFlush, JfrRecorderService, &JfrRecorderService::flush> FlushFunctor;

   592 typedef WriteContent<FlushFunctor> Flush;

   593 

   594 void JfrRecorderService::invoke_flush() {

   595   assert(JfrStream_lock->owned_by_self(), "invariant");

   518   pre_safepoint_write();

   597   Thread* const t = Thread::current();

   519   // Do not attempt safepoint dependent operations during emergency dump.

   598   ResourceMark rm(t);

   520   // Optimistically write tagged artifacts.

   599   HandleMark hm(t);

   521   _checkpoint_manager.shift_epoch();

   600   ++flushpoint_id;

   522   // update time

   601   reset_thread_local_buffer(t);

   523   _chunkwriter.time_stamp_chunk_now();

   602   FlushFunctor flushpoint(*this);

   524   post_safepoint_write();

   603   Flush fl(_chunkwriter, flushpoint);

   525   assert(!_chunkwriter.is_valid(), "invariant");

   604   invoke_with_flush_event(fl);

   605   write_thread_local_buffer(_chunkwriter, t);

   606   _repository.flush_chunk();

   607 }

   608 

   609 void JfrRecorderService::flushpoint() {

   610   MutexLocker lock(JfrStream_lock);

   611   invoke_flush();