1 /*

     2  * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "gc/g1/dirtyCardQueue.hpp"

    27 #include "gc/g1/g1CollectedHeap.inline.hpp"

    28 #include "gc/g1/heapRegionRemSet.hpp"

    29 #include "gc/shared/workgroup.hpp"

    30 #include "runtime/atomic.inline.hpp"

    31 #include "runtime/mutexLocker.hpp"

    32 #include "runtime/safepoint.hpp"

    33 #include "runtime/thread.inline.hpp"

    34 

    35 bool DirtyCardQueue::apply_closure(CardTableEntryClosure* cl,

    36                                    bool consume,

    37                                    uint worker_i) {

    38   bool res = true;

    39   if (_buf != NULL) {

    40     res = apply_closure_to_buffer(cl, _buf, _index, _sz,

    41                                   consume,

    42                                   worker_i);

    43     if (res && consume) _index = _sz;

    44   }

    45   return res;

    46 }

    47 

    48 bool DirtyCardQueue::apply_closure_to_buffer(CardTableEntryClosure* cl,

    49                                              void** buf,

    50                                              size_t index, size_t sz,

    51                                              bool consume,

    52                                              uint worker_i) {

    53   if (cl == NULL) return true;

    54   for (size_t i = index; i < sz; i += oopSize) {

    55     int ind = byte_index_to_index((int)i);

    56     jbyte* card_ptr = (jbyte*)buf[ind];

    57     if (card_ptr != NULL) {

    58       // Set the entry to null, so we don't do it again (via the test

    59       // above) if we reconsider this buffer.

    60       if (consume) buf[ind] = NULL;

    61       if (!cl->do_card_ptr(card_ptr, worker_i)) return false;

    62     }

    63   }

    64   return true;

    65 }

    66 

    67 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away

    68 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list

    69 #endif // _MSC_VER

    70 

    71 DirtyCardQueueSet::DirtyCardQueueSet(bool notify_when_complete) :

    72   PtrQueueSet(notify_when_complete),

    73   _mut_process_closure(NULL),

    74   _shared_dirty_card_queue(this, true /*perm*/),

    75   _free_ids(NULL),

    76   _processed_buffers_mut(0), _processed_buffers_rs_thread(0)

    77 {

    78   _all_active = true;

    79 }

    80 

    81 // Determines how many mutator threads can process the buffers in parallel.

    82 uint DirtyCardQueueSet::num_par_ids() {

    83   return (uint)os::processor_count();

    84 }

    85 

    86 void DirtyCardQueueSet::initialize(CardTableEntryClosure* cl, Monitor* cbl_mon, Mutex* fl_lock,

    87                                    int process_completed_threshold,

    88                                    int max_completed_queue,

    89                                    Mutex* lock, PtrQueueSet* fl_owner) {

    90   _mut_process_closure = cl;

    91   PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold,

    92                           max_completed_queue, fl_owner);

    93   set_buffer_size(G1UpdateBufferSize);

    94   _shared_dirty_card_queue.set_lock(lock);

    95   _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon);

    96 }

    97 

    98 void DirtyCardQueueSet::handle_zero_index_for_thread(JavaThread* t) {

    99   t->dirty_card_queue().handle_zero_index();

   100 }

   101 

   102 void DirtyCardQueueSet::iterate_closure_all_threads(CardTableEntryClosure* cl,

   103                                                     bool consume,

   104                                                     uint worker_i) {

   105   assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");

   106   for(JavaThread* t = Threads::first(); t; t = t->next()) {

   107     bool b = t->dirty_card_queue().apply_closure(cl, consume);

   108     guarantee(b, "Should not be interrupted.");

   109   }

   110   bool b = shared_dirty_card_queue()->apply_closure(cl,

   111                                                     consume,

   112                                                     worker_i);

   113   guarantee(b, "Should not be interrupted.");

   114 }

   115 

   116 bool DirtyCardQueueSet::mut_process_buffer(void** buf) {

   117 

   118   // Used to determine if we had already claimed a par_id

   119   // before entering this method.

   120   bool already_claimed = false;

   121 

   122   // We grab the current JavaThread.

   123   JavaThread* thread = JavaThread::current();

   124 

   125   // We get the the number of any par_id that this thread

   126   // might have already claimed.

   127   uint worker_i = thread->get_claimed_par_id();

   128 

   129   // If worker_i is not UINT_MAX then the thread has already claimed

   130   // a par_id. We make note of it using the already_claimed value

   131   if (worker_i != UINT_MAX) {

   132     already_claimed = true;

   133   } else {

   134 

   135     // Otherwise we need to claim a par id

   136     worker_i = _free_ids->claim_par_id();

   137 

   138     // And store the par_id value in the thread

   139     thread->set_claimed_par_id(worker_i);

   140   }

   141 

   142   bool b = false;

   143   if (worker_i != UINT_MAX) {

   144     b = DirtyCardQueue::apply_closure_to_buffer(_mut_process_closure, buf, 0,

   145                                                 _sz, true, worker_i);

   146     if (b) Atomic::inc(&_processed_buffers_mut);

   147 

   148     // If we had not claimed an id before entering the method

   149     // then we must release the id.

   150     if (!already_claimed) {

   151 

   152       // we release the id

   153       _free_ids->release_par_id(worker_i);

   154 

   155       // and set the claimed_id in the thread to UINT_MAX

   156       thread->set_claimed_par_id(UINT_MAX);

   157     }

   158   }

   159   return b;

   160 }

   161 

   162 

   163 BufferNode*

   164 DirtyCardQueueSet::get_completed_buffer(int stop_at) {

   165   BufferNode* nd = NULL;

   166   MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);

   167 

   168   if ((int)_n_completed_buffers <= stop_at) {

   169     _process_completed = false;

   170     return NULL;

   171   }

   172 

   173   if (_completed_buffers_head != NULL) {

   174     nd = _completed_buffers_head;

   175     _completed_buffers_head = nd->next();

   176     if (_completed_buffers_head == NULL)

   177       _completed_buffers_tail = NULL;

   178     _n_completed_buffers--;

   179     assert(_n_completed_buffers >= 0, "Invariant");

   180   }

   181   debug_only(assert_completed_buffer_list_len_correct_locked());

   182   return nd;

   183 }

   184 

   185 bool DirtyCardQueueSet::

   186 apply_closure_to_completed_buffer_helper(CardTableEntryClosure* cl,

   187                                          uint worker_i,

   188                                          BufferNode* nd) {

   189   if (nd != NULL) {

   190     void **buf = BufferNode::make_buffer_from_node(nd);

   191     size_t index = nd->index();

   192     bool b =

   193       DirtyCardQueue::apply_closure_to_buffer(cl, buf,

   194                                               index, _sz,

   195                                               true, worker_i);

   196     if (b) {

   197       deallocate_buffer(buf);

   198       return true;  // In normal case, go on to next buffer.

   199     } else {

   200       enqueue_complete_buffer(buf, index);

   201       return false;

   202     }

   203   } else {

   204     return false;

   205   }

   206 }

   207 

   208 bool DirtyCardQueueSet::apply_closure_to_completed_buffer(CardTableEntryClosure* cl,

   209                                                           uint worker_i,

   210                                                           int stop_at,

   211                                                           bool during_pause) {

   212   assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause");

   213   BufferNode* nd = get_completed_buffer(stop_at);

   214   bool res = apply_closure_to_completed_buffer_helper(cl, worker_i, nd);

   215   if (res) Atomic::inc(&_processed_buffers_rs_thread);

   216   return res;

   217 }

   218 

   219 void DirtyCardQueueSet::apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) {

   220   BufferNode* nd = _completed_buffers_head;

   221   while (nd != NULL) {

   222     bool b =

   223       DirtyCardQueue::apply_closure_to_buffer(cl,

   224                                               BufferNode::make_buffer_from_node(nd),

   225                                               0, _sz, false);

   226     guarantee(b, "Should not stop early.");

   227     nd = nd->next();

   228   }

   229 }

   230 

   231 void DirtyCardQueueSet::par_apply_closure_to_all_completed_buffers(CardTableEntryClosure* cl) {

   232   BufferNode* nd = _cur_par_buffer_node;

   233   while (nd != NULL) {

   234     BufferNode* next = (BufferNode*)nd->next();

   235     BufferNode* actual = (BufferNode*)Atomic::cmpxchg_ptr((void*)next, (volatile void*)&_cur_par_buffer_node, (void*)nd);

   236     if (actual == nd) {

   237       bool b =

   238         DirtyCardQueue::apply_closure_to_buffer(cl,

   239                                                 BufferNode::make_buffer_from_node(actual),

   240                                                 0, _sz, false);

   241       guarantee(b, "Should not stop early.");

   242       nd = next;

   243     } else {

   244       nd = actual;

   245     }

   246   }

   247 }

   248 

   249 // Deallocates any completed log buffers

   250 void DirtyCardQueueSet::clear() {

   251   BufferNode* buffers_to_delete = NULL;

   252   {

   253     MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag);

   254     while (_completed_buffers_head != NULL) {

   255       BufferNode* nd = _completed_buffers_head;

   256       _completed_buffers_head = nd->next();

   257       nd->set_next(buffers_to_delete);

   258       buffers_to_delete = nd;

   259     }

   260     _n_completed_buffers = 0;

   261     _completed_buffers_tail = NULL;

   262     debug_only(assert_completed_buffer_list_len_correct_locked());

   263   }

   264   while (buffers_to_delete != NULL) {

   265     BufferNode* nd = buffers_to_delete;

   266     buffers_to_delete = nd->next();

   267     deallocate_buffer(BufferNode::make_buffer_from_node(nd));

   268   }

   269 

   270 }

   271 

   272 void DirtyCardQueueSet::abandon_logs() {

   273   assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");

   274   clear();

   275   // Since abandon is done only at safepoints, we can safely manipulate

   276   // these queues.

   277   for (JavaThread* t = Threads::first(); t; t = t->next()) {

   278     t->dirty_card_queue().reset();

   279   }

   280   shared_dirty_card_queue()->reset();

   281 }

   282 

   283 

   284 void DirtyCardQueueSet::concatenate_logs() {

   285   // Iterate over all the threads, if we find a partial log add it to

   286   // the global list of logs.  Temporarily turn off the limit on the number

   287   // of outstanding buffers.

   288   int save_max_completed_queue = _max_completed_queue;

   289   _max_completed_queue = max_jint;

   290   assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint.");

   291   for (JavaThread* t = Threads::first(); t; t = t->next()) {

   292     DirtyCardQueue& dcq = t->dirty_card_queue();

   293     if (dcq.size() != 0) {

   294       void **buf = t->dirty_card_queue().get_buf();

   295       // We must NULL out the unused entries, then enqueue.

   296       for (size_t i = 0; i < t->dirty_card_queue().get_index(); i += oopSize) {

   297         buf[PtrQueue::byte_index_to_index((int)i)] = NULL;

   298       }

   299       enqueue_complete_buffer(dcq.get_buf(), dcq.get_index());

   300       dcq.reinitialize();

   301     }

   302   }

   303   if (_shared_dirty_card_queue.size() != 0) {

   304     enqueue_complete_buffer(_shared_dirty_card_queue.get_buf(),

   305                             _shared_dirty_card_queue.get_index());

   306     _shared_dirty_card_queue.reinitialize();

   307   }

   308   // Restore the completed buffer queue limit.

   309   _max_completed_queue = save_max_completed_queue;

   310 }