1 /*

     2  * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  *

    23  */

    24 

    25 #include "precompiled.hpp"

    26 #include "gc/parallel/gcTaskManager.hpp"

    27 #include "gc/parallel/gcTaskThread.hpp"

    28 #include "gc/shared/gcId.hpp"

    29 #include "gc/shared/workerManager.hpp"

    30 #include "gc/shared/workerPolicy.hpp"

    31 #include "logging/log.hpp"

    32 #include "logging/logStream.hpp"

    33 #include "memory/allocation.hpp"

    34 #include "memory/allocation.inline.hpp"

    35 #include "memory/resourceArea.hpp"

    36 #include "runtime/mutex.hpp"

    37 #include "runtime/mutexLocker.hpp"

    38 #include "runtime/orderAccess.hpp"

    39 #include "runtime/os.hpp"

    40 

    41 //

    42 // GCTask

    43 //

    44 

    45 const char* GCTask::Kind::to_string(kind value) {

    46   const char* result = "unknown GCTask kind";

    47   switch (value) {

    48   default:

    49     result = "unknown GCTask kind";

    50     break;

    51   case unknown_task:

    52     result = "unknown task";

    53     break;

    54   case ordinary_task:

    55     result = "ordinary task";

    56     break;

    57   case wait_for_barrier_task:

    58     result = "wait for barrier task";

    59     break;

    60   case noop_task:

    61     result = "noop task";

    62     break;

    63   case idle_task:

    64     result = "idle task";

    65     break;

    66   }

    67   return result;

    68 };

    69 

    70 GCTask::GCTask() {

    71   initialize(Kind::ordinary_task, GCId::current());

    72 }

    73 

    74 GCTask::GCTask(Kind::kind kind) {

    75   initialize(kind, GCId::current());

    76 }

    77 

    78 GCTask::GCTask(Kind::kind kind, uint gc_id) {

    79   initialize(kind, gc_id);

    80 }

    81 

    82 void GCTask::initialize(Kind::kind kind, uint gc_id) {

    83   _kind = kind;

    84   _affinity = GCTaskManager::sentinel_worker();

    85   _older = NULL;

    86   _newer = NULL;

    87   _gc_id = gc_id;

    88 }

    89 

    90 void GCTask::destruct() {

    91   assert(older() == NULL, "shouldn't have an older task");

    92   assert(newer() == NULL, "shouldn't have a newer task");

    93   // Nothing to do.

    94 }

    95 

    96 NOT_PRODUCT(

    97 void GCTask::print(const char* message) const {

    98   tty->print(INTPTR_FORMAT " <- " INTPTR_FORMAT "(%u) -> " INTPTR_FORMAT,

    99              p2i(newer()), p2i(this), affinity(), p2i(older()));

   100 }

   101 )

   102 

   103 //

   104 // GCTaskQueue

   105 //

   106 

   107 GCTaskQueue* GCTaskQueue::create() {

   108   GCTaskQueue* result = new GCTaskQueue(false);

   109   if (TraceGCTaskQueue) {

   110     tty->print_cr("GCTaskQueue::create()"

   111                   " returns " INTPTR_FORMAT, p2i(result));

   112   }

   113   return result;

   114 }

   115 

   116 GCTaskQueue* GCTaskQueue::create_on_c_heap() {

   117   GCTaskQueue* result = new(ResourceObj::C_HEAP, mtGC) GCTaskQueue(true);

   118   if (TraceGCTaskQueue) {

   119     tty->print_cr("GCTaskQueue::create_on_c_heap()"

   120                   " returns " INTPTR_FORMAT,

   121                   p2i(result));

   122   }

   123   return result;

   124 }

   125 

   126 GCTaskQueue::GCTaskQueue(bool on_c_heap) :

   127   _is_c_heap_obj(on_c_heap) {

   128   initialize();

   129   if (TraceGCTaskQueue) {

   130     tty->print_cr("[" INTPTR_FORMAT "]"

   131                   " GCTaskQueue::GCTaskQueue() constructor",

   132                   p2i(this));

   133   }

   134 }

   135 

   136 void GCTaskQueue::destruct() {

   137   // Nothing to do.

   138 }

   139 

   140 void GCTaskQueue::destroy(GCTaskQueue* that) {

   141   if (TraceGCTaskQueue) {

   142     tty->print_cr("[" INTPTR_FORMAT "]"

   143                   " GCTaskQueue::destroy()"

   144                   "  is_c_heap_obj:  %s",

   145                   p2i(that),

   146                   that->is_c_heap_obj() ? "true" : "false");

   147   }

   148   // That instance may have been allocated as a CHeapObj,

   149   // in which case we have to free it explicitly.

   150   if (that != NULL) {

   151     that->destruct();

   152     assert(that->is_empty(), "should be empty");

   153     if (that->is_c_heap_obj()) {

   154       FreeHeap(that);

   155     }

   156   }

   157 }

   158 

   159 void GCTaskQueue::initialize() {

   160   set_insert_end(NULL);

   161   set_remove_end(NULL);

   162   set_length(0);

   163 }

   164 

   165 // Enqueue one task.

   166 void GCTaskQueue::enqueue(GCTask* task) {

   167   if (TraceGCTaskQueue) {

   168     tty->print_cr("[" INTPTR_FORMAT "]"

   169                   " GCTaskQueue::enqueue(task: "

   170                   INTPTR_FORMAT ")",

   171                   p2i(this), p2i(task));

   172     print("before:");

   173   }

   174   assert(task != NULL, "shouldn't have null task");

   175   assert(task->older() == NULL, "shouldn't be on queue");

   176   assert(task->newer() == NULL, "shouldn't be on queue");

   177   task->set_newer(NULL);

   178   task->set_older(insert_end());

   179   if (is_empty()) {

   180     set_remove_end(task);

   181   } else {

   182     insert_end()->set_newer(task);

   183   }

   184   set_insert_end(task);

   185   increment_length();

   186   verify_length();

   187   if (TraceGCTaskQueue) {

   188     print("after:");

   189   }

   190 }

   191 

   192 // Enqueue a whole list of tasks.  Empties the argument list.

   193 void GCTaskQueue::enqueue(GCTaskQueue* list) {

   194   if (TraceGCTaskQueue) {

   195     tty->print_cr("[" INTPTR_FORMAT "]"

   196                   " GCTaskQueue::enqueue(list: "

   197                   INTPTR_FORMAT ")",

   198                   p2i(this), p2i(list));

   199     print("before:");

   200     list->print("list:");

   201   }

   202   if (list->is_empty()) {

   203     // Enqueueing the empty list: nothing to do.

   204     return;

   205   }

   206   uint list_length = list->length();

   207   if (is_empty()) {

   208     // Enqueueing to empty list: just acquire elements.

   209     set_insert_end(list->insert_end());

   210     set_remove_end(list->remove_end());

   211     set_length(list_length);

   212   } else {

   213     // Prepend argument list to our queue.

   214     list->remove_end()->set_older(insert_end());

   215     insert_end()->set_newer(list->remove_end());

   216     set_insert_end(list->insert_end());

   217     set_length(length() + list_length);

   218     // empty the argument list.

   219   }

   220   list->initialize();

   221   if (TraceGCTaskQueue) {

   222     print("after:");

   223     list->print("list:");

   224   }

   225   verify_length();

   226 }

   227 

   228 // Dequeue one task.

   229 GCTask* GCTaskQueue::dequeue() {

   230   if (TraceGCTaskQueue) {

   231     tty->print_cr("[" INTPTR_FORMAT "]"

   232                   " GCTaskQueue::dequeue()", p2i(this));

   233     print("before:");

   234   }

   235   assert(!is_empty(), "shouldn't dequeue from empty list");

   236   GCTask* result = remove();

   237   assert(result != NULL, "shouldn't have NULL task");

   238   if (TraceGCTaskQueue) {

   239     tty->print_cr("    return: " INTPTR_FORMAT, p2i(result));

   240     print("after:");

   241   }

   242   return result;

   243 }

   244 

   245 // Dequeue one task, preferring one with affinity.

   246 GCTask* GCTaskQueue::dequeue(uint affinity) {

   247   if (TraceGCTaskQueue) {

   248     tty->print_cr("[" INTPTR_FORMAT "]"

   249                   " GCTaskQueue::dequeue(%u)", p2i(this), affinity);

   250     print("before:");

   251   }

   252   assert(!is_empty(), "shouldn't dequeue from empty list");

   253   // Look down to the next barrier for a task with this affinity.

   254   GCTask* result = NULL;

   255   for (GCTask* element = remove_end();

   256        element != NULL;

   257        element = element->newer()) {

   258     if (element->is_barrier_task()) {

   259       // Don't consider barrier tasks, nor past them.

   260       result = NULL;

   261       break;

   262     }

   263     if (element->affinity() == affinity) {

   264       result = remove(element);

   265       break;

   266     }

   267   }

   268   // If we didn't find anything with affinity, just take the next task.

   269   if (result == NULL) {

   270     result = remove();

   271   }

   272   if (TraceGCTaskQueue) {

   273     tty->print_cr("    return: " INTPTR_FORMAT, p2i(result));

   274     print("after:");

   275   }

   276   return result;

   277 }

   278 

   279 GCTask* GCTaskQueue::remove() {

   280   // Dequeue from remove end.

   281   GCTask* result = remove_end();

   282   assert(result != NULL, "shouldn't have null task");

   283   assert(result->older() == NULL, "not the remove_end");

   284   set_remove_end(result->newer());

   285   if (remove_end() == NULL) {

   286     assert(insert_end() == result, "not a singleton");

   287     set_insert_end(NULL);

   288   } else {

   289     remove_end()->set_older(NULL);

   290   }

   291   result->set_newer(NULL);

   292   decrement_length();

   293   assert(result->newer() == NULL, "shouldn't be on queue");

   294   assert(result->older() == NULL, "shouldn't be on queue");

   295   verify_length();

   296   return result;

   297 }

   298 

   299 GCTask* GCTaskQueue::remove(GCTask* task) {

   300   // This is slightly more work, and has slightly fewer asserts

   301   // than removing from the remove end.

   302   assert(task != NULL, "shouldn't have null task");

   303   GCTask* result = task;

   304   if (result->newer() != NULL) {

   305     result->newer()->set_older(result->older());

   306   } else {

   307     assert(insert_end() == result, "not youngest");

   308     set_insert_end(result->older());

   309   }

   310   if (result->older() != NULL) {

   311     result->older()->set_newer(result->newer());

   312   } else {

   313     assert(remove_end() == result, "not oldest");

   314     set_remove_end(result->newer());

   315   }

   316   result->set_newer(NULL);

   317   result->set_older(NULL);

   318   decrement_length();

   319   verify_length();

   320   return result;

   321 }

   322 

   323 NOT_PRODUCT(

   324 // Count the elements in the queue and verify the length against

   325 // that count.

   326 void GCTaskQueue::verify_length() const {

   327   uint count = 0;

   328   for (GCTask* element = insert_end();

   329        element != NULL;

   330        element = element->older()) {

   331 

   332     count++;

   333   }

   334   assert(count == length(), "Length does not match queue");

   335 }

   336 

   337 void GCTaskQueue::print(const char* message) const {

   338   tty->print_cr("[" INTPTR_FORMAT "] GCTaskQueue:"

   339                 "  insert_end: " INTPTR_FORMAT

   340                 "  remove_end: " INTPTR_FORMAT

   341                 "  length:       %d"

   342                 "  %s",

   343                 p2i(this), p2i(insert_end()), p2i(remove_end()), length(), message);

   344   uint count = 0;

   345   for (GCTask* element = insert_end();

   346        element != NULL;

   347        element = element->older()) {

   348     element->print("    ");

   349     count++;

   350     tty->cr();

   351   }

   352   tty->print("Total tasks: %d", count);

   353 }

   354 )

   355 

   356 //

   357 // SynchronizedGCTaskQueue

   358 //

   359 

   360 SynchronizedGCTaskQueue::SynchronizedGCTaskQueue(GCTaskQueue* queue_arg,

   361                                                  Monitor *       lock_arg) :

   362   _unsynchronized_queue(queue_arg),

   363   _lock(lock_arg) {

   364   assert(unsynchronized_queue() != NULL, "null queue");

   365   assert(lock() != NULL, "null lock");

   366 }

   367 

   368 SynchronizedGCTaskQueue::~SynchronizedGCTaskQueue() {

   369   // Nothing to do.

   370 }

   371 

   372 //

   373 // GCTaskManager

   374 //

   375 GCTaskManager::GCTaskManager(uint workers) :

   376   _workers(workers),

   377   _created_workers(0),

   378   _active_workers(0),

   379   _idle_workers(0) {

   380   initialize();

   381 }

   382 

   383 GCTaskThread* GCTaskManager::install_worker(uint t) {

   384   GCTaskThread* new_worker = GCTaskThread::create(this, t, _processor_assignment[t]);

   385   set_thread(t, new_worker);

   386   return new_worker;

   387 }

   388 

   389 void GCTaskManager::add_workers(bool initializing) {

   390   os::ThreadType worker_type = os::pgc_thread;

   391   uint previous_created_workers = _created_workers;

   392 

   393   _created_workers = WorkerManager::add_workers(this,

   394                                                 _active_workers,

   395                                                 _workers,

   396                                                 _created_workers,

   397                                                 worker_type,

   398                                                 initializing);

   399   _active_workers = MIN2(_created_workers, _active_workers);

   400 

   401   WorkerManager::log_worker_creation(this, previous_created_workers, _active_workers, _created_workers, initializing);

   402 }

   403 

   404 const char* GCTaskManager::group_name() {

   405   return "ParGC Thread";

   406 }

   407 

   408 void GCTaskManager::initialize() {

   409   if (TraceGCTaskManager) {

   410     tty->print_cr("GCTaskManager::initialize: workers: %u", workers());

   411   }

   412   assert(workers() != 0, "no workers");

   413   _monitor = new Monitor(Mutex::barrier,                // rank

   414                          "GCTaskManager monitor",       // name

   415                          Mutex::_allow_vm_block_flag,   // allow_vm_block

   416                          Monitor::_safepoint_check_never);

   417   // The queue for the GCTaskManager must be a CHeapObj.

   418   GCTaskQueue* unsynchronized_queue = GCTaskQueue::create_on_c_heap();

   419   _queue = SynchronizedGCTaskQueue::create(unsynchronized_queue, lock());

   420   _noop_task = NoopGCTask::create_on_c_heap();

   421   _resource_flag = NEW_C_HEAP_ARRAY(bool, workers(), mtGC);

   422   {

   423     // Set up worker threads.

   424     //     Distribute the workers among the available processors,

   425     //     unless we were told not to, or if the os doesn't want to.

   426     _processor_assignment = NEW_C_HEAP_ARRAY(uint, workers(), mtGC);

   427     if (!BindGCTaskThreadsToCPUs ||

   428         !os::distribute_processes(workers(), _processor_assignment)) {

   429       for (uint a = 0; a < workers(); a += 1) {

   430         _processor_assignment[a] = sentinel_worker();

   431       }

   432     }

   433 

   434     _thread = NEW_C_HEAP_ARRAY(GCTaskThread*, workers(), mtGC);

   435     _active_workers = ParallelGCThreads;

   436     if (UseDynamicNumberOfGCThreads && !FLAG_IS_CMDLINE(ParallelGCThreads)) {

   437       _active_workers = 1U;

   438     }

   439 

   440     Log(gc, task, thread) log;

   441     if (log.is_trace()) {

   442       LogStream ls(log.trace());

   443       ls.print("GCTaskManager::initialize: distribution:");

   444       for (uint t = 0; t < workers(); t += 1) {

   445         ls.print("  %u", _processor_assignment[t]);

   446       }

   447       ls.cr();

   448     }

   449   }

   450   reset_busy_workers();

   451   set_unblocked();

   452   for (uint w = 0; w < workers(); w += 1) {

   453     set_resource_flag(w, false);

   454   }

   455   reset_delivered_tasks();

   456   reset_completed_tasks();

   457   reset_barriers();

   458   reset_emptied_queue();

   459 

   460   add_workers(true);

   461 }

   462 

   463 GCTaskManager::~GCTaskManager() {

   464   assert(busy_workers() == 0, "still have busy workers");

   465   assert(queue()->is_empty(), "still have queued work");

   466   NoopGCTask::destroy(_noop_task);

   467   _noop_task = NULL;

   468   if (_thread != NULL) {

   469     for (uint i = 0; i < created_workers(); i += 1) {

   470       GCTaskThread::destroy(thread(i));

   471       set_thread(i, NULL);

   472     }

   473     FREE_C_HEAP_ARRAY(GCTaskThread*, _thread);

   474     _thread = NULL;

   475   }

   476   if (_processor_assignment != NULL) {

   477     FREE_C_HEAP_ARRAY(uint, _processor_assignment);

   478     _processor_assignment = NULL;

   479   }

   480   if (_resource_flag != NULL) {

   481     FREE_C_HEAP_ARRAY(bool, _resource_flag);

   482     _resource_flag = NULL;

   483   }

   484   if (queue() != NULL) {

   485     GCTaskQueue* unsynchronized_queue = queue()->unsynchronized_queue();

   486     GCTaskQueue::destroy(unsynchronized_queue);

   487     SynchronizedGCTaskQueue::destroy(queue());

   488     _queue = NULL;

   489   }

   490   if (monitor() != NULL) {

   491     delete monitor();

   492     _monitor = NULL;

   493   }

   494 }

   495 

   496 void GCTaskManager::set_active_gang() {

   497   _active_workers =

   498     WorkerPolicy::calc_active_workers(workers(),

   499                                       active_workers(),

   500                                       Threads::number_of_non_daemon_threads());

   501 

   502   assert(!all_workers_active() || active_workers() == ParallelGCThreads,

   503          "all_workers_active() is  incorrect: "

   504          "active %d  ParallelGCThreads %u", active_workers(),

   505          ParallelGCThreads);

   506   _active_workers = MIN2(_active_workers, _workers);

   507   // "add_workers" does not guarantee any additional workers

   508   add_workers(false);

   509   log_trace(gc, task)("GCTaskManager::set_active_gang(): "

   510                       "all_workers_active()  %d  workers %d  "

   511                       "active  %d  ParallelGCThreads %u",

   512                       all_workers_active(), workers(),  active_workers(),

   513                       ParallelGCThreads);

   514 }

   515 

   516 // Create IdleGCTasks for inactive workers.

   517 // Creates tasks in a ResourceArea and assumes

   518 // an appropriate ResourceMark.

   519 void GCTaskManager::task_idle_workers() {

   520   {

   521     int more_inactive_workers = 0;

   522     {

   523       // Stop any idle tasks from exiting their IdleGCTask's

   524       // and get the count for additional IdleGCTask's under

   525       // the GCTaskManager's monitor so that the "more_inactive_workers"

   526       // count is correct.

   527       MutexLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   528       _wait_helper.set_should_wait(true);

   529       // active_workers are a number being requested.  idle_workers

   530       // are the number currently idle.  If all the workers are being

   531       // requested to be active but some are already idle, reduce

   532       // the number of active_workers to be consistent with the

   533       // number of idle_workers.  The idle_workers are stuck in

   534       // idle tasks and will no longer be release (since a new GC

   535       // is starting).  Try later to release enough idle_workers

   536       // to allow the desired number of active_workers.

   537       more_inactive_workers =

   538         created_workers() - active_workers() - idle_workers();

   539       if (more_inactive_workers < 0) {

   540         int reduced_active_workers = active_workers() + more_inactive_workers;

   541         update_active_workers(reduced_active_workers);

   542         more_inactive_workers = 0;

   543       }

   544       log_trace(gc, task)("JT: %d  workers %d  active  %d  idle %d  more %d",

   545                           Threads::number_of_non_daemon_threads(),

   546                           created_workers(),

   547                           active_workers(),

   548                           idle_workers(),

   549                           more_inactive_workers);

   550     }

   551     GCTaskQueue* q = GCTaskQueue::create();

   552     for(uint i = 0; i < (uint) more_inactive_workers; i++) {

   553       q->enqueue(IdleGCTask::create_on_c_heap());

   554       increment_idle_workers();

   555     }

   556     assert(created_workers() == active_workers() + idle_workers(),

   557       "total workers should equal active + inactive");

   558     add_list(q);

   559     // GCTaskQueue* q was created in a ResourceArea so a

   560     // destroy() call is not needed.

   561   }

   562 }

   563 

   564 void  GCTaskManager::release_idle_workers() {

   565   {

   566     MutexLocker ml(monitor(),

   567       Mutex::_no_safepoint_check_flag);

   568     _wait_helper.set_should_wait(false);

   569     monitor()->notify_all();

   570   // Release monitor

   571   }

   572 }

   573 

   574 void GCTaskManager::print_task_time_stamps() {

   575   if (!log_is_enabled(Debug, gc, task, time)) {

   576     return;

   577   }

   578   uint num_thr = created_workers();

   579   for(uint i=0; i < num_thr; i++) {

   580     GCTaskThread* t = thread(i);

   581     t->print_task_time_stamps();

   582   }

   583 }

   584 

   585 void GCTaskManager::print_threads_on(outputStream* st) {

   586   uint num_thr = created_workers();

   587   for (uint i = 0; i < num_thr; i++) {

   588     thread(i)->print_on(st);

   589     st->cr();

   590   }

   591 }

   592 

   593 void GCTaskManager::threads_do(ThreadClosure* tc) {

   594   assert(tc != NULL, "Null ThreadClosure");

   595   uint num_thr = created_workers();

   596   for (uint i = 0; i < num_thr; i++) {

   597     tc->do_thread(thread(i));

   598   }

   599 }

   600 

   601 GCTaskThread* GCTaskManager::thread(uint which) {

   602   assert(which < created_workers(), "index out of bounds");

   603   assert(_thread[which] != NULL, "shouldn't have null thread");

   604   return _thread[which];

   605 }

   606 

   607 void GCTaskManager::set_thread(uint which, GCTaskThread* value) {

   608   // "_created_workers" may not have been updated yet so use workers()

   609   assert(which < workers(), "index out of bounds");

   610   assert(value != NULL, "shouldn't have null thread");

   611   _thread[which] = value;

   612 }

   613 

   614 void GCTaskManager::add_task(GCTask* task) {

   615   assert(task != NULL, "shouldn't have null task");

   616   MutexLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   617   if (TraceGCTaskManager) {

   618     tty->print_cr("GCTaskManager::add_task(" INTPTR_FORMAT " [%s])",

   619                   p2i(task), GCTask::Kind::to_string(task->kind()));

   620   }

   621   queue()->enqueue(task);

   622   // Notify with the lock held to avoid missed notifies.

   623   if (TraceGCTaskManager) {

   624     tty->print_cr("    GCTaskManager::add_task (%s)->notify_all",

   625                   monitor()->name());

   626   }

   627   (void) monitor()->notify_all();

   628   // Release monitor().

   629 }

   630 

   631 void GCTaskManager::add_list(GCTaskQueue* list) {

   632   assert(list != NULL, "shouldn't have null task");

   633   MutexLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   634   if (TraceGCTaskManager) {

   635     tty->print_cr("GCTaskManager::add_list(%u)", list->length());

   636   }

   637   queue()->enqueue(list);

   638   // Notify with the lock held to avoid missed notifies.

   639   if (TraceGCTaskManager) {

   640     tty->print_cr("    GCTaskManager::add_list (%s)->notify_all",

   641                   monitor()->name());

   642   }

   643   (void) monitor()->notify_all();

   644   // Release monitor().

   645 }

   646 

   647 // GC workers wait in get_task() for new work to be added

   648 // to the GCTaskManager's queue.  When new work is added,

   649 // a notify is sent to the waiting GC workers which then

   650 // compete to get tasks.  If a GC worker wakes up and there

   651 // is no work on the queue, it is given a noop_task to execute

   652 // and then loops to find more work.

   653 

   654 GCTask* GCTaskManager::get_task(uint which) {

   655   GCTask* result = NULL;

   656   // Grab the queue lock.

   657   MonitorLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   658   // Wait while the queue is block or

   659   // there is nothing to do, except maybe release resources.

   660   while (is_blocked() ||

   661          (queue()->is_empty() && !should_release_resources(which))) {

   662     if (TraceGCTaskManager) {

   663       tty->print_cr("GCTaskManager::get_task(%u)"

   664                     "  blocked: %s"

   665                     "  empty: %s"

   666                     "  release: %s",

   667                     which,

   668                     is_blocked() ? "true" : "false",

   669                     queue()->is_empty() ? "true" : "false",

   670                     should_release_resources(which) ? "true" : "false");

   671       tty->print_cr("    => (%s)->wait()",

   672                     monitor()->name());

   673     }

   674     ml.wait(0);

   675   }

   676   // We've reacquired the queue lock here.

   677   // Figure out which condition caused us to exit the loop above.

   678   if (!queue()->is_empty()) {

   679     if (UseGCTaskAffinity) {

   680       result = queue()->dequeue(which);

   681     } else {

   682       result = queue()->dequeue();

   683     }

   684     if (result->is_barrier_task()) {

   685       assert(which != sentinel_worker(),

   686              "blocker shouldn't be bogus");

   687       set_blocking_worker(which);

   688     }

   689   } else {

   690     // The queue is empty, but we were woken up.

   691     // Just hand back a Noop task,

   692     // in case someone wanted us to release resources, or whatever.

   693     result = noop_task();

   694   }

   695   assert(result != NULL, "shouldn't have null task");

   696   if (TraceGCTaskManager) {

   697     tty->print_cr("GCTaskManager::get_task(%u) => " INTPTR_FORMAT " [%s]",

   698                   which, p2i(result), GCTask::Kind::to_string(result->kind()));

   699     tty->print_cr("     %s", result->name());

   700   }

   701   if (!result->is_idle_task()) {

   702     increment_busy_workers();

   703     increment_delivered_tasks();

   704   }

   705   return result;

   706   // Release monitor().

   707 }

   708 

   709 void GCTaskManager::note_completion(uint which) {

   710   MutexLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   711   if (TraceGCTaskManager) {

   712     tty->print_cr("GCTaskManager::note_completion(%u)", which);

   713   }

   714   // If we are blocked, check if the completing thread is the blocker.

   715   if (blocking_worker() == which) {

   716     assert(blocking_worker() != sentinel_worker(),

   717            "blocker shouldn't be bogus");

   718     increment_barriers();

   719     set_unblocked();

   720   }

   721   increment_completed_tasks();

   722   uint active = decrement_busy_workers();

   723   if ((active == 0) && (queue()->is_empty())) {

   724     increment_emptied_queue();

   725     if (TraceGCTaskManager) {

   726       tty->print_cr("    GCTaskManager::note_completion(%u) done", which);

   727     }

   728   }

   729   if (TraceGCTaskManager) {

   730     tty->print_cr("    GCTaskManager::note_completion(%u) (%s)->notify_all",

   731                   which, monitor()->name());

   732     tty->print_cr("  "

   733                   "  blocked: %s"

   734                   "  empty: %s"

   735                   "  release: %s",

   736                   is_blocked() ? "true" : "false",

   737                   queue()->is_empty() ? "true" : "false",

   738                   should_release_resources(which) ? "true" : "false");

   739     tty->print_cr("  "

   740                   "  delivered: %u"

   741                   "  completed: %u"

   742                   "  barriers: %u"

   743                   "  emptied: %u",

   744                   delivered_tasks(),

   745                   completed_tasks(),

   746                   barriers(),

   747                   emptied_queue());

   748   }

   749   // Tell everyone that a task has completed.

   750   (void) monitor()->notify_all();

   751   // Release monitor().

   752 }

   753 

   754 uint GCTaskManager::increment_busy_workers() {

   755   assert(queue()->own_lock(), "don't own the lock");

   756   _busy_workers += 1;

   757   return _busy_workers;

   758 }

   759 

   760 uint GCTaskManager::decrement_busy_workers() {

   761   assert(queue()->own_lock(), "don't own the lock");

   762   assert(_busy_workers > 0, "About to make a mistake");

   763   _busy_workers -= 1;

   764   return _busy_workers;

   765 }

   766 

   767 void GCTaskManager::release_all_resources() {

   768   // If you want this to be done atomically, do it in a WaitForBarrierGCTask.

   769   for (uint i = 0; i < created_workers(); i += 1) {

   770     set_resource_flag(i, true);

   771   }

   772 }

   773 

   774 bool GCTaskManager::should_release_resources(uint which) {

   775   // This can be done without a lock because each thread reads one element.

   776   return resource_flag(which);

   777 }

   778 

   779 void GCTaskManager::note_release(uint which) {

   780   // This can be done without a lock because each thread writes one element.

   781   set_resource_flag(which, false);

   782 }

   783 

   784 // "list" contains tasks that are ready to execute.  Those

   785 // tasks are added to the GCTaskManager's queue of tasks and

   786 // then the GC workers are notified that there is new work to

   787 // do.

   788 //

   789 // Typically different types of tasks can be added to the "list".

   790 // For example in PSScavenge OldToYoungRootsTask, SerialOldToYoungRootsTask,

   791 // ScavengeRootsTask, and StealTask tasks are all added to the list

   792 // and then the GC workers are notified of new work.  The tasks are

   793 // handed out in the order in which they are added to the list

   794 // (although execution is not necessarily in that order).  As long

   795 // as any tasks are running the GCTaskManager will wait for execution

   796 // to complete.  GC workers that execute a stealing task remain in

   797 // the stealing task until all stealing tasks have completed.  The load

   798 // balancing afforded by the stealing tasks work best if the stealing

   799 // tasks are added last to the list.

   800 

   801 void GCTaskManager::execute_and_wait(GCTaskQueue* list) {

   802   WaitForBarrierGCTask* fin = WaitForBarrierGCTask::create();

   803   list->enqueue(fin);

   804   // The barrier task will be read by one of the GC

   805   // workers once it is added to the list of tasks.

   806   // Be sure that is globally visible before the

   807   // GC worker reads it (which is after the task is added

   808   // to the list of tasks below).

   809   OrderAccess::storestore();

   810   add_list(list);

   811   fin->wait_for(true /* reset */);

   812   // We have to release the barrier tasks!

   813   WaitForBarrierGCTask::destroy(fin);

   814 }

   815 

   816 bool GCTaskManager::resource_flag(uint which) {

   817   assert(which < workers(), "index out of bounds");

   818   return _resource_flag[which];

   819 }

   820 

   821 void GCTaskManager::set_resource_flag(uint which, bool value) {

   822   assert(which < workers(), "index out of bounds");

   823   _resource_flag[which] = value;

   824 }

   825 

   826 //

   827 // NoopGCTask

   828 //

   829 

   830 NoopGCTask* NoopGCTask::create_on_c_heap() {

   831   NoopGCTask* result = new(ResourceObj::C_HEAP, mtGC) NoopGCTask();

   832   return result;

   833 }

   834 

   835 void NoopGCTask::destroy(NoopGCTask* that) {

   836   if (that != NULL) {

   837     that->destruct();

   838     FreeHeap(that);

   839   }

   840 }

   841 

   842 // This task should never be performing GC work that require

   843 // a valid GC id.

   844 NoopGCTask::NoopGCTask() : GCTask(GCTask::Kind::noop_task, GCId::undefined()) { }

   845 

   846 void NoopGCTask::destruct() {

   847   // This has to know it's superclass structure, just like the constructor.

   848   this->GCTask::destruct();

   849   // Nothing else to do.

   850 }

   851 

   852 //

   853 // IdleGCTask

   854 //

   855 

   856 IdleGCTask* IdleGCTask::create() {

   857   IdleGCTask* result = new IdleGCTask(false);

   858   assert(UseDynamicNumberOfGCThreads,

   859     "Should only be used with dynamic GC thread");

   860   return result;

   861 }

   862 

   863 IdleGCTask* IdleGCTask::create_on_c_heap() {

   864   IdleGCTask* result = new(ResourceObj::C_HEAP, mtGC) IdleGCTask(true);

   865   assert(UseDynamicNumberOfGCThreads,

   866     "Should only be used with dynamic GC thread");

   867   return result;

   868 }

   869 

   870 void IdleGCTask::do_it(GCTaskManager* manager, uint which) {

   871   WaitHelper* wait_helper = manager->wait_helper();

   872   log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask:::do_it() should_wait: %s",

   873       p2i(this), wait_helper->should_wait() ? "true" : "false");

   874 

   875   MonitorLocker ml(manager->monitor(), Mutex::_no_safepoint_check_flag);

   876   log_trace(gc, task)("--- idle %d", which);

   877   // Increment has to be done when the idle tasks are created.

   878   // manager->increment_idle_workers();

   879   ml.notify_all();

   880   while (wait_helper->should_wait()) {

   881     log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask::do_it()  [" INTPTR_FORMAT "] (%s)->wait()",

   882       p2i(this), p2i(manager->monitor()), manager->monitor()->name());

   883     ml.wait(0);

   884   }

   885   manager->decrement_idle_workers();

   886 

   887   log_trace(gc, task)("--- release %d", which);

   888   log_trace(gc, task)("[" INTPTR_FORMAT "] IdleGCTask::do_it() returns should_wait: %s",

   889     p2i(this), wait_helper->should_wait() ? "true" : "false");

   890   // Release monitor().

   891 }

   892 

   893 void IdleGCTask::destroy(IdleGCTask* that) {

   894   if (that != NULL) {

   895     that->destruct();

   896     if (that->is_c_heap_obj()) {

   897       FreeHeap(that);

   898     }

   899   }

   900 }

   901 

   902 void IdleGCTask::destruct() {

   903   // This has to know it's superclass structure, just like the constructor.

   904   this->GCTask::destruct();

   905   // Nothing else to do.

   906 }

   907 

   908 //

   909 // WaitForBarrierGCTask

   910 //

   911 WaitForBarrierGCTask* WaitForBarrierGCTask::create() {

   912   WaitForBarrierGCTask* result = new WaitForBarrierGCTask();

   913   return result;

   914 }

   915 

   916 WaitForBarrierGCTask::WaitForBarrierGCTask() : GCTask(GCTask::Kind::wait_for_barrier_task) { }

   917 

   918 void WaitForBarrierGCTask::destroy(WaitForBarrierGCTask* that) {

   919   if (that != NULL) {

   920     if (TraceGCTaskManager) {

   921       tty->print_cr("[" INTPTR_FORMAT "] WaitForBarrierGCTask::destroy()", p2i(that));

   922     }

   923     that->destruct();

   924   }

   925 }

   926 

   927 void WaitForBarrierGCTask::destruct() {

   928   if (TraceGCTaskManager) {

   929     tty->print_cr("[" INTPTR_FORMAT "] WaitForBarrierGCTask::destruct()", p2i(this));

   930   }

   931   this->GCTask::destruct();

   932   // Clean up that should be in the destructor,

   933   // except that ResourceMarks don't call destructors.

   934   _wait_helper.release_monitor();

   935 }

   936 

   937 void WaitForBarrierGCTask::do_it_internal(GCTaskManager* manager, uint which) {

   938   // Wait for this to be the only busy worker.

   939   assert(manager->monitor()->owned_by_self(), "don't own the lock");

   940   assert(manager->is_blocked(), "manager isn't blocked");

   941   while (manager->busy_workers() > 1) {

   942     if (TraceGCTaskManager) {

   943       tty->print_cr("WaitForBarrierGCTask::do_it(%u) waiting on %u workers",

   944                     which, manager->busy_workers());

   945     }

   946     manager->monitor()->wait_without_safepoint_check(0);

   947   }

   948 }

   949 

   950 void WaitForBarrierGCTask::do_it(GCTaskManager* manager, uint which) {

   951   if (TraceGCTaskManager) {

   952     tty->print_cr("[" INTPTR_FORMAT "]"

   953                   " WaitForBarrierGCTask::do_it() waiting for idle",

   954                   p2i(this));

   955   }

   956   {

   957     // First, wait for the barrier to arrive.

   958     MutexLocker ml(manager->lock(), Mutex::_no_safepoint_check_flag);

   959     do_it_internal(manager, which);

   960     // Release manager->lock().

   961   }

   962   // Then notify the waiter.

   963   _wait_helper.notify();

   964 }

   965 

   966 WaitHelper::WaitHelper() : _monitor(MonitorSupply::reserve()), _should_wait(true) {

   967   if (TraceGCTaskManager) {

   968     tty->print_cr("[" INTPTR_FORMAT "]"

   969                   " WaitHelper::WaitHelper()"

   970                   "  monitor: " INTPTR_FORMAT,

   971                   p2i(this), p2i(monitor()));

   972   }

   973 }

   974 

   975 void WaitHelper::release_monitor() {

   976   assert(_monitor != NULL, "");

   977   MonitorSupply::release(_monitor);

   978   _monitor = NULL;

   979 }

   980 

   981 WaitHelper::~WaitHelper() {

   982   release_monitor();

   983 }

   984 

   985 void WaitHelper::wait_for(bool reset) {

   986   if (TraceGCTaskManager) {

   987     tty->print_cr("[" INTPTR_FORMAT "]"

   988                   " WaitForBarrierGCTask::wait_for()"

   989       "  should_wait: %s",

   990       p2i(this), should_wait() ? "true" : "false");

   991   }

   992   {

   993     // Grab the lock and check again.

   994     MonitorLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

   995     while (should_wait()) {

   996       if (TraceGCTaskManager) {

   997         tty->print_cr("[" INTPTR_FORMAT "]"

   998                       " WaitForBarrierGCTask::wait_for()"

   999           "  [" INTPTR_FORMAT "] (%s)->wait()",

  1000           p2i(this), p2i(monitor()), monitor()->name());

  1001       }

  1002       ml.wait(0);

  1003     }

  1004     // Reset the flag in case someone reuses this task.

  1005     if (reset) {

  1006       set_should_wait(true);

  1007     }

  1008     if (TraceGCTaskManager) {

  1009       tty->print_cr("[" INTPTR_FORMAT "]"

  1010                     " WaitForBarrierGCTask::wait_for() returns"

  1011         "  should_wait: %s",

  1012         p2i(this), should_wait() ? "true" : "false");

  1013     }

  1014     // Release monitor().

  1015   }

  1016 }

  1017 

  1018 void WaitHelper::notify() {

  1019   MutexLocker ml(monitor(), Mutex::_no_safepoint_check_flag);

  1020   set_should_wait(false);

  1021   // Waiter doesn't miss the notify in the wait_for method

  1022   // since it checks the flag after grabbing the monitor.

  1023   if (TraceGCTaskManager) {

  1024     tty->print_cr("[" INTPTR_FORMAT "]"

  1025                   " WaitForBarrierGCTask::do_it()"

  1026                   "  [" INTPTR_FORMAT "] (%s)->notify_all()",

  1027                   p2i(this), p2i(monitor()), monitor()->name());

  1028   }

  1029   monitor()->notify_all();

  1030 }

  1031 

  1032 Mutex*                   MonitorSupply::_lock     = NULL;

  1033 GrowableArray<Monitor*>* MonitorSupply::_freelist = NULL;

  1034 

  1035 Monitor* MonitorSupply::reserve() {

  1036   Monitor* result = NULL;

  1037   // Lazy initialization: possible race.

  1038   if (lock() == NULL) {

  1039     _lock = new Mutex(Mutex::barrier,                  // rank

  1040                       "MonitorSupply mutex",           // name

  1041                       Mutex::_allow_vm_block_flag);    // allow_vm_block

  1042   }

  1043   {

  1044     MutexLocker ml(lock());

  1045     // Lazy initialization.

  1046     if (freelist() == NULL) {

  1047       _freelist =

  1048         new(ResourceObj::C_HEAP, mtGC) GrowableArray<Monitor*>(ParallelGCThreads,

  1049                                                          true);

  1050     }

  1051     if (! freelist()->is_empty()) {

  1052       result = freelist()->pop();

  1053     } else {

  1054       result = new Monitor(Mutex::barrier,                  // rank

  1055                            "MonitorSupply monitor",         // name

  1056                            Mutex::_allow_vm_block_flag,     // allow_vm_block

  1057                            Monitor::_safepoint_check_never);

  1058     }

  1059     guarantee(result != NULL, "shouldn't return NULL");

  1060     assert(!result->is_locked(), "shouldn't be locked");

  1061     // release lock().

  1062   }

  1063   return result;

  1064 }

  1065 

  1066 void MonitorSupply::release(Monitor* instance) {

  1067   assert(instance != NULL, "shouldn't release NULL");

  1068   assert(!instance->is_locked(), "shouldn't be locked");

  1069   {

  1070     MutexLocker ml(lock());

  1071     freelist()->push(instance);

  1072     // release lock().

  1073   }

  1074 }