src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp
changeset 57777 90ead0febf56
parent 57637 c38cca5ffb66
child 57782 ca133d5ea78a
equal deleted inserted replaced
57774:21dccfac0ec5 57777:90ead0febf56
  1008 // size of an object that may not yet have been initialized.
  1008 // size of an object that may not yet have been initialized.
  1009 
  1009 
  1010 // Things to support parallel young-gen collection.
  1010 // Things to support parallel young-gen collection.
  1011 oop
  1011 oop
  1012 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
  1012 ConcurrentMarkSweepGeneration::par_promote(int thread_num,
  1013                                            oop old, markOop m,
  1013                                            oop old, markWord m,
  1014                                            size_t word_sz) {
  1014                                            size_t word_sz) {
  1015 #ifndef PRODUCT
  1015 #ifndef PRODUCT
  1016   if (CMSHeap::heap()->promotion_should_fail()) {
  1016   if (CMSHeap::heap()->promotion_should_fail()) {
  1017     return NULL;
  1017     return NULL;
  1018   }
  1018   }
  7774 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
  7774 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) {
  7775   assert(stack->isEmpty(), "Expected precondition");
  7775   assert(stack->isEmpty(), "Expected precondition");
  7776   assert(stack->capacity() > num, "Shouldn't bite more than can chew");
  7776   assert(stack->capacity() > num, "Shouldn't bite more than can chew");
  7777   size_t i = num;
  7777   size_t i = num;
  7778   oop  cur = _overflow_list;
  7778   oop  cur = _overflow_list;
  7779   const markOop proto = markOopDesc::prototype();
  7779   const markWord proto = markWord::prototype();
  7780   NOT_PRODUCT(ssize_t n = 0;)
  7780   NOT_PRODUCT(ssize_t n = 0;)
  7781   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
  7781   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
  7782     next = oop(cur->mark_raw());
  7782     next = oop(cur->mark_raw().to_pointer());
  7783     cur->set_mark_raw(proto);   // until proven otherwise
  7783     cur->set_mark_raw(proto);   // until proven otherwise
  7784     assert(oopDesc::is_oop(cur), "Should be an oop");
  7784     assert(oopDesc::is_oop(cur), "Should be an oop");
  7785     bool res = stack->push(cur);
  7785     bool res = stack->push(cur);
  7786     assert(res, "Bit off more than can chew?");
  7786     assert(res, "Bit off more than can chew?");
  7787     NOT_PRODUCT(n++;)
  7787     NOT_PRODUCT(n++;)
  7861   }
  7861   }
  7862   assert(prefix != NULL && prefix != BUSY, "Error");
  7862   assert(prefix != NULL && prefix != BUSY, "Error");
  7863   size_t i = num;
  7863   size_t i = num;
  7864   oop cur = prefix;
  7864   oop cur = prefix;
  7865   // Walk down the first "num" objects, unless we reach the end.
  7865   // Walk down the first "num" objects, unless we reach the end.
  7866   for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--);
  7866   for (; i > 1 && cur->mark_raw().to_pointer() != NULL; cur = oop(cur->mark_raw().to_pointer()), i--);
  7867   if (cur->mark_raw() == NULL) {
  7867   if (cur->mark_raw().to_pointer() == NULL) {
  7868     // We have "num" or fewer elements in the list, so there
  7868     // We have "num" or fewer elements in the list, so there
  7869     // is nothing to return to the global list.
  7869     // is nothing to return to the global list.
  7870     // Write back the NULL in lieu of the BUSY we wrote
  7870     // Write back the NULL in lieu of the BUSY we wrote
  7871     // above, if it is still the same value.
  7871     // above, if it is still the same value.
  7872     if (_overflow_list == BUSY) {
  7872     if (_overflow_list == BUSY) {
  7873       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
  7873       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
  7874     }
  7874     }
  7875   } else {
  7875   } else {
  7876     // Chop off the suffix and return it to the global list.
  7876     // Chop off the suffix and return it to the global list.
  7877     assert(cur->mark_raw() != BUSY, "Error");
  7877     assert(cur->mark_raw().to_pointer() != (void*)BUSY, "Error");
  7878     oop suffix_head = cur->mark_raw(); // suffix will be put back on global list
  7878     oop suffix_head = oop(cur->mark_raw().to_pointer()); // suffix will be put back on global list
  7879     cur->set_mark_raw(NULL);           // break off suffix
  7879     cur->set_mark_raw(markWord::from_pointer(NULL));     // break off suffix
  7880     // It's possible that the list is still in the empty(busy) state
  7880     // It's possible that the list is still in the empty(busy) state
  7881     // we left it in a short while ago; in that case we may be
  7881     // we left it in a short while ago; in that case we may be
  7882     // able to place back the suffix without incurring the cost
  7882     // able to place back the suffix without incurring the cost
  7883     // of a walk down the list.
  7883     // of a walk down the list.
  7884     oop observed_overflow_list = _overflow_list;
  7884     oop observed_overflow_list = _overflow_list;
  7894     }
  7894     }
  7895     if (!attached) {
  7895     if (!attached) {
  7896       // Too bad, someone else sneaked in (at least) an element; we'll need
  7896       // Too bad, someone else sneaked in (at least) an element; we'll need
  7897       // to do a splice. Find tail of suffix so we can prepend suffix to global
  7897       // to do a splice. Find tail of suffix so we can prepend suffix to global
  7898       // list.
  7898       // list.
  7899       for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw()));
  7899       for (cur = suffix_head; cur->mark_raw().to_pointer() != NULL; cur = (oop)(cur->mark_raw().to_pointer()));
  7900       oop suffix_tail = cur;
  7900       oop suffix_tail = cur;
  7901       assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL,
  7901       assert(suffix_tail != NULL && suffix_tail->mark_raw().to_pointer() == NULL,
  7902              "Tautology");
  7902              "Tautology");
  7903       observed_overflow_list = _overflow_list;
  7903       observed_overflow_list = _overflow_list;
  7904       do {
  7904       do {
  7905         cur_overflow_list = observed_overflow_list;
  7905         cur_overflow_list = observed_overflow_list;
  7906         if (cur_overflow_list != BUSY) {
  7906         if (cur_overflow_list != BUSY) {
  7907           // Do the splice ...
  7907           // Do the splice ...
  7908           suffix_tail->set_mark_raw(markOop(cur_overflow_list));
  7908           suffix_tail->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list));
  7909         } else { // cur_overflow_list == BUSY
  7909         } else { // cur_overflow_list == BUSY
  7910           suffix_tail->set_mark_raw(NULL);
  7910           suffix_tail->set_mark_raw(markWord::from_pointer(NULL));
  7911         }
  7911         }
  7912         // ... and try to place spliced list back on overflow_list ...
  7912         // ... and try to place spliced list back on overflow_list ...
  7913         observed_overflow_list =
  7913         observed_overflow_list =
  7914           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
  7914           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
  7915       } while (cur_overflow_list != observed_overflow_list);
  7915       } while (cur_overflow_list != observed_overflow_list);
  7917     }
  7917     }
  7918   }
  7918   }
  7919 
  7919 
  7920   // Push the prefix elements on work_q
  7920   // Push the prefix elements on work_q
  7921   assert(prefix != NULL, "control point invariant");
  7921   assert(prefix != NULL, "control point invariant");
  7922   const markOop proto = markOopDesc::prototype();
  7922   const markWord proto = markWord::prototype();
  7923   oop next;
  7923   oop next;
  7924   NOT_PRODUCT(ssize_t n = 0;)
  7924   NOT_PRODUCT(ssize_t n = 0;)
  7925   for (cur = prefix; cur != NULL; cur = next) {
  7925   for (cur = prefix; cur != NULL; cur = next) {
  7926     next = oop(cur->mark_raw());
  7926     next = oop(cur->mark_raw().to_pointer());
  7927     cur->set_mark_raw(proto);   // until proven otherwise
  7927     cur->set_mark_raw(proto);   // until proven otherwise
  7928     assert(oopDesc::is_oop(cur), "Should be an oop");
  7928     assert(oopDesc::is_oop(cur), "Should be an oop");
  7929     bool res = work_q->push(cur);
  7929     bool res = work_q->push(cur);
  7930     assert(res, "Bit off more than we can chew?");
  7930     assert(res, "Bit off more than we can chew?");
  7931     NOT_PRODUCT(n++;)
  7931     NOT_PRODUCT(n++;)
  7940 // Single-threaded
  7940 // Single-threaded
  7941 void CMSCollector::push_on_overflow_list(oop p) {
  7941 void CMSCollector::push_on_overflow_list(oop p) {
  7942   NOT_PRODUCT(_num_par_pushes++;)
  7942   NOT_PRODUCT(_num_par_pushes++;)
  7943   assert(oopDesc::is_oop(p), "Not an oop");
  7943   assert(oopDesc::is_oop(p), "Not an oop");
  7944   preserve_mark_if_necessary(p);
  7944   preserve_mark_if_necessary(p);
  7945   p->set_mark_raw((markOop)_overflow_list);
  7945   p->set_mark_raw(markWord::from_pointer(_overflow_list));
  7946   _overflow_list = p;
  7946   _overflow_list = p;
  7947 }
  7947 }
  7948 
  7948 
  7949 // Multi-threaded; use CAS to prepend to overflow list
  7949 // Multi-threaded; use CAS to prepend to overflow list
  7950 void CMSCollector::par_push_on_overflow_list(oop p) {
  7950 void CMSCollector::par_push_on_overflow_list(oop p) {
  7954   oop observed_overflow_list = _overflow_list;
  7954   oop observed_overflow_list = _overflow_list;
  7955   oop cur_overflow_list;
  7955   oop cur_overflow_list;
  7956   do {
  7956   do {
  7957     cur_overflow_list = observed_overflow_list;
  7957     cur_overflow_list = observed_overflow_list;
  7958     if (cur_overflow_list != BUSY) {
  7958     if (cur_overflow_list != BUSY) {
  7959       p->set_mark_raw(markOop(cur_overflow_list));
  7959       p->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list));
  7960     } else {
  7960     } else {
  7961       p->set_mark_raw(NULL);
  7961       p->set_mark_raw(markWord::from_pointer(NULL));
  7962     }
  7962     }
  7963     observed_overflow_list =
  7963     observed_overflow_list =
  7964       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
  7964       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
  7965   } while (cur_overflow_list != observed_overflow_list);
  7965   } while (cur_overflow_list != observed_overflow_list);
  7966 }
  7966 }
  7978 // an indication of success or failure with the assumption that
  7978 // an indication of success or failure with the assumption that
  7979 // the caller may be able to recover from a failure; code in
  7979 // the caller may be able to recover from a failure; code in
  7980 // the VM can then be changed, incrementally, to deal with such
  7980 // the VM can then be changed, incrementally, to deal with such
  7981 // failures where possible, thus, incrementally hardening the VM
  7981 // failures where possible, thus, incrementally hardening the VM
  7982 // in such low resource situations.
  7982 // in such low resource situations.
  7983 void CMSCollector::preserve_mark_work(oop p, markOop m) {
  7983 void CMSCollector::preserve_mark_work(oop p, markWord m) {
  7984   _preserved_oop_stack.push(p);
  7984   _preserved_oop_stack.push(p);
  7985   _preserved_mark_stack.push(m);
  7985   _preserved_mark_stack.push(m);
  7986   assert(m == p->mark_raw(), "Mark word changed");
  7986   assert(m == p->mark_raw(), "Mark word changed");
  7987   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
  7987   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
  7988          "bijection");
  7988          "bijection");
  7989 }
  7989 }
  7990 
  7990 
  7991 // Single threaded
  7991 // Single threaded
  7992 void CMSCollector::preserve_mark_if_necessary(oop p) {
  7992 void CMSCollector::preserve_mark_if_necessary(oop p) {
  7993   markOop m = p->mark_raw();
  7993   markWord m = p->mark_raw();
  7994   if (m->must_be_preserved(p)) {
  7994   if (m.must_be_preserved(p)) {
  7995     preserve_mark_work(p, m);
  7995     preserve_mark_work(p, m);
  7996   }
  7996   }
  7997 }
  7997 }
  7998 
  7998 
  7999 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
  7999 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
  8000   markOop m = p->mark_raw();
  8000   markWord m = p->mark_raw();
  8001   if (m->must_be_preserved(p)) {
  8001   if (m.must_be_preserved(p)) {
  8002     MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
  8002     MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
  8003     // Even though we read the mark word without holding
  8003     // Even though we read the mark word without holding
  8004     // the lock, we are assured that it will not change
  8004     // the lock, we are assured that it will not change
  8005     // because we "own" this oop, so no other thread can
  8005     // because we "own" this oop, so no other thread can
  8006     // be trying to push it on the overflow list; see
  8006     // be trying to push it on the overflow list; see
  8036 
  8036 
  8037   while (!_preserved_oop_stack.is_empty()) {
  8037   while (!_preserved_oop_stack.is_empty()) {
  8038     oop p = _preserved_oop_stack.pop();
  8038     oop p = _preserved_oop_stack.pop();
  8039     assert(oopDesc::is_oop(p), "Should be an oop");
  8039     assert(oopDesc::is_oop(p), "Should be an oop");
  8040     assert(_span.contains(p), "oop should be in _span");
  8040     assert(_span.contains(p), "oop should be in _span");
  8041     assert(p->mark_raw() == markOopDesc::prototype(),
  8041     assert(p->mark_raw() == markWord::prototype(),
  8042            "Set when taken from overflow list");
  8042            "Set when taken from overflow list");
  8043     markOop m = _preserved_mark_stack.pop();
  8043     markWord m = _preserved_mark_stack.pop();
  8044     p->set_mark_raw(m);
  8044     p->set_mark_raw(m);
  8045   }
  8045   }
  8046   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
  8046   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
  8047          "stacks were cleared above");
  8047          "stacks were cleared above");
  8048 }
  8048 }