src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp
changeset 49722 a47d1e21b3f1
parent 49709 6705b1563b42
child 49821 02c08e20d66c
equal deleted inserted replaced
49721:ea0cc7c74e75 49722:a47d1e21b3f1
  1042   // Otherwise, copy the object.  Here we must be careful to insert the
  1042   // Otherwise, copy the object.  Here we must be careful to insert the
  1043   // klass pointer last, since this marks the block as an allocated object.
  1043   // klass pointer last, since this marks the block as an allocated object.
  1044   // Except with compressed oops it's the mark word.
  1044   // Except with compressed oops it's the mark word.
  1045   HeapWord* old_ptr = (HeapWord*)old;
  1045   HeapWord* old_ptr = (HeapWord*)old;
  1046   // Restore the mark word copied above.
  1046   // Restore the mark word copied above.
  1047   obj->set_mark(m);
  1047   obj->set_mark_raw(m);
  1048   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
  1048   assert(obj->klass_or_null() == NULL, "Object should be uninitialized here.");
  1049   assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
  1049   assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size");
  1050   OrderAccess::storestore();
  1050   OrderAccess::storestore();
  1051 
  1051 
  1052   if (UseCompressedClassPointers) {
  1052   if (UseCompressedClassPointers) {
  7814   size_t i = num;
  7814   size_t i = num;
  7815   oop  cur = _overflow_list;
  7815   oop  cur = _overflow_list;
  7816   const markOop proto = markOopDesc::prototype();
  7816   const markOop proto = markOopDesc::prototype();
  7817   NOT_PRODUCT(ssize_t n = 0;)
  7817   NOT_PRODUCT(ssize_t n = 0;)
  7818   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
  7818   for (oop next; i > 0 && cur != NULL; cur = next, i--) {
  7819     next = oop(cur->mark());
  7819     next = oop(cur->mark_raw());
  7820     cur->set_mark(proto);   // until proven otherwise
  7820     cur->set_mark_raw(proto);   // until proven otherwise
  7821     assert(oopDesc::is_oop(cur), "Should be an oop");
  7821     assert(oopDesc::is_oop(cur), "Should be an oop");
  7822     bool res = stack->push(cur);
  7822     bool res = stack->push(cur);
  7823     assert(res, "Bit off more than can chew?");
  7823     assert(res, "Bit off more than can chew?");
  7824     NOT_PRODUCT(n++;)
  7824     NOT_PRODUCT(n++;)
  7825   }
  7825   }
  7898   }
  7898   }
  7899   assert(prefix != NULL && prefix != BUSY, "Error");
  7899   assert(prefix != NULL && prefix != BUSY, "Error");
  7900   size_t i = num;
  7900   size_t i = num;
  7901   oop cur = prefix;
  7901   oop cur = prefix;
  7902   // Walk down the first "num" objects, unless we reach the end.
  7902   // Walk down the first "num" objects, unless we reach the end.
  7903   for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--);
  7903   for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--);
  7904   if (cur->mark() == NULL) {
  7904   if (cur->mark_raw() == NULL) {
  7905     // We have "num" or fewer elements in the list, so there
  7905     // We have "num" or fewer elements in the list, so there
  7906     // is nothing to return to the global list.
  7906     // is nothing to return to the global list.
  7907     // Write back the NULL in lieu of the BUSY we wrote
  7907     // Write back the NULL in lieu of the BUSY we wrote
  7908     // above, if it is still the same value.
  7908     // above, if it is still the same value.
  7909     if (_overflow_list == BUSY) {
  7909     if (_overflow_list == BUSY) {
  7910       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
  7910       Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY);
  7911     }
  7911     }
  7912   } else {
  7912   } else {
  7913     // Chop off the suffix and return it to the global list.
  7913     // Chop off the suffix and return it to the global list.
  7914     assert(cur->mark() != BUSY, "Error");
  7914     assert(cur->mark_raw() != BUSY, "Error");
  7915     oop suffix_head = cur->mark(); // suffix will be put back on global list
  7915     oop suffix_head = cur->mark_raw(); // suffix will be put back on global list
  7916     cur->set_mark(NULL);           // break off suffix
  7916     cur->set_mark_raw(NULL);           // break off suffix
  7917     // It's possible that the list is still in the empty(busy) state
  7917     // It's possible that the list is still in the empty(busy) state
  7918     // we left it in a short while ago; in that case we may be
  7918     // we left it in a short while ago; in that case we may be
  7919     // able to place back the suffix without incurring the cost
  7919     // able to place back the suffix without incurring the cost
  7920     // of a walk down the list.
  7920     // of a walk down the list.
  7921     oop observed_overflow_list = _overflow_list;
  7921     oop observed_overflow_list = _overflow_list;
  7931     }
  7931     }
  7932     if (!attached) {
  7932     if (!attached) {
  7933       // Too bad, someone else sneaked in (at least) an element; we'll need
  7933       // Too bad, someone else sneaked in (at least) an element; we'll need
  7934       // to do a splice. Find tail of suffix so we can prepend suffix to global
  7934       // to do a splice. Find tail of suffix so we can prepend suffix to global
  7935       // list.
  7935       // list.
  7936       for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark()));
  7936       for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw()));
  7937       oop suffix_tail = cur;
  7937       oop suffix_tail = cur;
  7938       assert(suffix_tail != NULL && suffix_tail->mark() == NULL,
  7938       assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL,
  7939              "Tautology");
  7939              "Tautology");
  7940       observed_overflow_list = _overflow_list;
  7940       observed_overflow_list = _overflow_list;
  7941       do {
  7941       do {
  7942         cur_overflow_list = observed_overflow_list;
  7942         cur_overflow_list = observed_overflow_list;
  7943         if (cur_overflow_list != BUSY) {
  7943         if (cur_overflow_list != BUSY) {
  7944           // Do the splice ...
  7944           // Do the splice ...
  7945           suffix_tail->set_mark(markOop(cur_overflow_list));
  7945           suffix_tail->set_mark_raw(markOop(cur_overflow_list));
  7946         } else { // cur_overflow_list == BUSY
  7946         } else { // cur_overflow_list == BUSY
  7947           suffix_tail->set_mark(NULL);
  7947           suffix_tail->set_mark_raw(NULL);
  7948         }
  7948         }
  7949         // ... and try to place spliced list back on overflow_list ...
  7949         // ... and try to place spliced list back on overflow_list ...
  7950         observed_overflow_list =
  7950         observed_overflow_list =
  7951           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
  7951           Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list);
  7952       } while (cur_overflow_list != observed_overflow_list);
  7952       } while (cur_overflow_list != observed_overflow_list);
  7958   assert(prefix != NULL, "control point invariant");
  7958   assert(prefix != NULL, "control point invariant");
  7959   const markOop proto = markOopDesc::prototype();
  7959   const markOop proto = markOopDesc::prototype();
  7960   oop next;
  7960   oop next;
  7961   NOT_PRODUCT(ssize_t n = 0;)
  7961   NOT_PRODUCT(ssize_t n = 0;)
  7962   for (cur = prefix; cur != NULL; cur = next) {
  7962   for (cur = prefix; cur != NULL; cur = next) {
  7963     next = oop(cur->mark());
  7963     next = oop(cur->mark_raw());
  7964     cur->set_mark(proto);   // until proven otherwise
  7964     cur->set_mark_raw(proto);   // until proven otherwise
  7965     assert(oopDesc::is_oop(cur), "Should be an oop");
  7965     assert(oopDesc::is_oop(cur), "Should be an oop");
  7966     bool res = work_q->push(cur);
  7966     bool res = work_q->push(cur);
  7967     assert(res, "Bit off more than we can chew?");
  7967     assert(res, "Bit off more than we can chew?");
  7968     NOT_PRODUCT(n++;)
  7968     NOT_PRODUCT(n++;)
  7969   }
  7969   }
  7977 // Single-threaded
  7977 // Single-threaded
  7978 void CMSCollector::push_on_overflow_list(oop p) {
  7978 void CMSCollector::push_on_overflow_list(oop p) {
  7979   NOT_PRODUCT(_num_par_pushes++;)
  7979   NOT_PRODUCT(_num_par_pushes++;)
  7980   assert(oopDesc::is_oop(p), "Not an oop");
  7980   assert(oopDesc::is_oop(p), "Not an oop");
  7981   preserve_mark_if_necessary(p);
  7981   preserve_mark_if_necessary(p);
  7982   p->set_mark((markOop)_overflow_list);
  7982   p->set_mark_raw((markOop)_overflow_list);
  7983   _overflow_list = p;
  7983   _overflow_list = p;
  7984 }
  7984 }
  7985 
  7985 
  7986 // Multi-threaded; use CAS to prepend to overflow list
  7986 // Multi-threaded; use CAS to prepend to overflow list
  7987 void CMSCollector::par_push_on_overflow_list(oop p) {
  7987 void CMSCollector::par_push_on_overflow_list(oop p) {
  7991   oop observed_overflow_list = _overflow_list;
  7991   oop observed_overflow_list = _overflow_list;
  7992   oop cur_overflow_list;
  7992   oop cur_overflow_list;
  7993   do {
  7993   do {
  7994     cur_overflow_list = observed_overflow_list;
  7994     cur_overflow_list = observed_overflow_list;
  7995     if (cur_overflow_list != BUSY) {
  7995     if (cur_overflow_list != BUSY) {
  7996       p->set_mark(markOop(cur_overflow_list));
  7996       p->set_mark_raw(markOop(cur_overflow_list));
  7997     } else {
  7997     } else {
  7998       p->set_mark(NULL);
  7998       p->set_mark_raw(NULL);
  7999     }
  7999     }
  8000     observed_overflow_list =
  8000     observed_overflow_list =
  8001       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
  8001       Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list);
  8002   } while (cur_overflow_list != observed_overflow_list);
  8002   } while (cur_overflow_list != observed_overflow_list);
  8003 }
  8003 }
  8018 // failures where possible, thus, incrementally hardening the VM
  8018 // failures where possible, thus, incrementally hardening the VM
  8019 // in such low resource situations.
  8019 // in such low resource situations.
  8020 void CMSCollector::preserve_mark_work(oop p, markOop m) {
  8020 void CMSCollector::preserve_mark_work(oop p, markOop m) {
  8021   _preserved_oop_stack.push(p);
  8021   _preserved_oop_stack.push(p);
  8022   _preserved_mark_stack.push(m);
  8022   _preserved_mark_stack.push(m);
  8023   assert(m == p->mark(), "Mark word changed");
  8023   assert(m == p->mark_raw(), "Mark word changed");
  8024   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
  8024   assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(),
  8025          "bijection");
  8025          "bijection");
  8026 }
  8026 }
  8027 
  8027 
  8028 // Single threaded
  8028 // Single threaded
  8029 void CMSCollector::preserve_mark_if_necessary(oop p) {
  8029 void CMSCollector::preserve_mark_if_necessary(oop p) {
  8030   markOop m = p->mark();
  8030   markOop m = p->mark_raw();
  8031   if (m->must_be_preserved(p)) {
  8031   if (m->must_be_preserved(p)) {
  8032     preserve_mark_work(p, m);
  8032     preserve_mark_work(p, m);
  8033   }
  8033   }
  8034 }
  8034 }
  8035 
  8035 
  8036 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
  8036 void CMSCollector::par_preserve_mark_if_necessary(oop p) {
  8037   markOop m = p->mark();
  8037   markOop m = p->mark_raw();
  8038   if (m->must_be_preserved(p)) {
  8038   if (m->must_be_preserved(p)) {
  8039     MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
  8039     MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
  8040     // Even though we read the mark word without holding
  8040     // Even though we read the mark word without holding
  8041     // the lock, we are assured that it will not change
  8041     // the lock, we are assured that it will not change
  8042     // because we "own" this oop, so no other thread can
  8042     // because we "own" this oop, so no other thread can
  8043     // be trying to push it on the overflow list; see
  8043     // be trying to push it on the overflow list; see
  8044     // the assertion in preserve_mark_work() that checks
  8044     // the assertion in preserve_mark_work() that checks
  8045     // that m == p->mark().
  8045     // that m == p->mark_raw().
  8046     preserve_mark_work(p, m);
  8046     preserve_mark_work(p, m);
  8047   }
  8047   }
  8048 }
  8048 }
  8049 
  8049 
  8050 // We should be able to do this multi-threaded,
  8050 // We should be able to do this multi-threaded,
  8073 
  8073 
  8074   while (!_preserved_oop_stack.is_empty()) {
  8074   while (!_preserved_oop_stack.is_empty()) {
  8075     oop p = _preserved_oop_stack.pop();
  8075     oop p = _preserved_oop_stack.pop();
  8076     assert(oopDesc::is_oop(p), "Should be an oop");
  8076     assert(oopDesc::is_oop(p), "Should be an oop");
  8077     assert(_span.contains(p), "oop should be in _span");
  8077     assert(_span.contains(p), "oop should be in _span");
  8078     assert(p->mark() == markOopDesc::prototype(),
  8078     assert(p->mark_raw() == markOopDesc::prototype(),
  8079            "Set when taken from overflow list");
  8079            "Set when taken from overflow list");
  8080     markOop m = _preserved_mark_stack.pop();
  8080     markOop m = _preserved_mark_stack.pop();
  8081     p->set_mark(m);
  8081     p->set_mark_raw(m);
  8082   }
  8082   }
  8083   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
  8083   assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(),
  8084          "stacks were cleared above");
  8084          "stacks were cleared above");
  8085 }
  8085 }
  8086 
  8086