7774 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) { |
7774 bool CMSCollector::take_from_overflow_list(size_t num, CMSMarkStack* stack) { |
7775 assert(stack->isEmpty(), "Expected precondition"); |
7775 assert(stack->isEmpty(), "Expected precondition"); |
7776 assert(stack->capacity() > num, "Shouldn't bite more than can chew"); |
7776 assert(stack->capacity() > num, "Shouldn't bite more than can chew"); |
7777 size_t i = num; |
7777 size_t i = num; |
7778 oop cur = _overflow_list; |
7778 oop cur = _overflow_list; |
7779 const markOop proto = markOopDesc::prototype(); |
7779 const markWord proto = markWord::prototype(); |
7780 NOT_PRODUCT(ssize_t n = 0;) |
7780 NOT_PRODUCT(ssize_t n = 0;) |
7781 for (oop next; i > 0 && cur != NULL; cur = next, i--) { |
7781 for (oop next; i > 0 && cur != NULL; cur = next, i--) { |
7782 next = oop(cur->mark_raw()); |
7782 next = oop(cur->mark_raw().to_pointer()); |
7783 cur->set_mark_raw(proto); // until proven otherwise |
7783 cur->set_mark_raw(proto); // until proven otherwise |
7784 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7784 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7785 bool res = stack->push(cur); |
7785 bool res = stack->push(cur); |
7786 assert(res, "Bit off more than can chew?"); |
7786 assert(res, "Bit off more than can chew?"); |
7787 NOT_PRODUCT(n++;) |
7787 NOT_PRODUCT(n++;) |
7861 } |
7861 } |
7862 assert(prefix != NULL && prefix != BUSY, "Error"); |
7862 assert(prefix != NULL && prefix != BUSY, "Error"); |
7863 size_t i = num; |
7863 size_t i = num; |
7864 oop cur = prefix; |
7864 oop cur = prefix; |
7865 // Walk down the first "num" objects, unless we reach the end. |
7865 // Walk down the first "num" objects, unless we reach the end. |
7866 for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--); |
7866 for (; i > 1 && cur->mark_raw().to_pointer() != NULL; cur = oop(cur->mark_raw().to_pointer()), i--); |
7867 if (cur->mark_raw() == NULL) { |
7867 if (cur->mark_raw().to_pointer() == NULL) { |
7868 // We have "num" or fewer elements in the list, so there |
7868 // We have "num" or fewer elements in the list, so there |
7869 // is nothing to return to the global list. |
7869 // is nothing to return to the global list. |
7870 // Write back the NULL in lieu of the BUSY we wrote |
7870 // Write back the NULL in lieu of the BUSY we wrote |
7871 // above, if it is still the same value. |
7871 // above, if it is still the same value. |
7872 if (_overflow_list == BUSY) { |
7872 if (_overflow_list == BUSY) { |
7873 Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY); |
7873 Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY); |
7874 } |
7874 } |
7875 } else { |
7875 } else { |
7876 // Chop off the suffix and return it to the global list. |
7876 // Chop off the suffix and return it to the global list. |
7877 assert(cur->mark_raw() != BUSY, "Error"); |
7877 assert(cur->mark_raw().to_pointer() != (void*)BUSY, "Error"); |
7878 oop suffix_head = cur->mark_raw(); // suffix will be put back on global list |
7878 oop suffix_head = oop(cur->mark_raw().to_pointer()); // suffix will be put back on global list |
7879 cur->set_mark_raw(NULL); // break off suffix |
7879 cur->set_mark_raw(markWord::from_pointer(NULL)); // break off suffix |
7880 // It's possible that the list is still in the empty(busy) state |
7880 // It's possible that the list is still in the empty(busy) state |
7881 // we left it in a short while ago; in that case we may be |
7881 // we left it in a short while ago; in that case we may be |
7882 // able to place back the suffix without incurring the cost |
7882 // able to place back the suffix without incurring the cost |
7883 // of a walk down the list. |
7883 // of a walk down the list. |
7884 oop observed_overflow_list = _overflow_list; |
7884 oop observed_overflow_list = _overflow_list; |
7894 } |
7894 } |
7895 if (!attached) { |
7895 if (!attached) { |
7896 // Too bad, someone else sneaked in (at least) an element; we'll need |
7896 // Too bad, someone else sneaked in (at least) an element; we'll need |
7897 // to do a splice. Find tail of suffix so we can prepend suffix to global |
7897 // to do a splice. Find tail of suffix so we can prepend suffix to global |
7898 // list. |
7898 // list. |
7899 for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw())); |
7899 for (cur = suffix_head; cur->mark_raw().to_pointer() != NULL; cur = (oop)(cur->mark_raw().to_pointer())); |
7900 oop suffix_tail = cur; |
7900 oop suffix_tail = cur; |
7901 assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL, |
7901 assert(suffix_tail != NULL && suffix_tail->mark_raw().to_pointer() == NULL, |
7902 "Tautology"); |
7902 "Tautology"); |
7903 observed_overflow_list = _overflow_list; |
7903 observed_overflow_list = _overflow_list; |
7904 do { |
7904 do { |
7905 cur_overflow_list = observed_overflow_list; |
7905 cur_overflow_list = observed_overflow_list; |
7906 if (cur_overflow_list != BUSY) { |
7906 if (cur_overflow_list != BUSY) { |
7907 // Do the splice ... |
7907 // Do the splice ... |
7908 suffix_tail->set_mark_raw(markOop(cur_overflow_list)); |
7908 suffix_tail->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list)); |
7909 } else { // cur_overflow_list == BUSY |
7909 } else { // cur_overflow_list == BUSY |
7910 suffix_tail->set_mark_raw(NULL); |
7910 suffix_tail->set_mark_raw(markWord::from_pointer(NULL)); |
7911 } |
7911 } |
7912 // ... and try to place spliced list back on overflow_list ... |
7912 // ... and try to place spliced list back on overflow_list ... |
7913 observed_overflow_list = |
7913 observed_overflow_list = |
7914 Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list); |
7914 Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list); |
7915 } while (cur_overflow_list != observed_overflow_list); |
7915 } while (cur_overflow_list != observed_overflow_list); |
7917 } |
7917 } |
7918 } |
7918 } |
7919 |
7919 |
7920 // Push the prefix elements on work_q |
7920 // Push the prefix elements on work_q |
7921 assert(prefix != NULL, "control point invariant"); |
7921 assert(prefix != NULL, "control point invariant"); |
7922 const markOop proto = markOopDesc::prototype(); |
7922 const markWord proto = markWord::prototype(); |
7923 oop next; |
7923 oop next; |
7924 NOT_PRODUCT(ssize_t n = 0;) |
7924 NOT_PRODUCT(ssize_t n = 0;) |
7925 for (cur = prefix; cur != NULL; cur = next) { |
7925 for (cur = prefix; cur != NULL; cur = next) { |
7926 next = oop(cur->mark_raw()); |
7926 next = oop(cur->mark_raw().to_pointer()); |
7927 cur->set_mark_raw(proto); // until proven otherwise |
7927 cur->set_mark_raw(proto); // until proven otherwise |
7928 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7928 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7929 bool res = work_q->push(cur); |
7929 bool res = work_q->push(cur); |
7930 assert(res, "Bit off more than we can chew?"); |
7930 assert(res, "Bit off more than we can chew?"); |
7931 NOT_PRODUCT(n++;) |
7931 NOT_PRODUCT(n++;) |
7940 // Single-threaded |
7940 // Single-threaded |
7941 void CMSCollector::push_on_overflow_list(oop p) { |
7941 void CMSCollector::push_on_overflow_list(oop p) { |
7942 NOT_PRODUCT(_num_par_pushes++;) |
7942 NOT_PRODUCT(_num_par_pushes++;) |
7943 assert(oopDesc::is_oop(p), "Not an oop"); |
7943 assert(oopDesc::is_oop(p), "Not an oop"); |
7944 preserve_mark_if_necessary(p); |
7944 preserve_mark_if_necessary(p); |
7945 p->set_mark_raw((markOop)_overflow_list); |
7945 p->set_mark_raw(markWord::from_pointer(_overflow_list)); |
7946 _overflow_list = p; |
7946 _overflow_list = p; |
7947 } |
7947 } |
7948 |
7948 |
7949 // Multi-threaded; use CAS to prepend to overflow list |
7949 // Multi-threaded; use CAS to prepend to overflow list |
7950 void CMSCollector::par_push_on_overflow_list(oop p) { |
7950 void CMSCollector::par_push_on_overflow_list(oop p) { |
7954 oop observed_overflow_list = _overflow_list; |
7954 oop observed_overflow_list = _overflow_list; |
7955 oop cur_overflow_list; |
7955 oop cur_overflow_list; |
7956 do { |
7956 do { |
7957 cur_overflow_list = observed_overflow_list; |
7957 cur_overflow_list = observed_overflow_list; |
7958 if (cur_overflow_list != BUSY) { |
7958 if (cur_overflow_list != BUSY) { |
7959 p->set_mark_raw(markOop(cur_overflow_list)); |
7959 p->set_mark_raw(markWord::from_pointer((void*)cur_overflow_list)); |
7960 } else { |
7960 } else { |
7961 p->set_mark_raw(NULL); |
7961 p->set_mark_raw(markWord::from_pointer(NULL)); |
7962 } |
7962 } |
7963 observed_overflow_list = |
7963 observed_overflow_list = |
7964 Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); |
7964 Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); |
7965 } while (cur_overflow_list != observed_overflow_list); |
7965 } while (cur_overflow_list != observed_overflow_list); |
7966 } |
7966 } |
7978 // an indication of success or failure with the assumption that |
7978 // an indication of success or failure with the assumption that |
7979 // the caller may be able to recover from a failure; code in |
7979 // the caller may be able to recover from a failure; code in |
7980 // the VM can then be changed, incrementally, to deal with such |
7980 // the VM can then be changed, incrementally, to deal with such |
7981 // failures where possible, thus, incrementally hardening the VM |
7981 // failures where possible, thus, incrementally hardening the VM |
7982 // in such low resource situations. |
7982 // in such low resource situations. |
7983 void CMSCollector::preserve_mark_work(oop p, markOop m) { |
7983 void CMSCollector::preserve_mark_work(oop p, markWord m) { |
7984 _preserved_oop_stack.push(p); |
7984 _preserved_oop_stack.push(p); |
7985 _preserved_mark_stack.push(m); |
7985 _preserved_mark_stack.push(m); |
7986 assert(m == p->mark_raw(), "Mark word changed"); |
7986 assert(m == p->mark_raw(), "Mark word changed"); |
7987 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), |
7987 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), |
7988 "bijection"); |
7988 "bijection"); |
7989 } |
7989 } |
7990 |
7990 |
7991 // Single threaded |
7991 // Single threaded |
7992 void CMSCollector::preserve_mark_if_necessary(oop p) { |
7992 void CMSCollector::preserve_mark_if_necessary(oop p) { |
7993 markOop m = p->mark_raw(); |
7993 markWord m = p->mark_raw(); |
7994 if (m->must_be_preserved(p)) { |
7994 if (m.must_be_preserved(p)) { |
7995 preserve_mark_work(p, m); |
7995 preserve_mark_work(p, m); |
7996 } |
7996 } |
7997 } |
7997 } |
7998 |
7998 |
7999 void CMSCollector::par_preserve_mark_if_necessary(oop p) { |
7999 void CMSCollector::par_preserve_mark_if_necessary(oop p) { |
8000 markOop m = p->mark_raw(); |
8000 markWord m = p->mark_raw(); |
8001 if (m->must_be_preserved(p)) { |
8001 if (m.must_be_preserved(p)) { |
8002 MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); |
8002 MutexLocker x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); |
8003 // Even though we read the mark word without holding |
8003 // Even though we read the mark word without holding |
8004 // the lock, we are assured that it will not change |
8004 // the lock, we are assured that it will not change |
8005 // because we "own" this oop, so no other thread can |
8005 // because we "own" this oop, so no other thread can |
8006 // be trying to push it on the overflow list; see |
8006 // be trying to push it on the overflow list; see |
8036 |
8036 |
8037 while (!_preserved_oop_stack.is_empty()) { |
8037 while (!_preserved_oop_stack.is_empty()) { |
8038 oop p = _preserved_oop_stack.pop(); |
8038 oop p = _preserved_oop_stack.pop(); |
8039 assert(oopDesc::is_oop(p), "Should be an oop"); |
8039 assert(oopDesc::is_oop(p), "Should be an oop"); |
8040 assert(_span.contains(p), "oop should be in _span"); |
8040 assert(_span.contains(p), "oop should be in _span"); |
8041 assert(p->mark_raw() == markOopDesc::prototype(), |
8041 assert(p->mark_raw() == markWord::prototype(), |
8042 "Set when taken from overflow list"); |
8042 "Set when taken from overflow list"); |
8043 markOop m = _preserved_mark_stack.pop(); |
8043 markWord m = _preserved_mark_stack.pop(); |
8044 p->set_mark_raw(m); |
8044 p->set_mark_raw(m); |
8045 } |
8045 } |
8046 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), |
8046 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), |
8047 "stacks were cleared above"); |
8047 "stacks were cleared above"); |
8048 } |
8048 } |