1042 // Otherwise, copy the object. Here we must be careful to insert the |
1042 // Otherwise, copy the object. Here we must be careful to insert the |
1043 // klass pointer last, since this marks the block as an allocated object. |
1043 // klass pointer last, since this marks the block as an allocated object. |
1044 // Except with compressed oops it's the mark word. |
1044 // Except with compressed oops it's the mark word. |
1045 HeapWord* old_ptr = (HeapWord*)old; |
1045 HeapWord* old_ptr = (HeapWord*)old; |
1046 // Restore the mark word copied above. |
1046 // Restore the mark word copied above. |
1047 obj->set_mark(m); |
1047 obj->set_mark_raw(m); |
1048 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); |
1048 assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); |
1049 assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); |
1049 assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); |
1050 OrderAccess::storestore(); |
1050 OrderAccess::storestore(); |
1051 |
1051 |
1052 if (UseCompressedClassPointers) { |
1052 if (UseCompressedClassPointers) { |
7814 size_t i = num; |
7814 size_t i = num; |
7815 oop cur = _overflow_list; |
7815 oop cur = _overflow_list; |
7816 const markOop proto = markOopDesc::prototype(); |
7816 const markOop proto = markOopDesc::prototype(); |
7817 NOT_PRODUCT(ssize_t n = 0;) |
7817 NOT_PRODUCT(ssize_t n = 0;) |
7818 for (oop next; i > 0 && cur != NULL; cur = next, i--) { |
7818 for (oop next; i > 0 && cur != NULL; cur = next, i--) { |
7819 next = oop(cur->mark()); |
7819 next = oop(cur->mark_raw()); |
7820 cur->set_mark(proto); // until proven otherwise |
7820 cur->set_mark_raw(proto); // until proven otherwise |
7821 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7821 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7822 bool res = stack->push(cur); |
7822 bool res = stack->push(cur); |
7823 assert(res, "Bit off more than can chew?"); |
7823 assert(res, "Bit off more than can chew?"); |
7824 NOT_PRODUCT(n++;) |
7824 NOT_PRODUCT(n++;) |
7825 } |
7825 } |
7898 } |
7898 } |
7899 assert(prefix != NULL && prefix != BUSY, "Error"); |
7899 assert(prefix != NULL && prefix != BUSY, "Error"); |
7900 size_t i = num; |
7900 size_t i = num; |
7901 oop cur = prefix; |
7901 oop cur = prefix; |
7902 // Walk down the first "num" objects, unless we reach the end. |
7902 // Walk down the first "num" objects, unless we reach the end. |
7903 for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--); |
7903 for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--); |
7904 if (cur->mark() == NULL) { |
7904 if (cur->mark_raw() == NULL) { |
7905 // We have "num" or fewer elements in the list, so there |
7905 // We have "num" or fewer elements in the list, so there |
7906 // is nothing to return to the global list. |
7906 // is nothing to return to the global list. |
7907 // Write back the NULL in lieu of the BUSY we wrote |
7907 // Write back the NULL in lieu of the BUSY we wrote |
7908 // above, if it is still the same value. |
7908 // above, if it is still the same value. |
7909 if (_overflow_list == BUSY) { |
7909 if (_overflow_list == BUSY) { |
7910 Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY); |
7910 Atomic::cmpxchg((oopDesc*)NULL, &_overflow_list, (oopDesc*)BUSY); |
7911 } |
7911 } |
7912 } else { |
7912 } else { |
7913 // Chop off the suffix and return it to the global list. |
7913 // Chop off the suffix and return it to the global list. |
7914 assert(cur->mark() != BUSY, "Error"); |
7914 assert(cur->mark_raw() != BUSY, "Error"); |
7915 oop suffix_head = cur->mark(); // suffix will be put back on global list |
7915 oop suffix_head = cur->mark_raw(); // suffix will be put back on global list |
7916 cur->set_mark(NULL); // break off suffix |
7916 cur->set_mark_raw(NULL); // break off suffix |
7917 // It's possible that the list is still in the empty(busy) state |
7917 // It's possible that the list is still in the empty(busy) state |
7918 // we left it in a short while ago; in that case we may be |
7918 // we left it in a short while ago; in that case we may be |
7919 // able to place back the suffix without incurring the cost |
7919 // able to place back the suffix without incurring the cost |
7920 // of a walk down the list. |
7920 // of a walk down the list. |
7921 oop observed_overflow_list = _overflow_list; |
7921 oop observed_overflow_list = _overflow_list; |
7931 } |
7931 } |
7932 if (!attached) { |
7932 if (!attached) { |
7933 // Too bad, someone else sneaked in (at least) an element; we'll need |
7933 // Too bad, someone else sneaked in (at least) an element; we'll need |
7934 // to do a splice. Find tail of suffix so we can prepend suffix to global |
7934 // to do a splice. Find tail of suffix so we can prepend suffix to global |
7935 // list. |
7935 // list. |
7936 for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark())); |
7936 for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw())); |
7937 oop suffix_tail = cur; |
7937 oop suffix_tail = cur; |
7938 assert(suffix_tail != NULL && suffix_tail->mark() == NULL, |
7938 assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL, |
7939 "Tautology"); |
7939 "Tautology"); |
7940 observed_overflow_list = _overflow_list; |
7940 observed_overflow_list = _overflow_list; |
7941 do { |
7941 do { |
7942 cur_overflow_list = observed_overflow_list; |
7942 cur_overflow_list = observed_overflow_list; |
7943 if (cur_overflow_list != BUSY) { |
7943 if (cur_overflow_list != BUSY) { |
7944 // Do the splice ... |
7944 // Do the splice ... |
7945 suffix_tail->set_mark(markOop(cur_overflow_list)); |
7945 suffix_tail->set_mark_raw(markOop(cur_overflow_list)); |
7946 } else { // cur_overflow_list == BUSY |
7946 } else { // cur_overflow_list == BUSY |
7947 suffix_tail->set_mark(NULL); |
7947 suffix_tail->set_mark_raw(NULL); |
7948 } |
7948 } |
7949 // ... and try to place spliced list back on overflow_list ... |
7949 // ... and try to place spliced list back on overflow_list ... |
7950 observed_overflow_list = |
7950 observed_overflow_list = |
7951 Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list); |
7951 Atomic::cmpxchg((oopDesc*)suffix_head, &_overflow_list, (oopDesc*)cur_overflow_list); |
7952 } while (cur_overflow_list != observed_overflow_list); |
7952 } while (cur_overflow_list != observed_overflow_list); |
7958 assert(prefix != NULL, "control point invariant"); |
7958 assert(prefix != NULL, "control point invariant"); |
7959 const markOop proto = markOopDesc::prototype(); |
7959 const markOop proto = markOopDesc::prototype(); |
7960 oop next; |
7960 oop next; |
7961 NOT_PRODUCT(ssize_t n = 0;) |
7961 NOT_PRODUCT(ssize_t n = 0;) |
7962 for (cur = prefix; cur != NULL; cur = next) { |
7962 for (cur = prefix; cur != NULL; cur = next) { |
7963 next = oop(cur->mark()); |
7963 next = oop(cur->mark_raw()); |
7964 cur->set_mark(proto); // until proven otherwise |
7964 cur->set_mark_raw(proto); // until proven otherwise |
7965 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7965 assert(oopDesc::is_oop(cur), "Should be an oop"); |
7966 bool res = work_q->push(cur); |
7966 bool res = work_q->push(cur); |
7967 assert(res, "Bit off more than we can chew?"); |
7967 assert(res, "Bit off more than we can chew?"); |
7968 NOT_PRODUCT(n++;) |
7968 NOT_PRODUCT(n++;) |
7969 } |
7969 } |
7977 // Single-threaded |
7977 // Single-threaded |
7978 void CMSCollector::push_on_overflow_list(oop p) { |
7978 void CMSCollector::push_on_overflow_list(oop p) { |
7979 NOT_PRODUCT(_num_par_pushes++;) |
7979 NOT_PRODUCT(_num_par_pushes++;) |
7980 assert(oopDesc::is_oop(p), "Not an oop"); |
7980 assert(oopDesc::is_oop(p), "Not an oop"); |
7981 preserve_mark_if_necessary(p); |
7981 preserve_mark_if_necessary(p); |
7982 p->set_mark((markOop)_overflow_list); |
7982 p->set_mark_raw((markOop)_overflow_list); |
7983 _overflow_list = p; |
7983 _overflow_list = p; |
7984 } |
7984 } |
7985 |
7985 |
7986 // Multi-threaded; use CAS to prepend to overflow list |
7986 // Multi-threaded; use CAS to prepend to overflow list |
7987 void CMSCollector::par_push_on_overflow_list(oop p) { |
7987 void CMSCollector::par_push_on_overflow_list(oop p) { |
7991 oop observed_overflow_list = _overflow_list; |
7991 oop observed_overflow_list = _overflow_list; |
7992 oop cur_overflow_list; |
7992 oop cur_overflow_list; |
7993 do { |
7993 do { |
7994 cur_overflow_list = observed_overflow_list; |
7994 cur_overflow_list = observed_overflow_list; |
7995 if (cur_overflow_list != BUSY) { |
7995 if (cur_overflow_list != BUSY) { |
7996 p->set_mark(markOop(cur_overflow_list)); |
7996 p->set_mark_raw(markOop(cur_overflow_list)); |
7997 } else { |
7997 } else { |
7998 p->set_mark(NULL); |
7998 p->set_mark_raw(NULL); |
7999 } |
7999 } |
8000 observed_overflow_list = |
8000 observed_overflow_list = |
8001 Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); |
8001 Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); |
8002 } while (cur_overflow_list != observed_overflow_list); |
8002 } while (cur_overflow_list != observed_overflow_list); |
8003 } |
8003 } |
8018 // failures where possible, thus, incrementally hardening the VM |
8018 // failures where possible, thus, incrementally hardening the VM |
8019 // in such low resource situations. |
8019 // in such low resource situations. |
8020 void CMSCollector::preserve_mark_work(oop p, markOop m) { |
8020 void CMSCollector::preserve_mark_work(oop p, markOop m) { |
8021 _preserved_oop_stack.push(p); |
8021 _preserved_oop_stack.push(p); |
8022 _preserved_mark_stack.push(m); |
8022 _preserved_mark_stack.push(m); |
8023 assert(m == p->mark(), "Mark word changed"); |
8023 assert(m == p->mark_raw(), "Mark word changed"); |
8024 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), |
8024 assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), |
8025 "bijection"); |
8025 "bijection"); |
8026 } |
8026 } |
8027 |
8027 |
8028 // Single threaded |
8028 // Single threaded |
8029 void CMSCollector::preserve_mark_if_necessary(oop p) { |
8029 void CMSCollector::preserve_mark_if_necessary(oop p) { |
8030 markOop m = p->mark(); |
8030 markOop m = p->mark_raw(); |
8031 if (m->must_be_preserved(p)) { |
8031 if (m->must_be_preserved(p)) { |
8032 preserve_mark_work(p, m); |
8032 preserve_mark_work(p, m); |
8033 } |
8033 } |
8034 } |
8034 } |
8035 |
8035 |
8036 void CMSCollector::par_preserve_mark_if_necessary(oop p) { |
8036 void CMSCollector::par_preserve_mark_if_necessary(oop p) { |
8037 markOop m = p->mark(); |
8037 markOop m = p->mark_raw(); |
8038 if (m->must_be_preserved(p)) { |
8038 if (m->must_be_preserved(p)) { |
8039 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); |
8039 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); |
8040 // Even though we read the mark word without holding |
8040 // Even though we read the mark word without holding |
8041 // the lock, we are assured that it will not change |
8041 // the lock, we are assured that it will not change |
8042 // because we "own" this oop, so no other thread can |
8042 // because we "own" this oop, so no other thread can |
8043 // be trying to push it on the overflow list; see |
8043 // be trying to push it on the overflow list; see |
8044 // the assertion in preserve_mark_work() that checks |
8044 // the assertion in preserve_mark_work() that checks |
8045 // that m == p->mark(). |
8045 // that m == p->mark_raw(). |
8046 preserve_mark_work(p, m); |
8046 preserve_mark_work(p, m); |
8047 } |
8047 } |
8048 } |
8048 } |
8049 |
8049 |
8050 // We should be able to do this multi-threaded, |
8050 // We should be able to do this multi-threaded, |
8073 |
8073 |
8074 while (!_preserved_oop_stack.is_empty()) { |
8074 while (!_preserved_oop_stack.is_empty()) { |
8075 oop p = _preserved_oop_stack.pop(); |
8075 oop p = _preserved_oop_stack.pop(); |
8076 assert(oopDesc::is_oop(p), "Should be an oop"); |
8076 assert(oopDesc::is_oop(p), "Should be an oop"); |
8077 assert(_span.contains(p), "oop should be in _span"); |
8077 assert(_span.contains(p), "oop should be in _span"); |
8078 assert(p->mark() == markOopDesc::prototype(), |
8078 assert(p->mark_raw() == markOopDesc::prototype(), |
8079 "Set when taken from overflow list"); |
8079 "Set when taken from overflow list"); |
8080 markOop m = _preserved_mark_stack.pop(); |
8080 markOop m = _preserved_mark_stack.pop(); |
8081 p->set_mark(m); |
8081 p->set_mark_raw(m); |
8082 } |
8082 } |
8083 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), |
8083 assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), |
8084 "stacks were cleared above"); |
8084 "stacks were cleared above"); |
8085 } |
8085 } |
8086 |
8086 |