# HG changeset patch # User rkennke # Date 1522918493 -7200 # Node ID a47d1e21b3f100bd0b6b37eb3646b26ae3a88fb5 # Parent ea0cc7c74e75a305480ecd3b313769058bed398f 8199735: Mark word updates need to use Access API Reviewed-by: shade, eosterlund diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/cpu/zero/cppInterpreter_zero.cpp --- a/src/hotspot/cpu/zero/cppInterpreter_zero.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/cpu/zero/cppInterpreter_zero.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -275,7 +275,7 @@ markOop disp = lockee->mark()->set_unlocked(); monitor->lock()->set_displaced_header(disp); - if (Atomic::cmpxchg((markOop)monitor, lockee->mark_addr(), disp) != disp) { + if (lockee->cas_set_mark((markOop)monitor, disp) != disp) { if (thread->is_lock_owned((address) disp->clear_lock_bits())) { monitor->lock()->set_displaced_header(NULL); } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/classfile/altHashing.cpp --- a/src/hotspot/share/classfile/altHashing.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/classfile/altHashing.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -27,6 +27,7 @@ #include "classfile/symbolTable.hpp" #include "classfile/systemDictionary.hpp" #include "oops/markOop.hpp" +#include "oops/oop.inline.hpp" #include "runtime/thread.hpp" // Get the hash code of the classes mirror if it exists, otherwise just diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp --- a/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/cms/compactibleFreeListSpace.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -420,7 +420,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - q->init_mark(); + q->init_mark_raw(); assert(q->forwardee() == NULL, "should be forwarded to NULL"); } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp --- a/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/cms/concurrentMarkSweepGeneration.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1044,7 +1044,7 @@ // Except with compressed oops it's the mark word. HeapWord* old_ptr = (HeapWord*)old; // Restore the mark word copied above. - obj->set_mark(m); + obj->set_mark_raw(m); assert(obj->klass_or_null() == NULL, "Object should be uninitialized here."); assert(!((FreeChunk*)obj_ptr)->is_free(), "Error, block will look free but show wrong size"); OrderAccess::storestore(); @@ -7816,8 +7816,8 @@ const markOop proto = markOopDesc::prototype(); NOT_PRODUCT(ssize_t n = 0;) for (oop next; i > 0 && cur != NULL; cur = next, i--) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise + next = oop(cur->mark_raw()); + cur->set_mark_raw(proto); // until proven otherwise assert(oopDesc::is_oop(cur), "Should be an oop"); bool res = stack->push(cur); assert(res, "Bit off more than can chew?"); @@ -7900,8 +7900,8 @@ size_t i = num; oop cur = prefix; // Walk down the first "num" objects, unless we reach the end. - for (; i > 1 && cur->mark() != NULL; cur = oop(cur->mark()), i--); - if (cur->mark() == NULL) { + for (; i > 1 && cur->mark_raw() != NULL; cur = oop(cur->mark_raw()), i--); + if (cur->mark_raw() == NULL) { // We have "num" or fewer elements in the list, so there // is nothing to return to the global list. // Write back the NULL in lieu of the BUSY we wrote @@ -7911,9 +7911,9 @@ } } else { // Chop off the suffix and return it to the global list. - assert(cur->mark() != BUSY, "Error"); - oop suffix_head = cur->mark(); // suffix will be put back on global list - cur->set_mark(NULL); // break off suffix + assert(cur->mark_raw() != BUSY, "Error"); + oop suffix_head = cur->mark_raw(); // suffix will be put back on global list + cur->set_mark_raw(NULL); // break off suffix // It's possible that the list is still in the empty(busy) state // we left it in a short while ago; in that case we may be // able to place back the suffix without incurring the cost @@ -7933,18 +7933,18 @@ // Too bad, someone else sneaked in (at least) an element; we'll need // to do a splice. Find tail of suffix so we can prepend suffix to global // list. - for (cur = suffix_head; cur->mark() != NULL; cur = (oop)(cur->mark())); + for (cur = suffix_head; cur->mark_raw() != NULL; cur = (oop)(cur->mark_raw())); oop suffix_tail = cur; - assert(suffix_tail != NULL && suffix_tail->mark() == NULL, + assert(suffix_tail != NULL && suffix_tail->mark_raw() == NULL, "Tautology"); observed_overflow_list = _overflow_list; do { cur_overflow_list = observed_overflow_list; if (cur_overflow_list != BUSY) { // Do the splice ... - suffix_tail->set_mark(markOop(cur_overflow_list)); + suffix_tail->set_mark_raw(markOop(cur_overflow_list)); } else { // cur_overflow_list == BUSY - suffix_tail->set_mark(NULL); + suffix_tail->set_mark_raw(NULL); } // ... and try to place spliced list back on overflow_list ... observed_overflow_list = @@ -7960,8 +7960,8 @@ oop next; NOT_PRODUCT(ssize_t n = 0;) for (cur = prefix; cur != NULL; cur = next) { - next = oop(cur->mark()); - cur->set_mark(proto); // until proven otherwise + next = oop(cur->mark_raw()); + cur->set_mark_raw(proto); // until proven otherwise assert(oopDesc::is_oop(cur), "Should be an oop"); bool res = work_q->push(cur); assert(res, "Bit off more than we can chew?"); @@ -7979,7 +7979,7 @@ NOT_PRODUCT(_num_par_pushes++;) assert(oopDesc::is_oop(p), "Not an oop"); preserve_mark_if_necessary(p); - p->set_mark((markOop)_overflow_list); + p->set_mark_raw((markOop)_overflow_list); _overflow_list = p; } @@ -7993,9 +7993,9 @@ do { cur_overflow_list = observed_overflow_list; if (cur_overflow_list != BUSY) { - p->set_mark(markOop(cur_overflow_list)); + p->set_mark_raw(markOop(cur_overflow_list)); } else { - p->set_mark(NULL); + p->set_mark_raw(NULL); } observed_overflow_list = Atomic::cmpxchg((oopDesc*)p, &_overflow_list, (oopDesc*)cur_overflow_list); @@ -8020,21 +8020,21 @@ void CMSCollector::preserve_mark_work(oop p, markOop m) { _preserved_oop_stack.push(p); _preserved_mark_stack.push(m); - assert(m == p->mark(), "Mark word changed"); + assert(m == p->mark_raw(), "Mark word changed"); assert(_preserved_oop_stack.size() == _preserved_mark_stack.size(), "bijection"); } // Single threaded void CMSCollector::preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); + markOop m = p->mark_raw(); if (m->must_be_preserved(p)) { preserve_mark_work(p, m); } } void CMSCollector::par_preserve_mark_if_necessary(oop p) { - markOop m = p->mark(); + markOop m = p->mark_raw(); if (m->must_be_preserved(p)) { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); // Even though we read the mark word without holding @@ -8042,7 +8042,7 @@ // because we "own" this oop, so no other thread can // be trying to push it on the overflow list; see // the assertion in preserve_mark_work() that checks - // that m == p->mark(). + // that m == p->mark_raw(). preserve_mark_work(p, m); } } @@ -8075,10 +8075,10 @@ oop p = _preserved_oop_stack.pop(); assert(oopDesc::is_oop(p), "Should be an oop"); assert(_span.contains(p), "oop should be in _span"); - assert(p->mark() == markOopDesc::prototype(), + assert(p->mark_raw() == markOopDesc::prototype(), "Set when taken from overflow list"); markOop m = _preserved_mark_stack.pop(); - p->set_mark(m); + p->set_mark_raw(m); } assert(_preserved_mark_stack.is_empty() && _preserved_oop_stack.is_empty(), "stacks were cleared above"); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/cms/parNewGeneration.cpp --- a/src/hotspot/share/gc/cms/parNewGeneration.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/cms/parNewGeneration.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1133,7 +1133,7 @@ // a forwarding pointer by a parallel thread. So we must save the mark // word in a local and then analyze it. oopDesc dummyOld; - dummyOld.set_mark(m); + dummyOld.set_mark_raw(m); assert(!dummyOld.is_forwarded(), "should not be called with forwarding pointer mark word."); @@ -1181,7 +1181,7 @@ assert(CMSHeap::heap()->is_in_reserved(new_obj), "illegal forwarding pointer value."); forward_ptr = old->forward_to_atomic(new_obj); // Restore the mark word copied above. - new_obj->set_mark(m); + new_obj->set_mark_raw(m); // Increment age if obj still in new generation new_obj->incr_age(); par_scan_state->age_table()->add(new_obj, sz); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/cms/parOopClosures.inline.hpp --- a/src/hotspot/share/gc/cms/parOopClosures.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/cms/parOopClosures.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -34,6 +34,7 @@ #include "logging/logStream.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" template inline void ParScanWeakRefClosure::do_oop_work(T* p) { oop obj = RawAccess::oop_load(p); @@ -43,7 +44,7 @@ // we need to ensure that it is copied (see comment in // ParScanClosure::do_oop_work). Klass* objK = obj->klass(); - markOop m = obj->mark(); + markOop m = obj->mark_raw(); oop new_obj; if (m->is_marked()) { // Contains forwarding pointer. new_obj = ParNewGeneration::real_forwardee(obj); @@ -107,7 +108,7 @@ // overwritten with an overflow next pointer after the object is // forwarded. Klass* objK = obj->klass(); - markOop m = obj->mark(); + markOop m = obj->mark_raw(); oop new_obj; if (m->is_marked()) { // Contains forwarding pointer. new_obj = ParNewGeneration::real_forwardee(obj); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/cms/promotionInfo.cpp --- a/src/hotspot/share/gc/cms/promotionInfo.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/cms/promotionInfo.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2010, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -85,10 +85,10 @@ } \ if (curObj->hasDisplacedMark()) { \ /* restore displaced header */ \ - oop(curObj)->set_mark(nextDisplacedHeader()); \ + oop(curObj)->set_mark_raw(nextDisplacedHeader()); \ } else { \ /* restore prototypical header */ \ - oop(curObj)->init_mark(); \ + oop(curObj)->init_mark_raw(); \ } \ /* The "promoted_mark" should now not be set */ \ assert(!curObj->hasPromotedMark(), \ @@ -147,7 +147,7 @@ void PromotionInfo::track(PromotedObject* trackOop, Klass* klassOfOop) { // make a copy of header as it may need to be spooled - markOop mark = oop(trackOop)->mark(); + markOop mark = oop(trackOop)->mark_raw(); trackOop->clear_next(); if (mark->must_be_preserved_for_cms_scavenge(klassOfOop)) { // save non-prototypical header, and mark oop @@ -287,7 +287,7 @@ // 2. each promoted object lies in this space debug_only( PromotedObject* junk = NULL; - assert(junk->next_addr() == (void*)(oop(junk)->mark_addr()), + assert(junk->next_addr() == (void*)(oop(junk)->mark_addr_raw()), "Offset of PromotedObject::_next is expected to align with " " the OopDesc::_mark within OopDesc"); ) diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1EvacFailure.cpp --- a/src/hotspot/share/gc/g1/g1EvacFailure.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1EvacFailure.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -36,6 +36,7 @@ #include "gc/shared/preservedMarks.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" class UpdateRSetDeferred : public ExtendedOopClosure { private: diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/heapRegion.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" #include "logging/log.hpp" +#include "oops/oop.inline.hpp" #include "utilities/ticks.inline.hpp" class G1ResetHumongousClosure : public HeapRegionClosure { @@ -47,7 +48,7 @@ if (_bitmap->is_marked(obj)) { // Clear bitmap and fix mark word. _bitmap->clear(obj); - obj->init_mark(); + obj->init_mark_raw(); } else { assert(current->is_empty(), "Should have been cleared in phase 2."); } @@ -70,7 +71,7 @@ HeapWord* obj_addr = (HeapWord*) obj; assert(obj_addr != destination, "everything in this pass should be moving"); Copy::aligned_conjoint_words(obj_addr, destination, size); - oop(destination)->init_mark(); + oop(destination)->init_mark_raw(); assert(oop(destination)->klass() != NULL, "should have a class"); return size; diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -112,15 +112,15 @@ // with BiasedLocking, in this case forwardee() will return NULL // even if the mark-word is used. This is no problem since // forwardee() will return NULL in the compaction phase as well. - object->init_mark(); + object->init_mark_raw(); } else { // Make sure object has the correct mark-word set or that it will be // fixed when restoring the preserved marks. - assert(object->mark() == markOopDesc::prototype_for_object(object) || // Correct mark - object->mark()->must_be_preserved(object) || // Will be restored by PreservedMarksSet - (UseBiasedLocking && object->has_bias_pattern()), // Will be restored by BiasedLocking + assert(object->mark_raw() == markOopDesc::prototype_for_object(object) || // Correct mark + object->mark_raw()->must_be_preserved(object) || // Will be restored by PreservedMarksSet + (UseBiasedLocking && object->has_bias_pattern_raw()), // Will be restored by BiasedLocking "should have correct prototype obj: " PTR_FORMAT " mark: " PTR_FORMAT " prototype: " PTR_FORMAT, - p2i(object), p2i(object->mark()), p2i(markOopDesc::prototype_for_object(object))); + p2i(object), p2i(object->mark_raw()), p2i(markOopDesc::prototype_for_object(object))); } assert(object->forwardee() == NULL, "should be forwarded to NULL"); } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp --- a/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCMarker.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,7 @@ #include "gc/shared/preservedMarks.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" #include "utilities/debug.hpp" inline bool G1FullGCMarker::mark_object(oop obj) { @@ -48,7 +49,7 @@ } // Marked by us, preserve if needed. - markOop mark = obj->mark(); + markOop mark = obj->mark_raw(); if (mark->must_be_preserved(obj) && !G1ArchiveAllocator::is_open_archive_object(obj)) { preserved_stack()->push(obj, mark); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -30,6 +30,7 @@ #include "logging/logStream.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" void G1MarkAndPushClosure::do_oop(oop* p) { do_oop_nv(p); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp --- a/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCOopClosures.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -33,6 +33,7 @@ #include "memory/iterator.inline.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" template inline void G1MarkAndPushClosure::do_oop_nv(T* p) { @@ -67,11 +68,11 @@ oop forwardee = obj->forwardee(); if (forwardee == NULL) { // Not forwarded, return current reference. - assert(obj->mark() == markOopDesc::prototype_for_object(obj) || // Correct mark - obj->mark()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet - (UseBiasedLocking && obj->has_bias_pattern()), // Will be restored by BiasedLocking + assert(obj->mark_raw() == markOopDesc::prototype_for_object(obj) || // Correct mark + obj->mark_raw()->must_be_preserved(obj) || // Will be restored by PreservedMarksSet + (UseBiasedLocking && obj->has_bias_pattern_raw()), // Will be restored by BiasedLocking "Must have correct prototype or be preserved, obj: " PTR_FORMAT ", mark: " PTR_FORMAT ", prototype: " PTR_FORMAT, - p2i(obj), p2i(obj->mark()), p2i(markOopDesc::prototype_for_object(obj))); + p2i(obj), p2i(obj->mark_raw()), p2i(markOopDesc::prototype_for_object(obj))); return; } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -35,6 +35,7 @@ #include "gc/shared/gcTraceTime.inline.hpp" #include "gc/shared/referenceProcessor.hpp" #include "logging/log.hpp" +#include "oops/oop.inline.hpp" #include "utilities/ticks.inline.hpp" bool G1FullGCPrepareTask::G1CalculatePointersClosure::do_heap_region(HeapRegion* hr) { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1OopClosures.inline.hpp --- a/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1OopClosures.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -36,6 +36,7 @@ #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oopsHierarchy.hpp" +#include "oops/oop.inline.hpp" #include "runtime/prefetch.inline.hpp" template @@ -45,8 +46,8 @@ // stall. We'll try to prefetch the object (for write, given that // we might need to install the forwarding reference) and we'll // get back to it when pop it from the queue - Prefetch::write(obj->mark_addr(), 0); - Prefetch::read(obj->mark_addr(), (HeapWordSize*2)); + Prefetch::write(obj->mark_addr_raw(), 0); + Prefetch::read(obj->mark_addr_raw(), (HeapWordSize*2)); // slightly paranoid test; I'm trying to catch potential // problems before we go into push_on_queue to know where the @@ -240,7 +241,7 @@ const InCSetState state = _g1->in_cset_state(obj); if (state.is_in_cset()) { oop forwardee; - markOop m = obj->mark(); + markOop m = obj->mark_raw(); if (m->is_marked()) { forwardee = (oop) m->decode_pointer(); } else { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1ParScanThreadState.cpp --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -282,15 +282,15 @@ // In this case, we have to install the mark word first, // otherwise obj looks to be forwarded (the old mark word, // which contains the forward pointer, was copied) - obj->set_mark(old_mark); + obj->set_mark_raw(old_mark); markOop new_mark = old_mark->displaced_mark_helper()->set_age(age); old_mark->set_displaced_mark_helper(new_mark); } else { - obj->set_mark(old_mark->set_age(age)); + obj->set_mark_raw(old_mark->set_age(age)); } _age_table.add(age, word_sz); } else { - obj->set_mark(old_mark); + obj->set_mark_raw(old_mark); } if (G1StringDedup::is_enabled()) { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp --- a/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1ParScanThreadState.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -40,7 +40,7 @@ // processed multiple times. So redo this check. const InCSetState in_cset_state = _g1h->in_cset_state(obj); if (in_cset_state.is_in_cset()) { - markOop m = obj->mark(); + markOop m = obj->mark_raw(); if (m->is_marked()) { obj = (oop) m->decode_pointer(); } else { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/g1/g1StringDedup.cpp --- a/src/hotspot/share/gc/g1/g1StringDedup.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/g1/g1StringDedup.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ #include "gc/g1/g1StringDedupStat.hpp" #include "gc/g1/g1StringDedupTable.hpp" #include "gc/g1/g1StringDedupThread.hpp" +#include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" bool G1StringDedup::_enabled = false; diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp --- a/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/parallel/psMarkSweepDecorator.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -112,8 +112,8 @@ const intx interval = PrefetchScanIntervalInBytes; while (q < t) { - assert(oop(q)->mark()->is_marked() || oop(q)->mark()->is_unlocked() || - oop(q)->mark()->has_bias_pattern(), + assert(oop(q)->mark_raw()->is_marked() || oop(q)->mark_raw()->is_unlocked() || + oop(q)->mark_raw()->has_bias_pattern(), "these are the only valid states during a mark sweep"); if (oop(q)->is_gc_marked()) { /* prefetch beyond q */ @@ -150,7 +150,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - oop(q)->init_mark(); + oop(q)->init_mark_raw(); assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); } @@ -210,7 +210,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - oop(q)->init_mark(); + oop(q)->init_mark_raw(); assert(oop(q)->forwardee() == NULL, "should be forwarded to NULL"); } @@ -258,7 +258,7 @@ if (allowed_deadspace_words >= deadlength) { allowed_deadspace_words -= deadlength; CollectedHeap::fill_with_object(q, deadlength); - oop(q)->set_mark(oop(q)->mark()->set_marked()); + oop(q)->set_mark_raw(oop(q)->mark_raw()->set_marked()); assert((int) deadlength == oop(q)->size(), "bad filler object size"); // Recall that we required "q == compaction_top". return true; @@ -349,7 +349,7 @@ q = t; } else { // $$$ Funky - q = (HeapWord*) oop(_first_dead)->mark()->decode_pointer(); + q = (HeapWord*) oop(_first_dead)->mark_raw()->decode_pointer(); } } @@ -360,7 +360,7 @@ if (!oop(q)->is_gc_marked()) { // mark is pointer to next marked oop debug_only(prev_q = q); - q = (HeapWord*) oop(q)->mark()->decode_pointer(); + q = (HeapWord*) oop(q)->mark_raw()->decode_pointer(); assert(q > prev_q, "we should be moving forward through memory"); } else { // prefetch beyond q @@ -376,7 +376,7 @@ // copy object and reinit its mark assert(q != compaction_top, "everything in this pass should be moving"); Copy::aligned_conjoint_words(q, compaction_top, size); - oop(compaction_top)->init_mark(); + oop(compaction_top)->init_mark_raw(); assert(oop(compaction_top)->klass() != NULL, "should have a class"); debug_only(prev_q = q); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/parallel/psPromotionLAB.cpp --- a/src/hotspot/share/gc/parallel/psPromotionLAB.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/parallel/psPromotionLAB.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -82,7 +82,7 @@ // so they can always fill with an array. HeapWord* tlab_end = end() + filler_header_size; typeArrayOop filler_oop = (typeArrayOop) top(); - filler_oop->set_mark(markOopDesc::prototype()); + filler_oop->set_mark_raw(markOopDesc::prototype()); filler_oop->set_klass(Universe::intArrayKlassObj()); const size_t array_length = pointer_delta(tlab_end, top()) - typeArrayOopDesc::header_size(T_INT); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp --- a/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2015, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -116,7 +116,7 @@ // NOTE! We must be very careful with any methods that access the mark // in o. There may be multiple threads racing on it, and it may be forwarded // at any time. Do not use oop methods for accessing the mark! - markOop test_mark = o->mark(); + markOop test_mark = o->mark_raw(); // The same test as "o->is_forwarded()" if (!test_mark->is_marked()) { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/parallel/psScavenge.inline.hpp --- a/src/hotspot/share/gc/parallel/psScavenge.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/parallel/psScavenge.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -32,6 +32,7 @@ #include "memory/iterator.hpp" #include "memory/resourceArea.hpp" #include "oops/access.inline.hpp" +#include "oops/oop.inline.hpp" #include "utilities/globalDefinitions.hpp" inline void PSScavenge::save_to_space_top_before_gc() { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/serial/defNewGeneration.cpp --- a/src/hotspot/share/gc/serial/defNewGeneration.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/serial/defNewGeneration.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -753,7 +753,7 @@ _promotion_failed = true; _promotion_failed_info.register_copy_failure(old->size()); - _preserved_marks_set.get()->push_if_necessary(old, old->mark()); + _preserved_marks_set.get()->push_if_necessary(old, old->mark_raw()); // forward to self old->forward_to(old); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/serial/markSweep.cpp --- a/src/hotspot/share/gc/serial/markSweep.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/serial/markSweep.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -66,8 +66,8 @@ inline void MarkSweep::mark_object(oop obj) { // some marks may contain information we need to preserve so we store them away // and overwrite the mark. We'll restore it at the end of markSweep. - markOop mark = obj->mark(); - obj->set_mark(markOopDesc::prototype()->set_marked()); + markOop mark = obj->mark_raw(); + obj->set_mark_raw(markOopDesc::prototype()->set_marked()); if (mark->must_be_preserved(obj)) { preserve_mark(obj, mark); @@ -78,7 +78,7 @@ T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { oop obj = CompressedOops::decode_not_null(heap_oop); - if (!obj->mark()->is_marked()) { + if (!obj->mark_raw()->is_marked()) { mark_object(obj); _marking_stack.push(obj); } @@ -174,7 +174,7 @@ T heap_oop = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(heap_oop)) { oop obj = CompressedOops::decode_not_null(heap_oop); - if (!obj->mark()->is_marked()) { + if (!obj->mark_raw()->is_marked()) { mark_object(obj); follow_object(obj); } @@ -190,7 +190,7 @@ } void PreservedMark::restore() { - _obj->set_mark(_mark); + _obj->set_mark_raw(_mark); } // We preserve the mark which should be replaced at the end and the location @@ -252,7 +252,7 @@ while (!_preserved_oop_stack.is_empty()) { oop obj = _preserved_oop_stack.pop(); markOop mark = _preserved_mark_stack.pop(); - obj->set_mark(mark); + obj->set_mark_raw(mark); } } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/serial/markSweep.inline.hpp --- a/src/hotspot/share/gc/serial/markSweep.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/serial/markSweep.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,11 +43,11 @@ oop obj = CompressedOops::decode_not_null(heap_oop); assert(Universe::heap()->is_in(obj), "should be in heap"); - oop new_obj = oop(obj->mark()->decode_pointer()); + oop new_obj = oop(obj->mark_raw()->decode_pointer()); assert(new_obj != NULL || // is forwarding ptr? - obj->mark() == markOopDesc::prototype() || // not gc marked? - (UseBiasedLocking && obj->mark()->has_bias_pattern()), + obj->mark_raw() == markOopDesc::prototype() || // not gc marked? + (UseBiasedLocking && obj->mark_raw()->has_bias_pattern()), // not gc marked? "should be forwarded"); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/collectedHeap.inline.hpp --- a/src/hotspot/share/gc/shared/collectedHeap.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/collectedHeap.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -61,10 +61,10 @@ assert(obj != NULL, "NULL object pointer"); if (UseBiasedLocking && (klass != NULL)) { - obj->set_mark(klass->prototype_header()); + obj->set_mark_raw(klass->prototype_header()); } else { // May be bootstrapping - obj->set_mark(markOopDesc::prototype()); + obj->set_mark_raw(markOopDesc::prototype()); } } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/genOopClosures.inline.hpp --- a/src/hotspot/share/gc/shared/genOopClosures.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/genOopClosures.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,7 @@ #include "gc/shared/space.hpp" #include "oops/access.inline.hpp" #include "oops/compressedOops.inline.hpp" +#include "oops/oop.inline.hpp" inline OopsInGenClosure::OopsInGenClosure(Generation* gen) : ExtendedOopClosure(gen->ref_processor()), _orig_gen(gen), _rs(NULL) { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/preservedMarks.cpp --- a/src/hotspot/share/gc/shared/preservedMarks.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/preservedMarks.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,6 +27,7 @@ #include "gc/shared/workgroup.hpp" #include "memory/allocation.inline.hpp" #include "memory/resourceArea.hpp" +#include "oops/oop.inline.hpp" #include "utilities/macros.hpp" void PreservedMarks::restore() { diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/preservedMarks.hpp --- a/src/hotspot/share/gc/shared/preservedMarks.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/preservedMarks.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -44,7 +44,7 @@ OopAndMarkOop(oop obj, markOop m) : _o(obj), _m(m) { } oop get_oop() { return _o; } - void set_mark() const { _o->set_mark(_m); } + inline void set_mark() const; void set_oop(oop obj) { _o = obj; } }; typedef Stack OopAndMarkOopStack; diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/preservedMarks.inline.hpp --- a/src/hotspot/share/gc/shared/preservedMarks.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/preservedMarks.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2016, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,7 +46,7 @@ } inline void PreservedMarks::init_forwarded_mark(oop obj) { - obj->init_mark(); + obj->init_mark_raw(); } inline void PreservedMarksSet::restore(RestorePreservedMarksTaskExecutor* executor) { @@ -78,4 +78,8 @@ // cache size to 0. 0 /* max_cache_size */) { } +void PreservedMarks::OopAndMarkOop::set_mark() const { + _o->set_mark_raw(_m); +} + #endif // SHARE_VM_GC_SHARED_PRESERVEDMARKS_INLINE_HPP diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/space.cpp --- a/src/hotspot/share/gc/shared/space.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/space.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -397,7 +397,7 @@ } else { // if the object isn't moving we can just set the mark to the default // mark and handle it specially later on. - q->init_mark(); + q->init_mark_raw(); assert(q->forwardee() == NULL, "should be forwarded to NULL"); } @@ -695,14 +695,14 @@ // allocate uninitialized int array typeArrayOop t = (typeArrayOop) allocate(size); assert(t != NULL, "allocation should succeed"); - t->set_mark(markOopDesc::prototype()); + t->set_mark_raw(markOopDesc::prototype()); t->set_klass(Universe::intArrayKlassObj()); t->set_length((int)length); } else { assert(size == CollectedHeap::min_fill_size(), "size for smallest fake object doesn't match"); instanceOop obj = (instanceOop) allocate(size); - obj->set_mark(markOopDesc::prototype()); + obj->set_mark_raw(markOopDesc::prototype()); obj->set_klass_gap(0); obj->set_klass(SystemDictionary::Object_klass()); } diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/gc/shared/space.inline.hpp --- a/src/hotspot/share/gc/shared/space.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/gc/shared/space.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/shared/spaceDecorator.hpp" #include "memory/universe.hpp" #include "oops/oopsHierarchy.hpp" +#include "oops/oop.inline.hpp" #include "runtime/prefetch.inline.hpp" #include "runtime/safepoint.hpp" @@ -112,7 +113,7 @@ _allowed_deadspace_words -= dead_length; CollectedHeap::fill_with_object(dead_start, dead_length); oop obj = oop(dead_start); - obj->set_mark(obj->mark()->set_marked()); + obj->set_mark_raw(obj->mark_raw()->set_marked()); assert(dead_length == (size_t)obj->size(), "bad filler object size"); log_develop_trace(gc, compaction)("Inserting object to dead space: " PTR_FORMAT ", " PTR_FORMAT ", " SIZE_FORMAT "b", @@ -159,8 +160,8 @@ while (cur_obj < scan_limit) { assert(!space->scanned_block_is_obj(cur_obj) || - oop(cur_obj)->mark()->is_marked() || oop(cur_obj)->mark()->is_unlocked() || - oop(cur_obj)->mark()->has_bias_pattern(), + oop(cur_obj)->mark_raw()->is_marked() || oop(cur_obj)->mark_raw()->is_unlocked() || + oop(cur_obj)->mark_raw()->has_bias_pattern(), "these are the only valid states during a mark sweep"); if (space->scanned_block_is_obj(cur_obj) && oop(cur_obj)->is_gc_marked()) { // prefetch beyond cur_obj @@ -335,7 +336,7 @@ // copy object and reinit its mark assert(cur_obj != compaction_top, "everything in this pass should be moving"); Copy::aligned_conjoint_words(cur_obj, compaction_top, size); - oop(compaction_top)->init_mark(); + oop(compaction_top)->init_mark_raw(); assert(oop(compaction_top)->klass() != NULL, "should have a class"); debug_only(prev_obj = cur_obj); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/interpreter/bytecodeInterpreter.cpp --- a/src/hotspot/share/interpreter/bytecodeInterpreter.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/interpreter/bytecodeInterpreter.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -689,7 +689,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, rcvr->mark_addr(), mark) == mark) { + if (rcvr->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -699,7 +699,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), mark) == mark) { + if (rcvr->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -718,7 +718,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // Debugging hint. DEBUG_ONLY(mon->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, rcvr->mark_addr(), header) == header) { + if (rcvr->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -734,7 +734,7 @@ markOop displaced = rcvr->mark()->set_unlocked(); mon->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)mon, rcvr->mark_addr(), displaced) != displaced) { + if (call_vm || rcvr->cas_set_mark((markOop)mon, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { mon->lock()->set_displaced_header(NULL); @@ -875,7 +875,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) { (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -886,7 +886,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -904,7 +904,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // debugging hint DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { + if (lockee->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) { (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -920,7 +920,7 @@ markOop displaced = lockee->mark()->set_unlocked(); entry->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { + if (call_vm || lockee->cas_set_mark((markOop)entry, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { entry->lock()->set_displaced_header(NULL); @@ -1816,7 +1816,7 @@ if (hash != markOopDesc::no_hash) { header = header->copy_set_hash(hash); } - if (Atomic::cmpxchg(header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(header, mark) == mark) { if (PrintBiasedLockingStatistics) (*BiasedLocking::revoked_lock_entry_count_addr())++; } @@ -1827,7 +1827,7 @@ if (hash != markOopDesc::no_hash) { new_header = new_header->copy_set_hash(hash); } - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), mark) == mark) { + if (lockee->cas_set_mark(new_header, mark) == mark) { if (PrintBiasedLockingStatistics) (* BiasedLocking::rebiased_lock_entry_count_addr())++; } @@ -1847,7 +1847,7 @@ markOop new_header = (markOop) ((uintptr_t) header | thread_ident); // debugging hint DEBUG_ONLY(entry->lock()->set_displaced_header((markOop) (uintptr_t) 0xdeaddead);) - if (Atomic::cmpxchg(new_header, lockee->mark_addr(), header) == header) { + if (lockee->cas_set_mark(new_header, header) == header) { if (PrintBiasedLockingStatistics) (* BiasedLocking::anonymously_biased_lock_entry_count_addr())++; } @@ -1863,7 +1863,7 @@ markOop displaced = lockee->mark()->set_unlocked(); entry->lock()->set_displaced_header(displaced); bool call_vm = UseHeavyMonitors; - if (call_vm || Atomic::cmpxchg((markOop)entry, lockee->mark_addr(), displaced) != displaced) { + if (call_vm || lockee->cas_set_mark((markOop)entry, displaced) != displaced) { // Is it simple recursive case? if (!call_vm && THREAD->is_lock_owned((address) displaced->clear_lock_bits())) { entry->lock()->set_displaced_header(NULL); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/oops/accessBackend.inline.hpp --- a/src/hotspot/share/oops/accessBackend.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/oops/accessBackend.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -316,7 +316,7 @@ reinterpret_cast((oopDesc*)dst), align_object_size(size) / HeapWordsPerLong); // Clear the header - dst->init_mark(); + dst->init_mark_raw(); } #endif // SHARE_VM_RUNTIME_ACCESSBACKEND_INLINE_HPP diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/oops/oop.cpp --- a/src/hotspot/share/oops/oop.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/oops/oop.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -134,7 +134,7 @@ if (ignore_mark_word) { return true; } - if (obj->mark() != NULL) { + if (obj->mark_raw() != NULL) { return true; } return !SafepointSynchronize::is_at_safepoint(); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/oops/oop.hpp --- a/src/hotspot/share/oops/oop.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/oops/oop.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -63,17 +63,21 @@ } _metadata; public: - markOop mark() const { return _mark; } - markOop* mark_addr() const { return (markOop*) &_mark; } + inline markOop mark() const; + inline markOop mark_raw() const; + inline markOop* mark_addr_raw() const; - void set_mark(volatile markOop m) { _mark = m; } + inline void set_mark(volatile markOop m); + inline void set_mark_raw(volatile markOop m); inline void release_set_mark(markOop m); inline markOop cas_set_mark(markOop new_mark, markOop old_mark); + inline markOop cas_set_mark_raw(markOop new_mark, markOop old_mark); // Used only to re-initialize the mark word (e.g., of promoted // objects during a GC) -- requires a valid klass pointer inline void init_mark(); + inline void init_mark_raw(); inline Klass* klass() const; inline Klass* klass_or_null() const volatile; @@ -237,6 +241,7 @@ inline bool is_locked() const; inline bool is_unlocked() const; inline bool has_bias_pattern() const; + inline bool has_bias_pattern_raw() const; // asserts and guarantees static bool is_oop(oop obj, bool ignore_mark_word = false); @@ -323,9 +328,9 @@ unsigned int new_hash(juint seed); // marks are forwarded to stack when object is locked - inline bool has_displaced_mark() const; - inline markOop displaced_mark() const; - inline void set_displaced_mark(markOop m); + inline bool has_displaced_mark_raw() const; + inline markOop displaced_mark_raw() const; + inline void set_displaced_mark_raw(markOop m); static bool has_klass_gap(); diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/oops/oop.inline.hpp --- a/src/hotspot/share/oops/oop.inline.hpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/oops/oop.inline.hpp Thu Apr 05 10:54:53 2018 +0200 @@ -44,11 +44,35 @@ // Implementation of all inlined member functions defined in oop.hpp // We need a separate file to avoid circular references +markOop oopDesc::mark() const { + return HeapAccess::load_at(as_oop(), mark_offset_in_bytes()); +} + +markOop oopDesc::mark_raw() const { + return _mark; +} + +markOop* oopDesc::mark_addr_raw() const { + return (markOop*) &_mark; +} + +void oopDesc::set_mark(volatile markOop m) { + HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m); +} + +void oopDesc::set_mark_raw(volatile markOop m) { + _mark = m; +} + void oopDesc::release_set_mark(markOop m) { - OrderAccess::release_store(&_mark, m); + HeapAccess::store_at(as_oop(), mark_offset_in_bytes(), m); } markOop oopDesc::cas_set_mark(markOop new_mark, markOop old_mark) { + return HeapAccess<>::atomic_cmpxchg_at(new_mark, as_oop(), mark_offset_in_bytes(), old_mark); +} + +markOop oopDesc::cas_set_mark_raw(markOop new_mark, markOop old_mark) { return Atomic::cmpxchg(new_mark, &_mark, old_mark); } @@ -56,6 +80,10 @@ set_mark(markOopDesc::prototype_for_object(this)); } +void oopDesc::init_mark_raw() { + set_mark_raw(markOopDesc::prototype_for_object(this)); +} + Klass* oopDesc::klass() const { if (UseCompressedClassPointers) { return Klass::decode_klass_not_null(_metadata._compressed_klass); @@ -281,16 +309,20 @@ return mark()->has_bias_pattern(); } +bool oopDesc::has_bias_pattern_raw() const { + return mark_raw()->has_bias_pattern(); +} + // Used only for markSweep, scavenging bool oopDesc::is_gc_marked() const { - return mark()->is_marked(); + return mark_raw()->is_marked(); } // Used by scavengers bool oopDesc::is_forwarded() const { // The extra heap check is needed since the obj might be locked, in which case the // mark would point to a stack location and have the sentinel bit cleared - return mark()->is_marked(); + return mark_raw()->is_marked(); } // Used by scavengers @@ -304,7 +336,7 @@ "forwarding archive object"); markOop m = markOopDesc::encode_pointer_as_mark(p); assert(m->decode_pointer() == p, "encoding must be reversable"); - set_mark(m); + set_mark_raw(m); } // Used by parallel scavengers @@ -315,12 +347,12 @@ "forwarding to something not in heap"); markOop m = markOopDesc::encode_pointer_as_mark(p); assert(m->decode_pointer() == p, "encoding must be reversable"); - return cas_set_mark(m, compare) == compare; + return cas_set_mark_raw(m, compare) == compare; } #if INCLUDE_ALL_GCS oop oopDesc::forward_to_atomic(oop p) { - markOop oldMark = mark(); + markOop oldMark = mark_raw(); markOop forwardPtrMark = markOopDesc::encode_pointer_as_mark(p); markOop curMark; @@ -328,7 +360,7 @@ assert(sizeof(markOop) == sizeof(intptr_t), "CAS below requires this."); while (!oldMark->is_marked()) { - curMark = Atomic::cmpxchg(forwardPtrMark, &_mark, oldMark); + curMark = cas_set_mark_raw(forwardPtrMark, oldMark); assert(is_forwarded(), "object should have been forwarded"); if (curMark == oldMark) { return NULL; @@ -346,25 +378,25 @@ // The forwardee is used when copying during scavenge and mark-sweep. // It does need to clear the low two locking- and GC-related bits. oop oopDesc::forwardee() const { - return (oop) mark()->decode_pointer(); + return (oop) mark_raw()->decode_pointer(); } // The following method needs to be MT safe. uint oopDesc::age() const { assert(!is_forwarded(), "Attempt to read age from forwarded mark"); - if (has_displaced_mark()) { - return displaced_mark()->age(); + if (has_displaced_mark_raw()) { + return displaced_mark_raw()->age(); } else { - return mark()->age(); + return mark_raw()->age(); } } void oopDesc::incr_age() { assert(!is_forwarded(), "Attempt to increment age of forwarded mark"); - if (has_displaced_mark()) { - set_displaced_mark(displaced_mark()->incr_age()); + if (has_displaced_mark_raw()) { + set_displaced_mark_raw(displaced_mark_raw()->incr_age()); } else { - set_mark(mark()->incr_age()); + set_mark_raw(mark_raw()->incr_age()); } } @@ -465,16 +497,16 @@ } } -bool oopDesc::has_displaced_mark() const { - return mark()->has_displaced_mark_helper(); +bool oopDesc::has_displaced_mark_raw() const { + return mark_raw()->has_displaced_mark_helper(); } -markOop oopDesc::displaced_mark() const { - return mark()->displaced_mark_helper(); +markOop oopDesc::displaced_mark_raw() const { + return mark_raw()->displaced_mark_helper(); } -void oopDesc::set_displaced_mark(markOop m) { - mark()->set_displaced_mark_helper(m); +void oopDesc::set_displaced_mark_raw(markOop m) { + mark_raw()->set_displaced_mark_helper(m); } #endif // SHARE_VM_OOPS_OOP_INLINE_HPP diff -r ea0cc7c74e75 -r a47d1e21b3f1 src/hotspot/share/runtime/biasedLocking.cpp --- a/src/hotspot/share/runtime/biasedLocking.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/src/hotspot/share/runtime/biasedLocking.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005, 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -616,7 +616,7 @@ // with it. markOop biased_value = mark; markOop res_mark = obj->cas_set_mark(prototype_header, mark); - assert(!(*(obj->mark_addr()))->has_bias_pattern(), "even if we raced, should still be revoked"); + assert(!obj->mark()->has_bias_pattern(), "even if we raced, should still be revoked"); return BIAS_REVOKED; } else if (prototype_header->bias_epoch() != mark->bias_epoch()) { // The epoch of this biasing has expired indicating that the diff -r ea0cc7c74e75 -r a47d1e21b3f1 test/hotspot/gtest/gc/shared/test_preservedMarks.cpp --- a/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp Mon Apr 09 08:19:26 2018 -0400 +++ b/test/hotspot/gtest/gc/shared/test_preservedMarks.cpp Thu Apr 05 10:54:53 2018 +0200 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2018 Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,7 @@ #include "precompiled.hpp" #include "gc/shared/preservedMarks.inline.hpp" +#include "oops/oop.inline.hpp" #include "unittest.hpp" class ScopedDisabledBiasedLocking { @@ -38,14 +39,14 @@ oopDesc _oop; public: - FakeOop() : _oop() { _oop.set_mark(originalMark()); } + FakeOop() : _oop() { _oop.set_mark_raw(originalMark()); } oop get_oop() { return &_oop; } - markOop mark() { return _oop.mark(); } - void set_mark(markOop m) { _oop.set_mark(m); } + markOop mark() { return _oop.mark_raw(); } + void set_mark(markOop m) { _oop.set_mark_raw(m); } void forward_to(oop obj) { markOop m = markOopDesc::encode_pointer_as_mark(obj); - _oop.set_mark(m); + _oop.set_mark_raw(m); } static markOop originalMark() { return markOop(markOopDesc::lock_mask_in_place); }