8220586: ZGC: Move relocation logic from ZPage to ZRelocate
Reviewed-by: stefank, eosterlund
--- a/src/hotspot/share/gc/z/zHeap.cpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zHeap.cpp Mon Mar 18 11:50:39 2019 +0100
@@ -497,7 +497,7 @@
assert(ZGlobalPhase == ZPhaseRelocate, "Relocate not allowed");
ZPage* const page = _pagetable.get(addr);
const bool retained = retain_page(page);
- const uintptr_t new_addr = page->relocate_object(addr);
+ const uintptr_t new_addr = _relocate.relocate_object(page, addr);
if (retained) {
release_page(page, true /* reclaimed */);
}
@@ -509,7 +509,7 @@
assert(ZGlobalPhase == ZPhaseMark ||
ZGlobalPhase == ZPhaseMarkCompleted, "Forward not allowed");
ZPage* const page = _pagetable.get(addr);
- return page->forward_object(addr);
+ return _relocate.forward_object(page, addr);
}
void ZHeap::relocate() {
--- a/src/hotspot/share/gc/z/zPage.cpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zPage.cpp Mon Mar 18 11:50:39 2019 +0100
@@ -22,24 +22,12 @@
*/
#include "precompiled.hpp"
-#include "gc/shared/collectedHeap.hpp"
-#include "gc/z/zAddress.inline.hpp"
-#include "gc/z/zForwardingTable.inline.hpp"
-#include "gc/z/zHeap.inline.hpp"
-#include "gc/z/zLiveMap.inline.hpp"
-#include "gc/z/zMark.hpp"
#include "gc/z/zPage.inline.hpp"
#include "gc/z/zPhysicalMemory.inline.hpp"
-#include "gc/z/zStat.hpp"
-#include "gc/z/zThread.hpp"
-#include "gc/z/zUtils.inline.hpp"
-#include "logging/log.hpp"
+#include "gc/z/zVirtualMemory.inline.hpp"
#include "runtime/orderAccess.hpp"
#include "utilities/align.hpp"
#include "utilities/debug.hpp"
-#include "utilities/globalDefinitions.hpp"
-
-static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
ZPage::ZPage(uint8_t type, ZVirtualMemory vmem, ZPhysicalMemory pmem) :
_type(type),
@@ -81,82 +69,6 @@
_refcount = 1;
}
-uintptr_t ZPage::relocate_object_inner(uintptr_t from_index, uintptr_t from_offset) {
- ZForwardingTableCursor cursor;
-
- // Lookup address in forwarding table
- const ZForwardingTableEntry entry = _forwarding.find(from_index, &cursor);
- if (entry.from_index() == from_index) {
- // Already relocated, return new address
- return entry.to_offset();
- }
-
- // Not found in forwarding table, relocate object
- assert(is_object_marked(from_offset), "Should be marked");
-
- if (is_pinned()) {
- // In-place forward
- return _forwarding.insert(from_index, from_offset, &cursor);
- }
-
- // Allocate object
- const uintptr_t from_good = ZAddress::good(from_offset);
- const size_t size = ZUtils::object_size(from_good);
- const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
- if (to_good == 0) {
- // Failed, in-place forward
- return _forwarding.insert(from_index, from_offset, &cursor);
- }
-
- // Copy object
- ZUtils::object_copy(from_good, to_good, size);
-
- // Update forwarding table
- const uintptr_t to_offset = ZAddress::offset(to_good);
- const uintptr_t to_offset_final = _forwarding.insert(from_index, to_offset, &cursor);
- if (to_offset_final == to_offset) {
- // Relocation succeeded
- return to_offset;
- }
-
- // Relocation contention
- ZStatInc(ZCounterRelocationContention);
- log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT
- ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
- ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size);
-
- // Try undo allocation
- ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
-
- return to_offset_final;
-}
-
-uintptr_t ZPage::relocate_object(uintptr_t from) {
- assert(ZHeap::heap()->is_relocating(from), "Should be relocating");
-
- const uintptr_t from_offset = ZAddress::offset(from);
- const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift();
- const uintptr_t to_offset = relocate_object_inner(from_index, from_offset);
- if (from_offset == to_offset) {
- // In-place forwarding, pin page
- set_pinned();
- }
-
- return ZAddress::good(to_offset);
-}
-
-uintptr_t ZPage::forward_object(uintptr_t from) {
- assert(ZHeap::heap()->is_relocating(from), "Should be relocated");
-
- // Lookup address in forwarding table
- const uintptr_t from_offset = ZAddress::offset(from);
- const uintptr_t from_index = (from_offset - start()) >> object_alignment_shift();
- const ZForwardingTableEntry entry = _forwarding.find(from_index);
- assert(entry.from_index() == from_index, "Should be forwarded");
-
- return ZAddress::good(entry.to_offset());
-}
-
void ZPage::print_on(outputStream* out) const {
out->print_cr(" %-6s " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s%s%s%s%s%s",
type_to_string(), start(), top(), end(),
--- a/src/hotspot/share/gc/z/zPage.hpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zPage.hpp Mon Mar 18 11:50:39 2019 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,7 +53,6 @@
const char* type_to_string() const;
uint32_t object_max_count() const;
- uintptr_t relocate_object_inner(uintptr_t from_index, uintptr_t from_offset);
bool is_object_marked(uintptr_t addr) const;
bool is_object_strongly_marked(uintptr_t addr) const;
@@ -101,6 +100,9 @@
bool is_forwarding() const;
void set_forwarding();
void reset_forwarding();
+ ZForwardingTableEntry find_forwarding(uintptr_t from_index);
+ ZForwardingTableEntry find_forwarding(uintptr_t from_index, ZForwardingTableCursor* cursor);
+ uintptr_t insert_forwarding(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor);
void verify_forwarding() const;
bool is_marked() const;
@@ -119,9 +121,6 @@
bool undo_alloc_object(uintptr_t addr, size_t size);
bool undo_alloc_object_atomic(uintptr_t addr, size_t size);
- uintptr_t relocate_object(uintptr_t from);
- uintptr_t forward_object(uintptr_t from);
-
void print_on(outputStream* out) const;
void print() const;
};
--- a/src/hotspot/share/gc/z/zPage.inline.hpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zPage.inline.hpp Mon Mar 18 11:50:39 2019 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -212,6 +212,18 @@
_pinned = 0;
}
+inline ZForwardingTableEntry ZPage::find_forwarding(uintptr_t from_index) {
+ return _forwarding.find(from_index);
+}
+
+inline ZForwardingTableEntry ZPage::find_forwarding(uintptr_t from_index, ZForwardingTableCursor* cursor) {
+ return _forwarding.find(from_index, cursor);
+}
+
+inline uintptr_t ZPage::insert_forwarding(uintptr_t from_index, uintptr_t to_offset, ZForwardingTableCursor* cursor) {
+ return _forwarding.insert(from_index, to_offset, cursor);
+}
+
inline void ZPage::verify_forwarding() const {
_forwarding.verify(object_max_count(), _livemap.live_objects());
}
--- a/src/hotspot/share/gc/z/zRelocate.cpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zRelocate.cpp Mon Mar 18 11:50:39 2019 +0100
@@ -30,9 +30,13 @@
#include "gc/z/zRelocate.hpp"
#include "gc/z/zRelocationSet.inline.hpp"
#include "gc/z/zRootsIterator.hpp"
+#include "gc/z/zStat.hpp"
#include "gc/z/zTask.hpp"
#include "gc/z/zThreadLocalAllocBuffer.hpp"
#include "gc/z/zWorkers.hpp"
+#include "logging/log.hpp"
+
+static const ZStatCounter ZCounterRelocationContention("Contention", "Relocation Contention", ZStatUnitOpsPerSecond);
ZRelocate::ZRelocate(ZWorkers* workers) :
_workers(workers) {}
@@ -78,16 +82,93 @@
_workers->run_parallel(&task);
}
+uintptr_t ZRelocate::relocate_object_inner(ZPage* page, uintptr_t from_index, uintptr_t from_offset) const {
+ ZForwardingTableCursor cursor;
+
+ // Lookup address in forwarding table
+ const ZForwardingTableEntry entry = page->find_forwarding(from_index, &cursor);
+ if (entry.from_index() == from_index) {
+ // Already relocated, return new address
+ return entry.to_offset();
+ }
+
+ assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live");
+
+ if (page->is_pinned()) {
+ // In-place forward
+ return page->insert_forwarding(from_index, from_offset, &cursor);
+ }
+
+ // Allocate object
+ const uintptr_t from_good = ZAddress::good(from_offset);
+ const size_t size = ZUtils::object_size(from_good);
+ const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size);
+ if (to_good == 0) {
+ // Failed, in-place forward
+ return page->insert_forwarding(from_index, from_offset, &cursor);
+ }
+
+ // Copy object
+ ZUtils::object_copy(from_good, to_good, size);
+
+ // Update forwarding table
+ const uintptr_t to_offset = ZAddress::offset(to_good);
+ const uintptr_t to_offset_final = page->insert_forwarding(from_index, to_offset, &cursor);
+ if (to_offset_final == to_offset) {
+ // Relocation succeeded
+ return to_offset;
+ }
+
+ // Relocation contention
+ ZStatInc(ZCounterRelocationContention);
+ log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT
+ ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT,
+ ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size);
+
+ // Try undo allocation
+ ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size);
+
+ return to_offset_final;
+}
+
+uintptr_t ZRelocate::relocate_object(ZPage* page, uintptr_t from_addr) const {
+ assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocating");
+
+ const uintptr_t from_offset = ZAddress::offset(from_addr);
+ const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
+ const uintptr_t to_offset = relocate_object_inner(page, from_index, from_offset);
+ if (from_offset == to_offset) {
+ // In-place forwarding, pin page
+ page->set_pinned();
+ }
+
+ return ZAddress::good(to_offset);
+}
+
+uintptr_t ZRelocate::forward_object(ZPage* page, uintptr_t from_addr) const {
+ assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocated");
+
+ // Lookup address in forwarding table
+ const uintptr_t from_offset = ZAddress::offset(from_addr);
+ const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift();
+ const ZForwardingTableEntry entry = page->find_forwarding(from_index);
+ assert(entry.from_index() == from_index, "Should be forwarded");
+
+ return ZAddress::good(entry.to_offset());
+}
+
class ZRelocateObjectClosure : public ObjectClosure {
private:
- ZPage* const _page;
+ ZRelocate* const _relocate;
+ ZPage* const _page;
public:
- ZRelocateObjectClosure(ZPage* page) :
+ ZRelocateObjectClosure(ZRelocate* relocate, ZPage* page) :
+ _relocate(relocate),
_page(page) {}
virtual void do_object(oop o) {
- _page->relocate_object(ZOop::to_address(o));
+ _relocate->relocate_object(_page, ZOop::to_address(o));
}
};
@@ -97,7 +178,7 @@
// Relocate pages in the relocation set
for (ZPage* page; iter->next(&page);) {
// Relocate objects in page
- ZRelocateObjectClosure cl(page);
+ ZRelocateObjectClosure cl(this, page);
page->object_iterate(&cl);
if (ZVerifyForwarding) {
--- a/src/hotspot/share/gc/z/zRelocate.hpp Mon Mar 18 11:50:38 2019 +0100
+++ b/src/hotspot/share/gc/z/zRelocate.hpp Mon Mar 18 11:50:39 2019 +0100
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -34,11 +34,15 @@
private:
ZWorkers* const _workers;
+ uintptr_t relocate_object_inner(ZPage* from_page, uintptr_t from_index, uintptr_t from_offset) const;
bool work(ZRelocationSetParallelIterator* iter);
public:
ZRelocate(ZWorkers* workers);
+ uintptr_t relocate_object(ZPage* from_page, uintptr_t from_addr) const;
+ uintptr_t forward_object(ZPage* from_page, uintptr_t from_addr) const;
+
void start();
bool relocate(ZRelocationSet* relocation_set);
};