80 void ZRelocate::start() { |
81 void ZRelocate::start() { |
81 ZRelocateRootsTask task; |
82 ZRelocateRootsTask task; |
82 _workers->run_parallel(&task); |
83 _workers->run_parallel(&task); |
83 } |
84 } |
84 |
85 |
85 uintptr_t ZRelocate::relocate_object_inner(ZPage* page, uintptr_t from_index, uintptr_t from_offset) const { |
86 ZForwarding* ZRelocate::forwarding_for_page(ZPage* page) const { |
86 ZForwardingTableCursor cursor; |
87 const uintptr_t addr = ZAddress::good(page->start()); |
87 |
88 return ZHeap::heap()->forwarding(addr); |
88 // Lookup address in forwarding table |
89 } |
89 const ZForwardingTableEntry entry = page->find_forwarding(from_index, &cursor); |
90 |
|
91 uintptr_t ZRelocate::relocate_object_inner(ZForwarding* forwarding, uintptr_t from_index, uintptr_t from_offset) const { |
|
92 ZForwardingCursor cursor; |
|
93 |
|
94 // Lookup forwarding entry |
|
95 const ZForwardingEntry entry = forwarding->find(from_index, &cursor); |
90 if (entry.from_index() == from_index) { |
96 if (entry.from_index() == from_index) { |
91 // Already relocated, return new address |
97 // Already relocated, return new address |
92 return entry.to_offset(); |
98 return entry.to_offset(); |
93 } |
99 } |
94 |
100 |
95 assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live"); |
101 assert(ZHeap::heap()->is_object_live(ZAddress::good(from_offset)), "Should be live"); |
96 |
102 |
97 if (page->is_pinned()) { |
103 if (forwarding->is_pinned()) { |
98 // In-place forward |
104 // In-place forward |
99 return page->insert_forwarding(from_index, from_offset, &cursor); |
105 return forwarding->insert(from_index, from_offset, &cursor); |
100 } |
106 } |
101 |
107 |
102 // Allocate object |
108 // Allocate object |
103 const uintptr_t from_good = ZAddress::good(from_offset); |
109 const uintptr_t from_good = ZAddress::good(from_offset); |
104 const size_t size = ZUtils::object_size(from_good); |
110 const size_t size = ZUtils::object_size(from_good); |
105 const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size); |
111 const uintptr_t to_good = ZHeap::heap()->alloc_object_for_relocation(size); |
106 if (to_good == 0) { |
112 if (to_good == 0) { |
107 // Failed, in-place forward |
113 // Failed, in-place forward |
108 return page->insert_forwarding(from_index, from_offset, &cursor); |
114 return forwarding->insert(from_index, from_offset, &cursor); |
109 } |
115 } |
110 |
116 |
111 // Copy object |
117 // Copy object |
112 ZUtils::object_copy(from_good, to_good, size); |
118 ZUtils::object_copy(from_good, to_good, size); |
113 |
119 |
114 // Update forwarding table |
120 // Insert forwarding entry |
115 const uintptr_t to_offset = ZAddress::offset(to_good); |
121 const uintptr_t to_offset = ZAddress::offset(to_good); |
116 const uintptr_t to_offset_final = page->insert_forwarding(from_index, to_offset, &cursor); |
122 const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, &cursor); |
117 if (to_offset_final == to_offset) { |
123 if (to_offset_final == to_offset) { |
118 // Relocation succeeded |
124 // Relocation succeeded |
119 return to_offset; |
125 return to_offset; |
120 } |
126 } |
121 |
127 |
122 // Relocation contention |
128 // Relocation contention |
123 ZStatInc(ZCounterRelocationContention); |
129 ZStatInc(ZCounterRelocationContention); |
124 log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT |
130 log_trace(gc)("Relocation contention, thread: " PTR_FORMAT " (%s), forwarding: " PTR_FORMAT |
125 ", entry: " SIZE_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT, |
131 ", entry: " UINT32_FORMAT ", oop: " PTR_FORMAT ", size: " SIZE_FORMAT, |
126 ZThread::id(), ZThread::name(), p2i(this), cursor, from_good, size); |
132 ZThread::id(), ZThread::name(), p2i(forwarding), cursor, from_good, size); |
127 |
133 |
128 // Try undo allocation |
134 // Try undo allocation |
129 ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size); |
135 ZHeap::heap()->undo_alloc_object_for_relocation(to_good, size); |
130 |
136 |
131 return to_offset_final; |
137 return to_offset_final; |
132 } |
138 } |
133 |
139 |
134 uintptr_t ZRelocate::relocate_object(ZPage* page, uintptr_t from_addr) const { |
140 uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const { |
135 assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocating"); |
|
136 |
|
137 const uintptr_t from_offset = ZAddress::offset(from_addr); |
141 const uintptr_t from_offset = ZAddress::offset(from_addr); |
138 const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift(); |
142 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift(); |
139 const uintptr_t to_offset = relocate_object_inner(page, from_index, from_offset); |
143 const uintptr_t to_offset = relocate_object_inner(forwarding, from_index, from_offset); |
|
144 |
140 if (from_offset == to_offset) { |
145 if (from_offset == to_offset) { |
141 // In-place forwarding, pin page |
146 // In-place forwarding, pin page |
142 page->set_pinned(); |
147 forwarding->set_pinned(); |
143 } |
148 } |
144 |
149 |
145 return ZAddress::good(to_offset); |
150 return ZAddress::good(to_offset); |
146 } |
151 } |
147 |
152 |
148 uintptr_t ZRelocate::forward_object(ZPage* page, uintptr_t from_addr) const { |
153 uintptr_t ZRelocate::forward_object(ZForwarding* forwarding, uintptr_t from_addr) const { |
149 assert(ZHeap::heap()->is_relocating(from_addr), "Should be relocated"); |
|
150 |
|
151 // Lookup address in forwarding table |
|
152 const uintptr_t from_offset = ZAddress::offset(from_addr); |
154 const uintptr_t from_offset = ZAddress::offset(from_addr); |
153 const uintptr_t from_index = (from_offset - page->start()) >> page->object_alignment_shift(); |
155 const uintptr_t from_index = (from_offset - forwarding->start()) >> forwarding->object_alignment_shift(); |
154 const ZForwardingTableEntry entry = page->find_forwarding(from_index); |
156 const ZForwardingEntry entry = forwarding->find(from_index); |
|
157 |
155 assert(entry.from_index() == from_index, "Should be forwarded"); |
158 assert(entry.from_index() == from_index, "Should be forwarded"); |
156 |
|
157 return ZAddress::good(entry.to_offset()); |
159 return ZAddress::good(entry.to_offset()); |
158 } |
160 } |
159 |
161 |
160 class ZRelocateObjectClosure : public ObjectClosure { |
162 class ZRelocateObjectClosure : public ObjectClosure { |
161 private: |
163 private: |
162 ZRelocate* const _relocate; |
164 ZRelocate* const _relocate; |
163 ZPage* const _page; |
165 ZForwarding* const _forwarding; |
164 |
166 |
165 public: |
167 public: |
166 ZRelocateObjectClosure(ZRelocate* relocate, ZPage* page) : |
168 ZRelocateObjectClosure(ZRelocate* relocate, ZForwarding* forwarding) : |
167 _relocate(relocate), |
169 _relocate(relocate), |
168 _page(page) {} |
170 _forwarding(forwarding) {} |
169 |
171 |
170 virtual void do_object(oop o) { |
172 virtual void do_object(oop o) { |
171 _relocate->relocate_object(_page, ZOop::to_address(o)); |
173 _relocate->relocate_object(_forwarding, ZOop::to_address(o)); |
172 } |
174 } |
173 }; |
175 }; |
174 |
176 |
175 bool ZRelocate::work(ZRelocationSetParallelIterator* iter) { |
177 bool ZRelocate::work(ZRelocationSetParallelIterator* iter) { |
176 bool success = true; |
178 bool success = true; |
177 |
179 |
178 // Relocate pages in the relocation set |
180 // Relocate pages in the relocation set |
179 for (ZPage* page; iter->next(&page);) { |
181 for (ZPage* page; iter->next(&page);) { |
180 // Relocate objects in page |
182 // Relocate objects in page |
181 ZRelocateObjectClosure cl(this, page); |
183 ZForwarding* const forwarding = forwarding_for_page(page); |
|
184 ZRelocateObjectClosure cl(this, forwarding); |
182 page->object_iterate(&cl); |
185 page->object_iterate(&cl); |
183 |
186 |
184 if (ZVerifyForwarding) { |
187 if (ZVerifyForwarding) { |
185 page->verify_forwarding(); |
188 forwarding->verify(page->object_max_count(), page->live_objects()); |
186 } |
189 } |
187 |
190 |
188 if (page->is_pinned()) { |
191 if (forwarding->is_pinned()) { |
189 // Relocation failed, page is now pinned |
192 // Relocation failed, page is now pinned |
190 success = false; |
193 success = false; |
191 } else { |
194 } else { |
192 // Relocation succeeded, release page |
195 // Relocation succeeded, release page |
193 ZHeap::heap()->release_page(page, true /* reclaimed */); |
196 if (forwarding->dec_refcount()) { |
|
197 ZHeap::heap()->free_page(page, true /* reclaimed */); |
|
198 } |
194 } |
199 } |
195 } |
200 } |
196 |
201 |
197 return success; |
202 return success; |
198 } |
203 } |