86 |
87 |
87 // HeapWords |
88 // HeapWords |
88 |
89 |
89 // Word-aligned words, conjoint, not atomic on each word |
90 // Word-aligned words, conjoint, not atomic on each word |
90 static void conjoint_words(const HeapWord* from, HeapWord* to, size_t count) { |
91 static void conjoint_words(const HeapWord* from, HeapWord* to, size_t count) { |
91 assert_params_ok(from, to, LogHeapWordSize); |
92 assert_params_ok(from, to, HeapWordSize); |
92 pd_conjoint_words(from, to, count); |
93 pd_conjoint_words(from, to, count); |
93 } |
94 } |
94 |
95 |
95 // Word-aligned words, disjoint, not atomic on each word |
96 // Word-aligned words, disjoint, not atomic on each word |
96 static void disjoint_words(const HeapWord* from, HeapWord* to, size_t count) { |
97 static void disjoint_words(const HeapWord* from, HeapWord* to, size_t count) { |
97 assert_params_ok(from, to, LogHeapWordSize); |
98 assert_params_ok(from, to, HeapWordSize); |
98 assert_disjoint(from, to, count); |
99 assert_disjoint(from, to, count); |
99 pd_disjoint_words(from, to, count); |
100 pd_disjoint_words(from, to, count); |
100 } |
101 } |
101 |
102 |
102 // Word-aligned words, disjoint, atomic on each word |
103 // Word-aligned words, disjoint, atomic on each word |
103 static void disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) { |
104 static void disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) { |
104 assert_params_ok(from, to, LogHeapWordSize); |
105 assert_params_ok(from, to, HeapWordSize); |
105 assert_disjoint(from, to, count); |
106 assert_disjoint(from, to, count); |
106 pd_disjoint_words_atomic(from, to, count); |
107 pd_disjoint_words_atomic(from, to, count); |
107 } |
108 } |
108 |
109 |
109 // Object-aligned words, conjoint, not atomic on each word |
110 // Object-aligned words, conjoint, not atomic on each word |
131 pd_conjoint_bytes(from, to, count); |
132 pd_conjoint_bytes(from, to, count); |
132 } |
133 } |
133 |
134 |
134 // jshorts, conjoint, atomic on each jshort |
135 // jshorts, conjoint, atomic on each jshort |
135 static void conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) { |
136 static void conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) { |
136 assert_params_ok(from, to, LogBytesPerShort); |
137 assert_params_ok(from, to, BytesPerShort); |
137 pd_conjoint_jshorts_atomic(from, to, count); |
138 pd_conjoint_jshorts_atomic(from, to, count); |
138 } |
139 } |
139 |
140 |
140 // jints, conjoint, atomic on each jint |
141 // jints, conjoint, atomic on each jint |
141 static void conjoint_jints_atomic(const jint* from, jint* to, size_t count) { |
142 static void conjoint_jints_atomic(const jint* from, jint* to, size_t count) { |
142 assert_params_ok(from, to, LogBytesPerInt); |
143 assert_params_ok(from, to, BytesPerInt); |
143 pd_conjoint_jints_atomic(from, to, count); |
144 pd_conjoint_jints_atomic(from, to, count); |
144 } |
145 } |
145 |
146 |
146 // jlongs, conjoint, atomic on each jlong |
147 // jlongs, conjoint, atomic on each jlong |
147 static void conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) { |
148 static void conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) { |
148 assert_params_ok(from, to, LogBytesPerLong); |
149 assert_params_ok(from, to, BytesPerLong); |
149 pd_conjoint_jlongs_atomic(from, to, count); |
150 pd_conjoint_jlongs_atomic(from, to, count); |
150 } |
151 } |
151 |
152 |
152 // oops, conjoint, atomic on each oop |
153 // oops, conjoint, atomic on each oop |
153 static void conjoint_oops_atomic(const oop* from, oop* to, size_t count) { |
154 static void conjoint_oops_atomic(const oop* from, oop* to, size_t count) { |
154 assert_params_ok(from, to, LogBytesPerHeapOop); |
155 assert_params_ok(from, to, BytesPerHeapOop); |
155 pd_conjoint_oops_atomic(from, to, count); |
156 pd_conjoint_oops_atomic(from, to, count); |
156 } |
157 } |
157 |
158 |
158 // overloaded for UseCompressedOops |
159 // overloaded for UseCompressedOops |
159 static void conjoint_oops_atomic(const narrowOop* from, narrowOop* to, size_t count) { |
160 static void conjoint_oops_atomic(const narrowOop* from, narrowOop* to, size_t count) { |
160 assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong"); |
161 assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong"); |
161 assert_params_ok(from, to, LogBytesPerInt); |
162 assert_params_ok(from, to, BytesPerInt); |
162 pd_conjoint_jints_atomic((const jint*)from, (jint*)to, count); |
163 pd_conjoint_jints_atomic((const jint*)from, (jint*)to, count); |
163 } |
164 } |
164 |
165 |
165 // Copy a span of memory. If the span is an integral number of aligned |
166 // Copy a span of memory. If the span is an integral number of aligned |
166 // longs, words, or ints, copy those units atomically. |
167 // longs, words, or ints, copy those units atomically. |
173 pd_arrayof_conjoint_bytes(from, to, count); |
174 pd_arrayof_conjoint_bytes(from, to, count); |
174 } |
175 } |
175 |
176 |
176 // jshorts, conjoint array, atomic on each jshort |
177 // jshorts, conjoint array, atomic on each jshort |
177 static void arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) { |
178 static void arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) { |
178 assert_params_ok(from, to, LogBytesPerShort); |
179 assert_params_ok(from, to, BytesPerShort); |
179 pd_arrayof_conjoint_jshorts(from, to, count); |
180 pd_arrayof_conjoint_jshorts(from, to, count); |
180 } |
181 } |
181 |
182 |
182 // jints, conjoint array, atomic on each jint |
183 // jints, conjoint array, atomic on each jint |
183 static void arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) { |
184 static void arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) { |
184 assert_params_ok(from, to, LogBytesPerInt); |
185 assert_params_ok(from, to, BytesPerInt); |
185 pd_arrayof_conjoint_jints(from, to, count); |
186 pd_arrayof_conjoint_jints(from, to, count); |
186 } |
187 } |
187 |
188 |
188 // jlongs, conjoint array, atomic on each jlong |
189 // jlongs, conjoint array, atomic on each jlong |
189 static void arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) { |
190 static void arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) { |
190 assert_params_ok(from, to, LogBytesPerLong); |
191 assert_params_ok(from, to, BytesPerLong); |
191 pd_arrayof_conjoint_jlongs(from, to, count); |
192 pd_arrayof_conjoint_jlongs(from, to, count); |
192 } |
193 } |
193 |
194 |
194 // oops, conjoint array, atomic on each oop |
195 // oops, conjoint array, atomic on each oop |
195 static void arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) { |
196 static void arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) { |
196 assert_params_ok(from, to, LogBytesPerHeapOop); |
197 assert_params_ok(from, to, BytesPerHeapOop); |
197 pd_arrayof_conjoint_oops(from, to, count); |
198 pd_arrayof_conjoint_oops(from, to, count); |
198 } |
199 } |
199 |
200 |
200 // Known overlap methods |
201 // Known overlap methods |
201 |
202 |
202 // Copy word-aligned words from higher to lower addresses, not atomic on each word |
203 // Copy word-aligned words from higher to lower addresses, not atomic on each word |
203 inline static void conjoint_words_to_lower(const HeapWord* from, HeapWord* to, size_t byte_count) { |
204 inline static void conjoint_words_to_lower(const HeapWord* from, HeapWord* to, size_t byte_count) { |
204 // byte_count is in bytes to check its alignment |
205 // byte_count is in bytes to check its alignment |
205 assert_params_ok(from, to, LogHeapWordSize); |
206 assert_params_ok(from, to, HeapWordSize); |
206 assert_byte_count_ok(byte_count, HeapWordSize); |
207 assert_byte_count_ok(byte_count, HeapWordSize); |
207 |
208 |
208 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; |
209 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; |
209 assert(to <= from || from + count <= to, "do not overwrite source data"); |
210 assert(to <= from || from + count <= to, "do not overwrite source data"); |
210 |
211 |
214 } |
215 } |
215 |
216 |
216 // Copy word-aligned words from lower to higher addresses, not atomic on each word |
217 // Copy word-aligned words from lower to higher addresses, not atomic on each word |
217 inline static void conjoint_words_to_higher(const HeapWord* from, HeapWord* to, size_t byte_count) { |
218 inline static void conjoint_words_to_higher(const HeapWord* from, HeapWord* to, size_t byte_count) { |
218 // byte_count is in bytes to check its alignment |
219 // byte_count is in bytes to check its alignment |
219 assert_params_ok(from, to, LogHeapWordSize); |
220 assert_params_ok(from, to, HeapWordSize); |
220 assert_byte_count_ok(byte_count, HeapWordSize); |
221 assert_byte_count_ok(byte_count, HeapWordSize); |
221 |
222 |
222 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; |
223 size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize; |
223 assert(from <= to || to + count <= from, "do not overwrite source data"); |
224 assert(from <= to || to + count <= from, "do not overwrite source data"); |
224 |
225 |
269 // Fill methods |
270 // Fill methods |
270 |
271 |
271 // Fill word-aligned words, not atomic on each word |
272 // Fill word-aligned words, not atomic on each word |
272 // set_words |
273 // set_words |
273 static void fill_to_words(HeapWord* to, size_t count, juint value = 0) { |
274 static void fill_to_words(HeapWord* to, size_t count, juint value = 0) { |
274 assert_params_ok(to, LogHeapWordSize); |
275 assert_params_ok(to, HeapWordSize); |
275 pd_fill_to_words(to, count, value); |
276 pd_fill_to_words(to, count, value); |
276 } |
277 } |
277 |
278 |
278 static void fill_to_aligned_words(HeapWord* to, size_t count, juint value = 0) { |
279 static void fill_to_aligned_words(HeapWord* to, size_t count, juint value = 0) { |
279 assert_params_aligned(to); |
280 assert_params_aligned(to); |
313 } |
314 } |
314 |
315 |
315 // These methods raise a fatal if they detect a problem. |
316 // These methods raise a fatal if they detect a problem. |
316 |
317 |
317 static void assert_disjoint(const HeapWord* from, HeapWord* to, size_t count) { |
318 static void assert_disjoint(const HeapWord* from, HeapWord* to, size_t count) { |
318 #ifdef ASSERT |
319 assert(params_disjoint(from, to, count), "source and dest overlap"); |
319 if (!params_disjoint(from, to, count)) |
320 } |
320 basic_fatal("source and dest overlap"); |
321 |
321 #endif |
322 static void assert_params_ok(const void* from, void* to, intptr_t alignment) { |
322 } |
323 assert(is_aligned(from, alignment), "must be aligned: " INTPTR_FORMAT, p2i(from)); |
323 |
324 assert(is_aligned(to, alignment), "must be aligned: " INTPTR_FORMAT, p2i(to)); |
324 static void assert_params_ok(const void* from, void* to, intptr_t log_align) { |
325 } |
325 #ifdef ASSERT |
326 |
326 if (mask_bits((uintptr_t)from, right_n_bits(log_align)) != 0) |
327 static void assert_params_ok(HeapWord* to, intptr_t alignment) { |
327 basic_fatal("not aligned"); |
328 assert(is_aligned(to, alignment), "must be aligned: " INTPTR_FORMAT, p2i(to)); |
328 if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0) |
329 } |
329 basic_fatal("not aligned"); |
330 |
330 #endif |
|
331 } |
|
332 |
|
333 static void assert_params_ok(HeapWord* to, intptr_t log_align) { |
|
334 #ifdef ASSERT |
|
335 if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0) |
|
336 basic_fatal("not word aligned"); |
|
337 #endif |
|
338 } |
|
339 static void assert_params_aligned(const HeapWord* from, HeapWord* to) { |
331 static void assert_params_aligned(const HeapWord* from, HeapWord* to) { |
340 #ifdef ASSERT |
332 assert(is_aligned(from, BytesPerLong), "must be aligned: " INTPTR_FORMAT, p2i(from)); |
341 if (mask_bits((uintptr_t)from, BytesPerLong-1) != 0) |
333 assert(is_aligned(to, BytesPerLong), "must be aligned: " INTPTR_FORMAT, p2i(to)); |
342 basic_fatal("not long aligned"); |
|
343 if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0) |
|
344 basic_fatal("not long aligned"); |
|
345 #endif |
|
346 } |
334 } |
347 |
335 |
348 static void assert_params_aligned(HeapWord* to) { |
336 static void assert_params_aligned(HeapWord* to) { |
349 #ifdef ASSERT |
337 assert(is_aligned(to, BytesPerLong), "must be aligned: " INTPTR_FORMAT, p2i(to)); |
350 if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0) |
|
351 basic_fatal("not long aligned"); |
|
352 #endif |
|
353 } |
338 } |
354 |
339 |
355 static void assert_byte_count_ok(size_t byte_count, size_t unit_size) { |
340 static void assert_byte_count_ok(size_t byte_count, size_t unit_size) { |
356 #ifdef ASSERT |
341 assert(is_aligned(byte_count, unit_size), "byte count must be aligned"); |
357 if (!is_aligned(byte_count, unit_size)) { |
|
358 basic_fatal("byte count must be aligned"); |
|
359 } |
|
360 #endif |
|
361 } |
342 } |
362 |
343 |
363 // Platform dependent implementations of the above methods. |
344 // Platform dependent implementations of the above methods. |
364 #include CPU_HEADER(copy) |
345 #include CPU_HEADER(copy) |
365 |
346 |