src/hotspot/share/utilities/copy.hpp
changeset 49011 a0e246b7403a
parent 48951 950c35ea6237
child 53244 9807daeb47c4
equal deleted inserted replaced
49010:9010e596f391 49011:a0e246b7403a
    25 #ifndef SHARE_VM_UTILITIES_COPY_HPP
    25 #ifndef SHARE_VM_UTILITIES_COPY_HPP
    26 #define SHARE_VM_UTILITIES_COPY_HPP
    26 #define SHARE_VM_UTILITIES_COPY_HPP
    27 
    27 
    28 #include "runtime/stubRoutines.hpp"
    28 #include "runtime/stubRoutines.hpp"
    29 #include "utilities/align.hpp"
    29 #include "utilities/align.hpp"
       
    30 #include "utilities/debug.hpp"
    30 #include "utilities/macros.hpp"
    31 #include "utilities/macros.hpp"
    31 
    32 
    32 // Assembly code for platforms that need it.
    33 // Assembly code for platforms that need it.
    33 extern "C" {
    34 extern "C" {
    34   void _Copy_conjoint_words(const HeapWord* from, HeapWord* to, size_t count);
    35   void _Copy_conjoint_words(const HeapWord* from, HeapWord* to, size_t count);
    86 
    87 
    87   // HeapWords
    88   // HeapWords
    88 
    89 
    89   // Word-aligned words,    conjoint, not atomic on each word
    90   // Word-aligned words,    conjoint, not atomic on each word
    90   static void conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
    91   static void conjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
    91     assert_params_ok(from, to, LogHeapWordSize);
    92     assert_params_ok(from, to, HeapWordSize);
    92     pd_conjoint_words(from, to, count);
    93     pd_conjoint_words(from, to, count);
    93   }
    94   }
    94 
    95 
    95   // Word-aligned words,    disjoint, not atomic on each word
    96   // Word-aligned words,    disjoint, not atomic on each word
    96   static void disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
    97   static void disjoint_words(const HeapWord* from, HeapWord* to, size_t count) {
    97     assert_params_ok(from, to, LogHeapWordSize);
    98     assert_params_ok(from, to, HeapWordSize);
    98     assert_disjoint(from, to, count);
    99     assert_disjoint(from, to, count);
    99     pd_disjoint_words(from, to, count);
   100     pd_disjoint_words(from, to, count);
   100   }
   101   }
   101 
   102 
   102   // Word-aligned words,    disjoint, atomic on each word
   103   // Word-aligned words,    disjoint, atomic on each word
   103   static void disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) {
   104   static void disjoint_words_atomic(const HeapWord* from, HeapWord* to, size_t count) {
   104     assert_params_ok(from, to, LogHeapWordSize);
   105     assert_params_ok(from, to, HeapWordSize);
   105     assert_disjoint(from, to, count);
   106     assert_disjoint(from, to, count);
   106     pd_disjoint_words_atomic(from, to, count);
   107     pd_disjoint_words_atomic(from, to, count);
   107   }
   108   }
   108 
   109 
   109   // Object-aligned words,  conjoint, not atomic on each word
   110   // Object-aligned words,  conjoint, not atomic on each word
   131     pd_conjoint_bytes(from, to, count);
   132     pd_conjoint_bytes(from, to, count);
   132   }
   133   }
   133 
   134 
   134   // jshorts,               conjoint, atomic on each jshort
   135   // jshorts,               conjoint, atomic on each jshort
   135   static void conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
   136   static void conjoint_jshorts_atomic(const jshort* from, jshort* to, size_t count) {
   136     assert_params_ok(from, to, LogBytesPerShort);
   137     assert_params_ok(from, to, BytesPerShort);
   137     pd_conjoint_jshorts_atomic(from, to, count);
   138     pd_conjoint_jshorts_atomic(from, to, count);
   138   }
   139   }
   139 
   140 
   140   // jints,                 conjoint, atomic on each jint
   141   // jints,                 conjoint, atomic on each jint
   141   static void conjoint_jints_atomic(const jint* from, jint* to, size_t count) {
   142   static void conjoint_jints_atomic(const jint* from, jint* to, size_t count) {
   142     assert_params_ok(from, to, LogBytesPerInt);
   143     assert_params_ok(from, to, BytesPerInt);
   143     pd_conjoint_jints_atomic(from, to, count);
   144     pd_conjoint_jints_atomic(from, to, count);
   144   }
   145   }
   145 
   146 
   146   // jlongs,                conjoint, atomic on each jlong
   147   // jlongs,                conjoint, atomic on each jlong
   147   static void conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
   148   static void conjoint_jlongs_atomic(const jlong* from, jlong* to, size_t count) {
   148     assert_params_ok(from, to, LogBytesPerLong);
   149     assert_params_ok(from, to, BytesPerLong);
   149     pd_conjoint_jlongs_atomic(from, to, count);
   150     pd_conjoint_jlongs_atomic(from, to, count);
   150   }
   151   }
   151 
   152 
   152   // oops,                  conjoint, atomic on each oop
   153   // oops,                  conjoint, atomic on each oop
   153   static void conjoint_oops_atomic(const oop* from, oop* to, size_t count) {
   154   static void conjoint_oops_atomic(const oop* from, oop* to, size_t count) {
   154     assert_params_ok(from, to, LogBytesPerHeapOop);
   155     assert_params_ok(from, to, BytesPerHeapOop);
   155     pd_conjoint_oops_atomic(from, to, count);
   156     pd_conjoint_oops_atomic(from, to, count);
   156   }
   157   }
   157 
   158 
   158   // overloaded for UseCompressedOops
   159   // overloaded for UseCompressedOops
   159   static void conjoint_oops_atomic(const narrowOop* from, narrowOop* to, size_t count) {
   160   static void conjoint_oops_atomic(const narrowOop* from, narrowOop* to, size_t count) {
   160     assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong");
   161     assert(sizeof(narrowOop) == sizeof(jint), "this cast is wrong");
   161     assert_params_ok(from, to, LogBytesPerInt);
   162     assert_params_ok(from, to, BytesPerInt);
   162     pd_conjoint_jints_atomic((const jint*)from, (jint*)to, count);
   163     pd_conjoint_jints_atomic((const jint*)from, (jint*)to, count);
   163   }
   164   }
   164 
   165 
   165   // Copy a span of memory.  If the span is an integral number of aligned
   166   // Copy a span of memory.  If the span is an integral number of aligned
   166   // longs, words, or ints, copy those units atomically.
   167   // longs, words, or ints, copy those units atomically.
   173     pd_arrayof_conjoint_bytes(from, to, count);
   174     pd_arrayof_conjoint_bytes(from, to, count);
   174   }
   175   }
   175 
   176 
   176   // jshorts,               conjoint array, atomic on each jshort
   177   // jshorts,               conjoint array, atomic on each jshort
   177   static void arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) {
   178   static void arrayof_conjoint_jshorts(const HeapWord* from, HeapWord* to, size_t count) {
   178     assert_params_ok(from, to, LogBytesPerShort);
   179     assert_params_ok(from, to, BytesPerShort);
   179     pd_arrayof_conjoint_jshorts(from, to, count);
   180     pd_arrayof_conjoint_jshorts(from, to, count);
   180   }
   181   }
   181 
   182 
   182   // jints,                 conjoint array, atomic on each jint
   183   // jints,                 conjoint array, atomic on each jint
   183   static void arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) {
   184   static void arrayof_conjoint_jints(const HeapWord* from, HeapWord* to, size_t count) {
   184     assert_params_ok(from, to, LogBytesPerInt);
   185     assert_params_ok(from, to, BytesPerInt);
   185     pd_arrayof_conjoint_jints(from, to, count);
   186     pd_arrayof_conjoint_jints(from, to, count);
   186   }
   187   }
   187 
   188 
   188   // jlongs,                conjoint array, atomic on each jlong
   189   // jlongs,                conjoint array, atomic on each jlong
   189   static void arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) {
   190   static void arrayof_conjoint_jlongs(const HeapWord* from, HeapWord* to, size_t count) {
   190     assert_params_ok(from, to, LogBytesPerLong);
   191     assert_params_ok(from, to, BytesPerLong);
   191     pd_arrayof_conjoint_jlongs(from, to, count);
   192     pd_arrayof_conjoint_jlongs(from, to, count);
   192   }
   193   }
   193 
   194 
   194   // oops,                  conjoint array, atomic on each oop
   195   // oops,                  conjoint array, atomic on each oop
   195   static void arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) {
   196   static void arrayof_conjoint_oops(const HeapWord* from, HeapWord* to, size_t count) {
   196     assert_params_ok(from, to, LogBytesPerHeapOop);
   197     assert_params_ok(from, to, BytesPerHeapOop);
   197     pd_arrayof_conjoint_oops(from, to, count);
   198     pd_arrayof_conjoint_oops(from, to, count);
   198   }
   199   }
   199 
   200 
   200   // Known overlap methods
   201   // Known overlap methods
   201 
   202 
   202   // Copy word-aligned words from higher to lower addresses, not atomic on each word
   203   // Copy word-aligned words from higher to lower addresses, not atomic on each word
   203   inline static void conjoint_words_to_lower(const HeapWord* from, HeapWord* to, size_t byte_count) {
   204   inline static void conjoint_words_to_lower(const HeapWord* from, HeapWord* to, size_t byte_count) {
   204     // byte_count is in bytes to check its alignment
   205     // byte_count is in bytes to check its alignment
   205     assert_params_ok(from, to, LogHeapWordSize);
   206     assert_params_ok(from, to, HeapWordSize);
   206     assert_byte_count_ok(byte_count, HeapWordSize);
   207     assert_byte_count_ok(byte_count, HeapWordSize);
   207 
   208 
   208     size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize;
   209     size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize;
   209     assert(to <= from || from + count <= to, "do not overwrite source data");
   210     assert(to <= from || from + count <= to, "do not overwrite source data");
   210 
   211 
   214   }
   215   }
   215 
   216 
   216   // Copy word-aligned words from lower to higher addresses, not atomic on each word
   217   // Copy word-aligned words from lower to higher addresses, not atomic on each word
   217   inline static void conjoint_words_to_higher(const HeapWord* from, HeapWord* to, size_t byte_count) {
   218   inline static void conjoint_words_to_higher(const HeapWord* from, HeapWord* to, size_t byte_count) {
   218     // byte_count is in bytes to check its alignment
   219     // byte_count is in bytes to check its alignment
   219     assert_params_ok(from, to, LogHeapWordSize);
   220     assert_params_ok(from, to, HeapWordSize);
   220     assert_byte_count_ok(byte_count, HeapWordSize);
   221     assert_byte_count_ok(byte_count, HeapWordSize);
   221 
   222 
   222     size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize;
   223     size_t count = align_up(byte_count, HeapWordSize) >> LogHeapWordSize;
   223     assert(from <= to || to + count <= from, "do not overwrite source data");
   224     assert(from <= to || to + count <= from, "do not overwrite source data");
   224 
   225 
   269   // Fill methods
   270   // Fill methods
   270 
   271 
   271   // Fill word-aligned words, not atomic on each word
   272   // Fill word-aligned words, not atomic on each word
   272   // set_words
   273   // set_words
   273   static void fill_to_words(HeapWord* to, size_t count, juint value = 0) {
   274   static void fill_to_words(HeapWord* to, size_t count, juint value = 0) {
   274     assert_params_ok(to, LogHeapWordSize);
   275     assert_params_ok(to, HeapWordSize);
   275     pd_fill_to_words(to, count, value);
   276     pd_fill_to_words(to, count, value);
   276   }
   277   }
   277 
   278 
   278   static void fill_to_aligned_words(HeapWord* to, size_t count, juint value = 0) {
   279   static void fill_to_aligned_words(HeapWord* to, size_t count, juint value = 0) {
   279     assert_params_aligned(to);
   280     assert_params_aligned(to);
   293 
   294 
   294   // Zero-fill methods
   295   // Zero-fill methods
   295 
   296 
   296   // Zero word-aligned words, not atomic on each word
   297   // Zero word-aligned words, not atomic on each word
   297   static void zero_to_words(HeapWord* to, size_t count) {
   298   static void zero_to_words(HeapWord* to, size_t count) {
   298     assert_params_ok(to, LogHeapWordSize);
   299     assert_params_ok(to, HeapWordSize);
   299     pd_zero_to_words(to, count);
   300     pd_zero_to_words(to, count);
   300   }
   301   }
   301 
   302 
   302   // Zero bytes
   303   // Zero bytes
   303   static void zero_to_bytes(void* to, size_t count) {
   304   static void zero_to_bytes(void* to, size_t count) {
   313   }
   314   }
   314 
   315 
   315   // These methods raise a fatal if they detect a problem.
   316   // These methods raise a fatal if they detect a problem.
   316 
   317 
   317   static void assert_disjoint(const HeapWord* from, HeapWord* to, size_t count) {
   318   static void assert_disjoint(const HeapWord* from, HeapWord* to, size_t count) {
   318 #ifdef ASSERT
   319     assert(params_disjoint(from, to, count), "source and dest overlap");
   319     if (!params_disjoint(from, to, count))
   320   }
   320       basic_fatal("source and dest overlap");
   321 
   321 #endif
   322   static void assert_params_ok(const void* from, void* to, intptr_t alignment) {
   322   }
   323     assert(is_aligned(from, alignment), "must be aligned: " INTPTR_FORMAT, p2i(from));
   323 
   324     assert(is_aligned(to, alignment),   "must be aligned: " INTPTR_FORMAT, p2i(to));
   324   static void assert_params_ok(const void* from, void* to, intptr_t log_align) {
   325   }
   325 #ifdef ASSERT
   326 
   326     if (mask_bits((uintptr_t)from, right_n_bits(log_align)) != 0)
   327   static void assert_params_ok(HeapWord* to, intptr_t alignment) {
   327       basic_fatal("not aligned");
   328     assert(is_aligned(to, alignment), "must be aligned: " INTPTR_FORMAT, p2i(to));
   328     if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0)
   329   }
   329       basic_fatal("not aligned");
   330 
   330 #endif
       
   331   }
       
   332 
       
   333   static void assert_params_ok(HeapWord* to, intptr_t log_align) {
       
   334 #ifdef ASSERT
       
   335     if (mask_bits((uintptr_t)to, right_n_bits(log_align)) != 0)
       
   336       basic_fatal("not word aligned");
       
   337 #endif
       
   338   }
       
   339   static void assert_params_aligned(const HeapWord* from, HeapWord* to) {
   331   static void assert_params_aligned(const HeapWord* from, HeapWord* to) {
   340 #ifdef ASSERT
   332     assert(is_aligned(from, BytesPerLong), "must be aligned: " INTPTR_FORMAT, p2i(from));
   341     if (mask_bits((uintptr_t)from, BytesPerLong-1) != 0)
   333     assert(is_aligned(to, BytesPerLong),   "must be aligned: " INTPTR_FORMAT, p2i(to));
   342       basic_fatal("not long aligned");
       
   343     if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0)
       
   344       basic_fatal("not long aligned");
       
   345 #endif
       
   346   }
   334   }
   347 
   335 
   348   static void assert_params_aligned(HeapWord* to) {
   336   static void assert_params_aligned(HeapWord* to) {
   349 #ifdef ASSERT
   337     assert(is_aligned(to, BytesPerLong), "must be aligned: " INTPTR_FORMAT, p2i(to));
   350     if (mask_bits((uintptr_t)to, BytesPerLong-1) != 0)
       
   351       basic_fatal("not long aligned");
       
   352 #endif
       
   353   }
   338   }
   354 
   339 
   355   static void assert_byte_count_ok(size_t byte_count, size_t unit_size) {
   340   static void assert_byte_count_ok(size_t byte_count, size_t unit_size) {
   356 #ifdef ASSERT
   341     assert(is_aligned(byte_count, unit_size), "byte count must be aligned");
   357     if (!is_aligned(byte_count, unit_size)) {
       
   358       basic_fatal("byte count must be aligned");
       
   359     }
       
   360 #endif
       
   361   }
   342   }
   362 
   343 
   363   // Platform dependent implementations of the above methods.
   344   // Platform dependent implementations of the above methods.
   364 #include CPU_HEADER(copy)
   345 #include CPU_HEADER(copy)
   365 
   346