/*

 * Copyright (c) 2000, 2016, Oracle and/or its affiliates. All rights reserved.

 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

 *

 * This code is free software; you can redistribute it and/or modify it

 * under the terms of the GNU General Public License version 2 only, as

 * published by the Free Software Foundation.

 *

 * This code is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

 * version 2 for more details (a copy is included in the LICENSE file that

 * accompanied this code).

 *

 * You should have received a copy of the GNU General Public License version

 * 2 along with this work; if not, write to the Free Software Foundation,

 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

 *

 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

 * or visit www.oracle.com if you need additional information or have any

 * questions.

 *

 */

#include "precompiled.hpp"

#include "classfile/classFileStream.hpp"

#include "classfile/vmSymbols.hpp"

#include "memory/allocation.inline.hpp"

#include "memory/resourceArea.hpp"

#include "oops/objArrayOop.inline.hpp"

#include "oops/oop.inline.hpp"

#include "prims/jni.h"

#include "prims/jvm.h"

#include "prims/unsafe.hpp"

#include "runtime/atomic.inline.hpp"

#include "runtime/globals.hpp"

#include "runtime/interfaceSupport.hpp"

#include "runtime/orderAccess.inline.hpp"

#include "runtime/reflection.hpp"

#include "runtime/vm_version.hpp"

#include "services/threadService.hpp"

#include "trace/tracing.hpp"

#include "utilities/copy.hpp"

#include "utilities/dtrace.hpp"

#include "utilities/macros.hpp"

#if INCLUDE_ALL_GCS

#include "gc/g1/g1SATBCardTableModRefBS.hpp"

#endif // INCLUDE_ALL_GCS

/**

 * Implementation of the jdk.internal.misc.Unsafe class

 */

#define MAX_OBJECT_SIZE \

  ( arrayOopDesc::header_size(T_DOUBLE) * HeapWordSize \

    + ((julong)max_jint * sizeof(double)) )

#define UNSAFE_ENTRY(result_type, header) \

  JVM_ENTRY(static result_type, header)

#define UNSAFE_LEAF(result_type, header) \

  JVM_LEAF(static result_type, header)

#define UNSAFE_END JVM_END

static inline void* addr_from_java(jlong addr) {

  // This assert fails in a variety of ways on 32-bit systems.

  // It is impossible to predict whether native code that converts

  // pointers to longs will sign-extend or zero-extend the addresses.

  //assert(addr == (uintptr_t)addr, "must not be odd high bits");

  return (void*)(uintptr_t)addr;

}

static inline jlong addr_to_java(void* p) {

  assert(p == (void*)(uintptr_t)p, "must not be odd high bits");

  return (uintptr_t)p;

}

// Note: The VM's obj_field and related accessors use byte-scaled

// ("unscaled") offsets, just as the unsafe methods do.

// However, the method Unsafe.fieldOffset explicitly declines to

// guarantee this.  The field offset values manipulated by the Java user

// through the Unsafe API are opaque cookies that just happen to be byte

// offsets.  We represent this state of affairs by passing the cookies

// through conversion functions when going between the VM and the Unsafe API.

// The conversion functions just happen to be no-ops at present.

static inline jlong field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

static inline jlong field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

static inline void* index_oop_from_field_offset_long(oop p, jlong field_offset) {

  jlong byte_offset = field_offset_to_byte_offset(field_offset);

#ifdef ASSERT

  if (p != NULL) {

    assert(byte_offset >= 0 && byte_offset <= (jlong)MAX_OBJECT_SIZE, "sane offset");

    if (byte_offset == (jint)byte_offset) {

      void* ptr_plus_disp = (address)p + byte_offset;

      assert((void*)p->obj_field_addr<oop>((jint)byte_offset) == ptr_plus_disp,

             "raw [ptr+disp] must be consistent with oop::field_base");

    }

    jlong p_size = HeapWordSize * (jlong)(p->size());

    assert(byte_offset < p_size, "Unsafe access: offset " INT64_FORMAT " > object's size " INT64_FORMAT, byte_offset, p_size);

  }

#endif

  if (sizeof(char*) == sizeof(jint)) {   // (this constant folds!)

    return (address)p + (jint) byte_offset;

  } else {

    return (address)p +        byte_offset;

  }

}

// Externally callable versions:

// (Use these in compiler intrinsics which emulate unsafe primitives.)

jlong Unsafe_field_offset_to_byte_offset(jlong field_offset) {

  return field_offset;

}

jlong Unsafe_field_offset_from_byte_offset(jlong byte_offset) {

  return byte_offset;

}

///// Data read/writes on the Java heap and in native (off-heap) memory

/**

 * Helper class for accessing memory.

 *

 * Normalizes values and wraps accesses in

 * JavaThread::doing_unsafe_access() if needed.

 */

class MemoryAccess : StackObj {

  JavaThread* _thread;

  jobject _obj;

  jlong _offset;

  // Resolves and returns the address of the memory access

  void* addr() {

    return index_oop_from_field_offset_long(JNIHandles::resolve(_obj), _offset);

  }

  template <typename T>

  T normalize(T x) {

    return x;

  }

  jboolean normalize(jboolean x) {

    return x & 1;

  }

  /**

   * Helper class to wrap memory accesses in JavaThread::doing_unsafe_access()

   */

  class GuardUnsafeAccess {

    JavaThread* _thread;

    bool _active;

  public:

    GuardUnsafeAccess(JavaThread* thread, jobject _obj) : _thread(thread) {

      if (JNIHandles::resolve(_obj) == NULL) {

        // native/off-heap access which may raise SIGBUS if accessing

        // memory mapped file data in a region of the file which has

        // been truncated and is now invalid

        _thread->set_doing_unsafe_access(true);

        _active = true;

      } else {

        _active = false;

      }

    }

    ~GuardUnsafeAccess() {

      if (_active) {

        _thread->set_doing_unsafe_access(false);

      }

    }

  };

public:

  MemoryAccess(JavaThread* thread, jobject obj, jlong offset)

    : _thread(thread), _obj(obj), _offset(offset) {

  }

  template <typename T>

  T get() {

    GuardUnsafeAccess guard(_thread, _obj);

    T* p = (T*)addr();

    T x = *p;

    return x;

  }

  template <typename T>

  void put(T x) {

    GuardUnsafeAccess guard(_thread, _obj);

    T* p = (T*)addr();

    *p = normalize(x);

  }

  template <typename T>

  T get_volatile() {

    GuardUnsafeAccess guard(_thread, _obj);

    T* p = (T*)addr();

    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {

      OrderAccess::fence();

    }

    T x = OrderAccess::load_acquire((volatile T*)p);

    return x;

  }

  template <typename T>

  void put_volatile(T x) {

    GuardUnsafeAccess guard(_thread, _obj);

    T* p = (T*)addr();

    OrderAccess::release_store_fence((volatile T*)p, normalize(x));

  }

#ifndef SUPPORTS_NATIVE_CX8

  jlong get_jlong_locked() {

    GuardUnsafeAccess guard(_thread, _obj);

    MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag);

    jlong* p = (jlong*)addr();

    jlong x = Atomic::load(p);

    return x;

  }

  void put_jlong_locked(jlong x) {

    GuardUnsafeAccess guard(_thread, _obj);

    MutexLockerEx mu(UnsafeJlong_lock, Mutex::_no_safepoint_check_flag);

    jlong* p = (jlong*)addr();

    Atomic::store(normalize(x),  p);

  }

#endif

};

// Get/PutObject must be special-cased, since it works with handles.

// These functions allow a null base pointer with an arbitrary address.

// But if the base pointer is non-null, the offset should make some sense.

// That is, it should be in the range [0, MAX_OBJECT_SIZE].

UNSAFE_ENTRY(jobject, Unsafe_GetObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  oop v;

  if (UseCompressedOops) {

    narrowOop n = *(narrowOop*)index_oop_from_field_offset_long(p, offset);

    v = oopDesc::decode_heap_oop(n);

  } else {

    v = *(oop*)index_oop_from_field_offset_long(p, offset);

  }

  jobject ret = JNIHandles::make_local(env, v);

#if INCLUDE_ALL_GCS

  // We could be accessing the referent field in a reference

  // object. If G1 is enabled then we need to register non-null

  // referent with the SATB barrier.

  if (UseG1GC) {

    bool needs_barrier = false;

    if (ret != NULL) {

      if (offset == java_lang_ref_Reference::referent_offset && obj != NULL) {

        oop o = JNIHandles::resolve(obj);

        Klass* k = o->klass();

        if (InstanceKlass::cast(k)->reference_type() != REF_NONE) {

          assert(InstanceKlass::cast(k)->is_subclass_of(SystemDictionary::Reference_klass()), "sanity");

          needs_barrier = true;

        }

      }

    }

    if (needs_barrier) {

      oop referent = JNIHandles::resolve(ret);

      G1SATBCardTableModRefBS::enqueue(referent);

    }

  }

#endif // INCLUDE_ALL_GCS

  return ret;

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutObject(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  if (UseCompressedOops) {

    oop_store((narrowOop*)index_oop_from_field_offset_long(p, offset), x);

  } else {

    oop_store((oop*)index_oop_from_field_offset_long(p, offset), x);

  }

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetObjectVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  oop p = JNIHandles::resolve(obj);

  void* addr = index_oop_from_field_offset_long(p, offset);

  volatile oop v;

  if (UseCompressedOops) {

    volatile narrowOop n = *(volatile narrowOop*) addr;

    (void)const_cast<oop&>(v = oopDesc::decode_heap_oop(n));

  } else {

    (void)const_cast<oop&>(v = *(volatile oop*) addr);

  }

  OrderAccess::acquire();

  return JNIHandles::make_local(env, v);

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutObjectVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jobject x_h)) {

  oop x = JNIHandles::resolve(x_h);

  oop p = JNIHandles::resolve(obj);

  void* addr = index_oop_from_field_offset_long(p, offset);

  OrderAccess::release();

  if (UseCompressedOops) {

    oop_store((narrowOop*)addr, x);

  } else {

    oop_store((oop*)addr, x);

  }

  OrderAccess::fence();

} UNSAFE_END

UNSAFE_ENTRY(jobject, Unsafe_GetUncompressedObject(JNIEnv *env, jobject unsafe, jlong addr)) {

  oop v = *(oop*) (address) addr;

  return JNIHandles::make_local(env, v);

} UNSAFE_END

#ifndef SUPPORTS_NATIVE_CX8

// VM_Version::supports_cx8() is a surrogate for 'supports atomic long memory ops'.

//

// On platforms which do not support atomic compare-and-swap of jlong (8 byte)

// values we have to use a lock-based scheme to enforce atomicity. This has to be

// applied to all Unsafe operations that set the value of a jlong field. Even so

// the compareAndSwapLong operation will not be atomic with respect to direct stores

// to the field from Java code. It is important therefore that any Java code that

// utilizes these Unsafe jlong operations does not perform direct stores. To permit

// direct loads of the field from Java code we must also use Atomic::store within the

// locked regions. And for good measure, in case there are direct stores, we also

// employ Atomic::load within those regions. Note that the field in question must be

// volatile and so must have atomic load/store accesses applied at the Java level.

//

// The locking scheme could utilize a range of strategies for controlling the locking

// granularity: from a lock per-field through to a single global lock. The latter is

// the simplest and is used for the current implementation. Note that the Java object

// that contains the field, can not, in general, be used for locking. To do so can lead

// to deadlocks as we may introduce locking into what appears to the Java code to be a

// lock-free path.

//

// As all the locked-regions are very short and themselves non-blocking we can treat

// them as leaf routines and elide safepoint checks (ie we don't perform any thread

// state transitions even when blocking for the lock). Note that if we do choose to

// add safepoint checks and thread state transitions, we must ensure that we calculate

// the address of the field _after_ we have acquired the lock, else the object may have

// been moved by the GC

UNSAFE_ENTRY(jlong, Unsafe_GetLongVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) {

  if (VM_Version::supports_cx8()) {

    return MemoryAccess(thread, obj, offset).get_volatile<jlong>();

  } else {

    return MemoryAccess(thread, obj, offset).get_jlong_locked();

  }

} UNSAFE_END

UNSAFE_ENTRY(void, Unsafe_PutLongVolatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, jlong x)) {

  if (VM_Version::supports_cx8()) {

    MemoryAccess(thread, obj, offset).put_volatile<jlong>(x);

  } else {

    MemoryAccess(thread, obj, offset).put_jlong_locked(x);

  }

} UNSAFE_END

#endif // not SUPPORTS_NATIVE_CX8

UNSAFE_LEAF(jboolean, Unsafe_isBigEndian0(JNIEnv *env, jobject unsafe)) {

#ifdef VM_LITTLE_ENDIAN

  return false;

#else

  return true;

#endif

} UNSAFE_END

UNSAFE_LEAF(jint, Unsafe_unalignedAccess0(JNIEnv *env, jobject unsafe)) {

  return UseUnalignedAccesses;

} UNSAFE_END

#define DEFINE_GETSETOOP(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess(thread, obj, offset).get<java_type>(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess(thread, obj, offset).put<java_type>(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP.

DEFINE_GETSETOOP(jboolean, Boolean)

DEFINE_GETSETOOP(jbyte, Byte)

DEFINE_GETSETOOP(jshort, Short);

DEFINE_GETSETOOP(jchar, Char);

DEFINE_GETSETOOP(jint, Int);

DEFINE_GETSETOOP(jlong, Long);

DEFINE_GETSETOOP(jfloat, Float);

DEFINE_GETSETOOP(jdouble, Double);

#undef DEFINE_GETSETOOP

#define DEFINE_GETSETOOP_VOLATILE(java_type, Type) \

 \

UNSAFE_ENTRY(java_type, Unsafe_Get##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset)) { \

  return MemoryAccess(thread, obj, offset).get_volatile<java_type>(); \

} UNSAFE_END \

 \

UNSAFE_ENTRY(void, Unsafe_Put##Type##Volatile(JNIEnv *env, jobject unsafe, jobject obj, jlong offset, java_type x)) { \

  MemoryAccess(thread, obj, offset).put_volatile<java_type>(x); \

} UNSAFE_END \

 \

// END DEFINE_GETSETOOP_VOLATILE.

DEFINE_GETSETOOP_VOLATILE(jboolean, Boolean)

DEFINE_GETSETOOP_VOLATILE(jbyte, Byte)

DEFINE_GETSETOOP_VOLATILE(jshort, Short);

DEFINE_GETSETOOP_VOLATILE(jchar, Char);

DEFINE_GETSETOOP_VOLATILE(jint, Int);

DEFINE_GETSETOOP_VOLATILE(jfloat, Float);

DEFINE_GETSETOOP_VOLATILE(jdouble, Double);

#ifdef SUPPORTS_NATIVE_CX8

DEFINE_GETSETOOP_VOLATILE(jlong, Long);

#endif

#undef DEFINE_GETSETOOP_VOLATILE

UNSAFE_LEAF(void, Unsafe_LoadFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::acquire();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_StoreFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::release();

} UNSAFE_END

UNSAFE_LEAF(void, Unsafe_FullFence(JNIEnv *env, jobject unsafe)) {

  OrderAccess::fence();

} UNSAFE_END

////// Allocation requests

UNSAFE_ENTRY(jobject, Unsafe_AllocateInstance(JNIEnv *env, jobject unsafe, jclass cls)) {

  ThreadToNativeFromVM ttnfv(thread);

  return env->AllocObject(cls);

} UNSAFE_END

UNSAFE_ENTRY(jlong, Unsafe_AllocateMemory0(JNIEnv *env, jobject unsafe, jlong size)) {

  size_t sz = (size_t)size;

  sz = round_to(sz, HeapWordSize);

  void* x = os::malloc(sz, mtInternal);

  return addr_to_java(x);