--- a/jdk/src/java.instrument/share/classes/java/lang/instrument/package.html	Fri Sep 02 12:30:46 2016 -0400
+++ b/jdk/src/java.instrument/share/classes/java/lang/instrument/package.html	Mon Sep 12 15:00:58 2016 -0700
@@ -105,6 +105,10 @@
 The agent class will be loaded by the system class loader
 (see {@link java.lang.ClassLoader#getSystemClassLoader ClassLoader.getSystemClassLoader}). This is
 the class loader which typically loads the class containing the application <code>main</code> method.
+The system class loader must support a mechanism to add an agent JAR file to the system class path.
+If it is a custom system class loader then it must define the
+<code>appendToClassPathForInstrumentation</code> method as specified in
+{@link Instrumentation#appendToSystemClassLoaderSearch appendToSystemClassLoaderSearch}.
 The <code>premain</code> methods will be run under the same security and classloader 
 rules as the application <code>main</code> method.
 There are no modeling restrictions on what the agent <code>premain</code> method may do.
@@ -140,7 +144,10 @@
 
   <li><p>The system class loader (
       {@link java.lang.ClassLoader#getSystemClassLoader ClassLoader.getSystemClassLoader}) must
-      support a mechanism to add an agent JAR file to the system class path.</li>
+      support a mechanism to add an agent JAR file to the system class path.
+      If it is a custom system class loader then it must define the
+      <code>appendToClassPathForInstrumentation</code> method as specified in
+      {@link Instrumentation#appendToSystemClassLoaderSearch appendToSystemClassLoaderSearch}.</li>
 </ol>  
 
 <P>

--- a/jdk/src/java.instrument/share/native/libinstrument/InvocationAdapter.c	Fri Sep 02 12:30:46 2016 -0400
+++ b/jdk/src/java.instrument/share/native/libinstrument/InvocationAdapter.c	Mon Sep 12 15:00:58 2016 -0700
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -190,10 +190,8 @@
             return JNI_ERR;
         }
 
-        /*
-         * Add to the jarfile
-         */
-        appendClassPath(agent, jarfile);
+        /* Save the jarfile name */
+        agent->mJarfile = jarfile;
 
         /*
          * The value of the Premain-Class attribute becomes the agent
@@ -241,7 +239,6 @@
         /*
          * Clean-up
          */
-        free(jarfile);
         if (options != NULL) free(options);
         freeAttributes(attributes);
         free(premainClass);
@@ -459,7 +456,23 @@
 
     /* process the premain calls on the all the JPL agents */
     if ( environment != NULL ) {
-        jthrowable outstandingException = preserveThrowable(jnienv);
+        jthrowable outstandingException = NULL;
+        /*
+         * Add the jarfile to the system class path
+         */
+        JPLISAgent * agent = environment->mAgent;
+        if (appendClassPath(agent, agent->mJarfile)) {
+            fprintf(stderr, "Unable to add %s to system class path - "
+                    "the system class loader does not define the "
+                    "appendToClassPathForInstrumentation method or the method failed\n",
+                    agent->mJarfile);
+            free((void *)agent->mJarfile);
+            abortJVM(jnienv, JPLIS_ERRORMESSAGE_CANNOTSTART);
+        }
+        free((void *)agent->mJarfile);
+        agent->mJarfile = NULL;
+
+        outstandingException = preserveThrowable(jnienv);
         success = processJavaStart( environment->mAgent,
                                     jnienv);
         restoreThrowable(jnienv, outstandingException);
@@ -631,32 +644,19 @@
     jvmtierr = (*jvmtienv)->AddToSystemClassLoaderSearch(jvmtienv, jarfile);
     check_phase_ret_1(jvmtierr);
 
-    if (jvmtierr == JVMTI_ERROR_NONE) {
-        return 0;
-    } else {
-        jvmtiPhase phase;
-        jvmtiError err;
-
-        err = (*jvmtienv)->GetPhase(jvmtienv, &phase);
-        /* can be called from any phase */
-        jplis_assert(err == JVMTI_ERROR_NONE);
-
-        if (phase == JVMTI_PHASE_LIVE) {
-            switch (jvmtierr) {
-                case JVMTI_ERROR_CLASS_LOADER_UNSUPPORTED :
-                    fprintf(stderr, "System class loader does not support adding "
-                        "JAR file to system class path during the live phase!\n");
-                        break;
-                default:
-                    fprintf(stderr, "Unexpected error (%d) returned by "
-                        "AddToSystemClassLoaderSearch\n", jvmtierr);
-                    break;
-            }
-            return -1;
-        }
-        jplis_assert(0);
+    switch (jvmtierr) {
+        case JVMTI_ERROR_NONE :
+            return 0;
+        case JVMTI_ERROR_CLASS_LOADER_UNSUPPORTED :
+            fprintf(stderr, "System class loader does not define "
+                "the appendToClassPathForInstrumentation method\n");
+            break;
+        default:
+            fprintf(stderr, "Unexpected error (%d) returned by "
+                "AddToSystemClassLoaderSearch\n", jvmtierr);
+            break;
     }
-    return -2;
+    return -1;
 }
 
 

--- a/jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.c	Fri Sep 02 12:30:46 2016 -0400
+++ b/jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.c	Mon Sep 12 15:00:58 2016 -0700
@@ -272,6 +272,7 @@
     agent->mNativeMethodPrefixAdded                  = JNI_FALSE;
     agent->mAgentClassName                           = NULL;
     agent->mOptionsString                            = NULL;
+    agent->mJarfile                                  = NULL;
 
     /* make sure we can recover either handle in either direction.
      * the agent has a ref to the jvmti; make it mutual

--- a/jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.h	Fri Sep 02 12:30:46 2016 -0400
+++ b/jdk/src/java.instrument/share/native/libinstrument/JPLISAgent.h	Mon Sep 12 15:00:58 2016 -0700
@@ -107,6 +107,7 @@
     jboolean                mNativeMethodPrefixAdded;     /* indicates if can_set_native_method_prefix capability has been added */
     char const *            mAgentClassName;        /* agent class name */
     char const *            mOptionsString;         /* -javaagent options string */
+    const char *            mJarfile;               /* agent jar file name */
 };
 
 /*

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/java/lang/instrument/CustomSystemLoader/Agent.java	Mon Sep 12 15:00:58 2016 -0700
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.PrintStream;
+import java.lang.instrument.*;
+import java.lang.reflect.Field;
+
+/**
+  * @test
+  * @bug 8160950
+  * @summary test for custom system class loader
+  *
+  * @run build App Agent CustomLoader
+  * @run shell ../MakeJAR3.sh Agent 'Can-Retransform-Classes: true'
+  * @run main/othervm -javaagent:Agent.jar -Djava.system.class.loader=CustomLoader App
+  */
+
+public class Agent {
+    private static PrintStream err = System.err;
+    private static PrintStream out = System.out;
+    public  static boolean failed = false;
+
+    public static void premain(String agentArgs, Instrumentation instrumentation) {
+        ClassLoader myClassLoader = Agent.class.getClassLoader();
+        out.println("Agent: started; myClassLoader: " + myClassLoader);
+        try {
+            Field fld = myClassLoader.getClass().getField("agentClassLoader");
+            fld.set(myClassLoader.getClass(), myClassLoader);
+        } catch (Exception ex) {
+            failed = true;
+            ex.printStackTrace();
+        }
+        out.println("Agent: finished");
+   }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/java/lang/instrument/CustomSystemLoader/App.java	Mon Sep 12 15:00:58 2016 -0700
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.PrintStream;
+
+public class App {
+
+    public static void main(String args[]) throws Exception {
+        (new App()).run(args, System.out);
+    }
+
+    public void run(String args[], PrintStream out) throws Exception {
+        out.println("App: Test started");
+        if (CustomLoader.agentClassLoader != CustomLoader.myself) {
+            System.out.println("App: agentClassLoader:    " + CustomLoader.agentClassLoader);
+            System.out.println("App: CustomLoader.myself: " + CustomLoader.myself);
+            System.out.println("App: myClassLoader:       " + App.class.getClassLoader());
+            throw new Exception("App: Agent's system class loader is not CustomLoader");
+        } else if (Agent.failed) {
+            throw new Exception("App: Agent failed");
+        } else if (CustomLoader.failed) {
+            throw new Exception("App: CustomLoader failed");
+        }
+        out.println("App: Test passed");
+    }
+}

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/java/lang/instrument/CustomSystemLoader/CustomLoader.java	Mon Sep 12 15:00:58 2016 -0700
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+import java.io.DataInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.jar.JarFile;
+import java.util.zip.ZipEntry;
+
+public class CustomLoader extends ClassLoader {
+    private static PrintStream out = System.out;
+    public  static ClassLoader myself;
+    public  static ClassLoader agentClassLoader;
+    public  static boolean failed = true;
+
+    public CustomLoader(ClassLoader classLoader) {
+        super(classLoader);
+        myself = this;
+    }
+
+    @Override
+    public Class<?> loadClass(String name) throws ClassNotFoundException {
+        out.println("CustomLoader: loading class: " + name);
+        if (name.equals("Agent")) {
+            Class c = null;
+            try {
+                byte[] buf = locateBytes();
+                c = defineClass(name, buf, 0, buf.length);
+            } catch (IOException ex) {
+                throw new ClassNotFoundException(ex.getMessage());
+            }
+            resolveClass(c);
+            out.println("CustomLoader.loadClass after resolveClass: " + name +
+                        "; Class: " + c + "; ClassLoader: " + c.getClassLoader());
+            return c;
+        }
+        return super.loadClass(name);
+    }
+
+    private byte[] locateBytes() throws IOException {
+        try {
+            JarFile jar = new JarFile("Agent.jar");
+            InputStream is = jar.getInputStream(jar.getEntry("Agent.class"));
+            int len = is.available();
+            byte[] buf = new byte[len];
+            DataInputStream in = new DataInputStream(is);
+            in.readFully(buf);
+            return buf;
+        } catch (IOException ioe) {
+            ioe.printStackTrace();
+            throw new IOException("Test failed due to IOException!");
+        }
+    }
+
+    void appendToClassPathForInstrumentation(String path) {
+        out.println("CustomLoader.appendToClassPathForInstrumentation: " +
+                    this + ", jar: " + path);
+        failed = false;
+    }
+}