6843127: krb5 should not try to access unavailable kdc too often
authorweijun
Thu, 24 Dec 2009 13:56:19 +0800
changeset 4531 3a9206343ab2
parent 4530 cff832a17f52
child 4532 f39917c8cf46
6843127: krb5 should not try to access unavailable kdc too often Reviewed-by: valeriep, mullan
jdk/src/share/classes/sun/security/krb5/Config.java
jdk/src/share/classes/sun/security/krb5/KrbKdcReq.java
jdk/src/share/lib/security/java.security
jdk/test/sun/security/krb5/auto/BadKdc.java
jdk/test/sun/security/krb5/auto/BadKdc1.java
jdk/test/sun/security/krb5/auto/BadKdc2.java
jdk/test/sun/security/krb5/auto/BadKdc3.java
jdk/test/sun/security/krb5/auto/BadKdc4.java
jdk/test/sun/security/krb5/auto/KDC.java
jdk/test/sun/security/krb5/auto/OneKDC.java
--- a/jdk/src/share/classes/sun/security/krb5/Config.java	Wed Dec 23 15:57:14 2009 -0800
+++ b/jdk/src/share/classes/sun/security/krb5/Config.java	Thu Dec 24 13:56:19 2009 +0800
@@ -109,6 +109,7 @@
     public static synchronized void refresh() throws KrbException {
         singleton = new Config();
         KeyTab.refresh();
+        KrbKdcReq.KdcAccessibility.reset();
     }
 
 
--- a/jdk/src/share/classes/sun/security/krb5/KrbKdcReq.java	Wed Dec 23 15:57:14 2009 -0800
+++ b/jdk/src/share/classes/sun/security/krb5/KrbKdcReq.java	Thu Dec 24 13:56:19 2009 +0800
@@ -31,25 +31,26 @@
 
 package sun.security.krb5;
 
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.security.Security;
+import java.util.Locale;
 import sun.security.krb5.internal.Krb5;
 import sun.security.krb5.internal.UDPClient;
 import sun.security.krb5.internal.TCPClient;
 import java.io.IOException;
-import java.io.InterruptedIOException;
 import java.net.SocketTimeoutException;
-import java.net.UnknownHostException;
 import java.util.StringTokenizer;
 import java.security.AccessController;
 import java.security.PrivilegedExceptionAction;
 import java.security.PrivilegedActionException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+import java.util.HashSet;
 
 public abstract class KrbKdcReq {
 
-    /**
-     * Default port for a KDC.
-     */
-    private static final int DEFAULT_KDC_PORT = Krb5.KDC_INET_DEFAULT_PORT;
-
     // Currently there is no option to specify retries
     // in the kerberos configuration file
 
@@ -66,7 +67,48 @@
 
     private static int udpPrefLimit = -1;
 
+    private static final String BAD_POLICY_KEY = "krb5.kdc.bad.policy";
+
+    /**
+     * What to do when a KDC is unavailable, specified in the
+     * java.security file with key krb5.kdc.bad.policy.
+     * Possible values can be TRY_LAST or TRY_LESS
+     */
+    private enum BpType {
+        NONE, TRY_LAST, TRY_LESS
+    }
+    private static int tryLessMaxRetries = 1;
+    private static int tryLessTimeout = 5000;
+
+    private static final BpType badPolicy;
+
     static {
+        String value = AccessController.doPrivileged(
+        new PrivilegedAction<String>() {
+            public String run() {
+                return Security.getProperty(BAD_POLICY_KEY);
+            }
+        });
+        if (value != null) {
+            value = value.toLowerCase(Locale.ENGLISH);
+            String[] ss = value.split(":");
+            if ("tryless".equals(ss[0])) {
+                if (ss.length > 1) {
+                    String[] params = ss[1].split(",");
+                    tryLessMaxRetries = Integer.parseInt(params[0]);
+                    if (params.length > 1) {
+                        tryLessTimeout = Integer.parseInt(params[1]);
+                    }
+                }
+                badPolicy = BpType.TRY_LESS;
+            } else if ("trylast".equals(ss[0])) {
+                badPolicy = BpType.TRY_LAST;
+            } else {
+                badPolicy = BpType.NONE;
+            }
+        } else {
+            badPolicy = BpType.NONE;
+        }
 
         /*
          * Get default timeout.
@@ -131,22 +173,16 @@
             }
         }
 
-        /*
-         * Get timeout.
-         */
-
-        int timeout = getKdcTimeout(realm);
-
         String kdcList = cfg.getKDCList(realm);
         if (kdcList == null) {
             throw new KrbException("Cannot get kdc for realm " + realm);
         }
         String tempKdc = null; // may include the port number also
-        StringTokenizer st = new StringTokenizer(kdcList);
-        while (st.hasMoreTokens()) {
-            tempKdc = st.nextToken();
+        for (String tmp: KdcAccessibility.list(kdcList)) {
+            tempKdc = tmp;
             try {
                 send(realm,tempKdc,useTCP);
+                KdcAccessibility.removeBad(tempKdc);
                 break;
             } catch (Exception e) {
                 if (DEBUG) {
@@ -154,6 +190,7 @@
                             tempKdc);
                     e.printStackTrace(System.out);
                 }
+                KdcAccessibility.addBad(tempKdc);
                 savedException = e;
             }
         }
@@ -174,16 +211,21 @@
 
         if (obuf == null)
             return;
-        PrivilegedActionException savedException = null;
+
         int port = Krb5.KDC_INET_DEFAULT_PORT;
+        int retries = DEFAULT_KDC_RETRY_LIMIT;
+        int timeout = getKdcTimeout(realm);
 
-        /*
-         * Get timeout.
-         */
-        int timeout = getKdcTimeout(realm);
-        /*
-         * Get port number for this KDC.
-         */
+        if (badPolicy == BpType.TRY_LESS &&
+                KdcAccessibility.isBad(tempKdc)) {
+            if (retries > tryLessMaxRetries) {
+                retries = tryLessMaxRetries; // less retries
+            }
+            if (timeout > tryLessTimeout) {
+                timeout = tryLessTimeout; // less time
+            }
+        }
+
         String kdc = null;
         String portStr = null;
 
@@ -225,12 +267,12 @@
                                +  port +  ", timeout="
                                + timeout
                                + ", number of retries ="
-                               + DEFAULT_KDC_RETRY_LIMIT
+                               + retries
                                + ", #bytes=" + obuf.length);
         }
 
         KdcCommunication kdcCommunication =
-            new KdcCommunication(kdc, port, useTCP, timeout, obuf);
+            new KdcCommunication(kdc, port, useTCP, timeout, retries, obuf);
         try {
             ibuf = AccessController.doPrivileged(kdcCommunication);
             if (DEBUG) {
@@ -258,14 +300,16 @@
         private int port;
         private boolean useTCP;
         private int timeout;
+        private int retries;
         private byte[] obuf;
 
         public KdcCommunication(String kdc, int port, boolean useTCP,
-                                int timeout, byte[] obuf) {
+                                int timeout, int retries, byte[] obuf) {
             this.kdc = kdc;
             this.port = port;
             this.useTCP = useTCP;
             this.timeout = timeout;
+            this.retries = retries;
             this.obuf = obuf;
         }
 
@@ -294,7 +338,7 @@
             } else {
                 // For each KDC we try DEFAULT_KDC_RETRY_LIMIT (3) times to
                 // get the response
-                for (int i=1; i <= DEFAULT_KDC_RETRY_LIMIT; i++) {
+                for (int i=1; i <= retries; i++) {
                     UDPClient kdcClient = new UDPClient(kdc, port, timeout);
 
                     if (DEBUG) {
@@ -310,7 +354,7 @@
                          * Send the data to the kdc.
                          */
 
-                    kdcClient.send(obuf);
+                        kdcClient.send(obuf);
 
                         /*
                          * And get a response.
@@ -323,7 +367,7 @@
                                 System.out.println ("SocketTimeOutException with " +
                                                     "attempt: " + i);
                             }
-                            if (i == DEFAULT_KDC_RETRY_LIMIT) {
+                            if (i == retries) {
                                 ibuf = null;
                                 throw se;
                             }
@@ -385,4 +429,67 @@
 
         return -1;
     }
+
+    /**
+     * Maintains a KDC accessible list. Unavailable KDCs are put into a
+     * blacklist, when a KDC in the blacklist is available, it's removed
+     * from there. No insertion order in the blacklist.
+     *
+     * There are two methods to deal with KDCs in the blacklist. 1. Only try
+     * them when there's no KDC not on the blacklist. 2. Still try them, but
+     * with lesser number of retries and smaller timeout value.
+     */
+    static class KdcAccessibility {
+        // Known bad KDCs
+        private static Set<String> bads = new HashSet<String>();
+
+        private static synchronized void addBad(String kdc) {
+            if (DEBUG) {
+                System.out.println(">>> KdcAccessibility: add " + kdc);
+            }
+            bads.add(kdc);
+        }
+
+        private static synchronized void removeBad(String kdc) {
+            if (DEBUG) {
+                System.out.println(">>> KdcAccessibility: remove " + kdc);
+            }
+            bads.remove(kdc);
+        }
+
+        private static synchronized boolean isBad(String kdc) {
+            return bads.contains(kdc);
+        }
+
+        public static synchronized void reset() {
+            if (DEBUG) {
+                System.out.println(">>> KdcAccessibility: reset");
+            }
+            bads.clear();
+        }
+
+        // Returns a preferred KDC list by putting the bad ones at the end
+        private static synchronized String[] list(String kdcList) {
+            StringTokenizer st = new StringTokenizer(kdcList);
+            List<String> list = new ArrayList<String>();
+            if (badPolicy == BpType.TRY_LAST) {
+                List<String> badkdcs = new ArrayList<String>();
+                while (st.hasMoreTokens()) {
+                    String t = st.nextToken();
+                    if (bads.contains(t)) badkdcs.add(t);
+                    else list.add(t);
+                }
+                // Bad KDCs are put at last
+                list.addAll(badkdcs);
+            } else {
+                // All KDCs are returned in their original order,
+                // This include TRY_LESS and NONE
+                while (st.hasMoreTokens()) {
+                    list.add(st.nextToken());
+                }
+            }
+            return list.toArray(new String[list.size()]);
+        }
+    }
 }
+
--- a/jdk/src/share/lib/security/java.security	Wed Dec 23 15:57:14 2009 -0800
+++ b/jdk/src/share/lib/security/java.security	Thu Dec 24 13:56:19 2009 +0800
@@ -55,10 +55,10 @@
 
 #
 # Select the source of seed data for SecureRandom. By default an
-# attempt is made to use the entropy gathering device specified by 
+# attempt is made to use the entropy gathering device specified by
 # the securerandom.source property. If an exception occurs when
-# accessing the URL then the traditional system/thread activity 
-# algorithm is used. 
+# accessing the URL then the traditional system/thread activity
+# algorithm is used.
 #
 # On Solaris and Linux systems, if file:/dev/urandom is specified and it
 # exists, a special SecureRandom implementation is activated by default.
@@ -72,7 +72,7 @@
 # The entropy gathering device is described as a URL and can also
 # be specified with the system property "java.security.egd". For example,
 #   -Djava.security.egd=file:/dev/urandom
-# Specifying this system property will override the securerandom.source 
+# Specifying this system property will override the securerandom.source
 # setting.
 
 #
@@ -149,7 +149,7 @@
 security.overridePropertiesFile=true
 
 #
-# Determines the default key and trust manager factory algorithms for 
+# Determines the default key and trust manager factory algorithms for
 # the javax.net.ssl package.
 #
 ssl.KeyManagerFactory.algorithm=SunX509
@@ -168,10 +168,10 @@
 # is to cache for 30 seconds.
 #
 # NOTE: setting this to anything other than the default value can have
-#       serious security implications. Do not set it unless 
+#       serious security implications. Do not set it unless
 #       you are sure you are not exposed to DNS spoofing attack.
 #
-#networkaddress.cache.ttl=-1 
+#networkaddress.cache.ttl=-1
 
 # The Java-level namelookup cache policy for failed lookups:
 #
@@ -183,7 +183,7 @@
 # the WINS name service in addition to DNS, name service lookups
 # that fail may take a noticeably long time to return (approx. 5 seconds).
 # For this reason the default caching policy is to maintain these
-# results for 10 seconds. 
+# results for 10 seconds.
 #
 #
 networkaddress.cache.negative.ttl=10
@@ -192,7 +192,7 @@
 # Properties to configure OCSP for certificate revocation checking
 #
 
-# Enable OCSP 
+# Enable OCSP
 #
 # By default, OCSP is not used for certificate revocation checking.
 # This property enables the use of OCSP when set to the value "true".
@@ -201,7 +201,7 @@
 #
 # Example,
 #   ocsp.enable=true
- 
+
 #
 # Location of the OCSP responder
 #
@@ -213,15 +213,15 @@
 #
 # Example,
 #   ocsp.responderURL=http://ocsp.example.net:80
- 
+
 #
 # Subject name of the OCSP responder's certificate
 #
 # By default, the certificate of the OCSP responder is that of the issuer
 # of the certificate being validated. This property identifies the certificate
-# of the OCSP responder when the default does not apply. Its value is a string 
-# distinguished name (defined in RFC 2253) which identifies a certificate in 
-# the set of certificates supplied during cert path validation. In cases where 
+# of the OCSP responder when the default does not apply. Its value is a string
+# distinguished name (defined in RFC 2253) which identifies a certificate in
+# the set of certificates supplied during cert path validation. In cases where
 # the subject name alone is not sufficient to uniquely identify the certificate
 # then both the "ocsp.responderCertIssuerName" and
 # "ocsp.responderCertSerialNumber" properties must be used instead. When this
@@ -237,14 +237,14 @@
 # of the certificate being validated. This property identifies the certificate
 # of the OCSP responder when the default does not apply. Its value is a string
 # distinguished name (defined in RFC 2253) which identifies a certificate in
-# the set of certificates supplied during cert path validation. When this 
-# property is set then the "ocsp.responderCertSerialNumber" property must also 
-# be set. When the "ocsp.responderCertSubjectName" property is set then this 
+# the set of certificates supplied during cert path validation. When this
+# property is set then the "ocsp.responderCertSerialNumber" property must also
+# be set. When the "ocsp.responderCertSubjectName" property is set then this
 # property is ignored.
 #
 # Example,
 #   ocsp.responderCertIssuerName="CN=Enterprise CA, O=XYZ Corp"
- 
+
 #
 # Serial number of the OCSP responder's certificate
 #
@@ -259,4 +259,31 @@
 #
 # Example,
 #   ocsp.responderCertSerialNumber=2A:FF:00
- 
+
+#
+# Policy for failed Kerberos KDC lookups:
+#
+# When a KDC is unavailable (network error, service failure, etc), it is
+# put inside a blacklist and accessed less often for future requests. The
+# value (case-insensitive) for this policy can be:
+#
+# tryLast
+#    KDCs in the blacklist are always tried after those not on the list.
+#
+# tryLess[:max_retries,timeout]
+#    KDCs in the blacklist are still tried by their order in the configuration,
+#    but with smaller max_retries and timeout values. max_retries and timeout
+#    are optional numerical parameters (default 1 and 5000, which means once
+#    and 5 seconds). Please notes that if any of the values defined here is
+#    more than what is defined in krb5.conf, it will be ignored.
+#
+# Whenever a KDC is detected as available, it is removed from the blacklist.
+# The blacklist is reset when krb5.conf is reloaded. You can add
+# refreshKrb5Config=true to a JAAS configuration file so that krb5.conf is
+# reloaded whenever a JAAS authentication is attempted.
+#
+# Example,
+#   krb5.kdc.bad.policy = tryLast
+#   krb5.kdc.bad.policy = tryLess:2,2000
+krb5.kdc.bad.policy = tryLast
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/sun/security/krb5/auto/BadKdc.java	Thu Dec 24 13:56:19 2009 +0800
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+import java.io.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import sun.security.krb5.Config;
+
+public class BadKdc {
+
+    // Matches the krb5 debug output:
+    // >>> KDCCommunication: kdc=kdc.rabbit.hole UDP:14319, timeout=2000,...
+    //                                               ^ kdc#         ^ timeout
+    static final Pattern re = Pattern.compile(
+            ">>> KDCCommunication: kdc=kdc.rabbit.hole UDP:(\\d)...., " +
+            "timeout=(\\d)000,");
+    public static void go(int[]... expected)
+            throws Exception {
+        System.setProperty("sun.security.krb5.debug", "true");
+
+        // Make sure KDCs' ports starts with 1 and 2 and 3,
+        // useful for checking debug output.
+        int p1 = 10000 + new java.util.Random().nextInt(10000);
+        int p2 = 20000 + new java.util.Random().nextInt(10000);
+        int p3 = 30000 + new java.util.Random().nextInt(10000);
+
+        FileWriter fw = new FileWriter("alternative-krb5.conf");
+
+        fw.write("[libdefaults]\n" +
+                "default_realm = " + OneKDC.REALM + "\n" +
+                "kdc_timeout = 2000\n");
+        fw.write("[realms]\n" + OneKDC.REALM + " = {\n" +
+                "kdc = " + OneKDC.KDCHOST + ":" + p1 + "\n" +
+                "kdc = " + OneKDC.KDCHOST + ":" + p2 + "\n" +
+                "kdc = " + OneKDC.KDCHOST + ":" + p3 + "\n" +
+                "}\n");
+
+        fw.close();
+        System.setProperty("java.security.krb5.conf", "alternative-krb5.conf");
+        Config.refresh();
+
+        // Turn on k3 only
+        KDC k3 = on(p3);
+
+        test(expected[0]);
+        test(expected[1]);
+        Config.refresh();
+        test(expected[2]);
+
+        k3.terminate(); // shutdown k3
+        on(p2);         // k2 is on
+        test(expected[3]);
+        on(p1);         // k1 and k2 is on
+        test(expected[4]);
+    }
+
+    private static KDC on(int p) throws Exception {
+        KDC k = new KDC(OneKDC.REALM, OneKDC.KDCHOST, p, true);
+        k.addPrincipal(OneKDC.USER, OneKDC.PASS);
+        k.addPrincipalRandKey("krbtgt/" + OneKDC.REALM);
+        return k;
+    }
+
+    /**
+     * One round of test for max_retries and timeout.
+     * @param timeout the expected timeout
+     * @param expected the expected kdc# timeout kdc# timeout...
+     */
+    private static void test(int... expected) throws Exception {
+        ByteArrayOutputStream bo = new ByteArrayOutputStream();
+        PrintStream oldout = System.out;
+        System.setOut(new PrintStream(bo));
+        Context c = Context.fromUserPass(OneKDC.USER, OneKDC.PASS, false);
+        System.setOut(oldout);
+
+        String[] lines = new String(bo.toByteArray()).split("\n");
+        System.out.println("----------------- TEST -----------------");
+        int count = 0;
+        for (String line: lines) {
+            Matcher m = re.matcher(line);
+            if (m.find()) {
+                System.out.println(line);
+                if (Integer.parseInt(m.group(1)) != expected[count++] ||
+                        Integer.parseInt(m.group(2)) != expected[count++]) {
+                    throw new Exception("Fail here");
+                }
+            }
+        }
+        if (count != expected.length) {
+            throw new Exception("Less rounds");
+        }
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/sun/security/krb5/auto/BadKdc1.java	Thu Dec 24 13:56:19 2009 +0800
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6843127
+ * @run main/timeout=300 BadKdc1
+ * @summary krb5 should not try to access unavailable kdc too often
+ */
+
+import java.io.*;
+import java.security.Security;
+
+public class BadKdc1 {
+
+   public static void main(String[] args)
+           throws Exception {
+       Security.setProperty("krb5.kdc.bad.policy", "tryLess");
+       BadKdc.go(
+               new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,2,2,2,3,2}, // 1, 2
+               // The above line means try kdc1 for 2 seconds, then kdc1
+               // for 2 seconds,..., finally kdc3 for 2 seconds.
+               new int[]{1,2,2,2,3,2,1,2,2,2,3,2}, // 1, 2
+               // refresh
+               new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,2,2,2,3,2}, // 1, 2
+               // k3 off, k2 on
+               new int[]{1,2,2,2,1,2,2,2}, // 1
+               // k1 on
+               new int[]{1,2,1,2}  // empty
+       );
+   }
+}
+
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/sun/security/krb5/auto/BadKdc2.java	Thu Dec 24 13:56:19 2009 +0800
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6843127
+ * @run main/timeout=300 BadKdc2
+ * @summary krb5 should not try to access unavailable kdc too often
+ */
+
+import java.io.*;
+import java.security.Security;
+
+public class BadKdc2 {
+
+    public static void main(String[] args)
+            throws Exception {
+        Security.setProperty("krb5.kdc.bad.policy", "tryLess:2,1000");
+        BadKdc.go(
+                new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,1,1,1,2,1,2,1,3,2}, // 1, 2
+                new int[]{1,1,1,1,2,1,2,1,3,2,1,1,1,1,2,1,2,1,3,2}, // 1, 2
+                // refresh
+                new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,1,1,1,2,1,2,1,3,2}, // 1, 2
+                // k3 off, k2 on
+                new int[]{1,1,1,1,2,1,1,1,1,1,2,2}, // 1
+                // k1 on
+                new int[]{1,1,1,2}  // empty
+        );
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/sun/security/krb5/auto/BadKdc3.java	Thu Dec 24 13:56:19 2009 +0800
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6843127
+ * @run main/timeout=300 BadKdc3
+ * @summary krb5 should not try to access unavailable kdc too often
+ */
+
+import java.io.*;
+import java.security.Security;
+
+public class BadKdc3 {
+
+    public static void main(String[] args)
+            throws Exception {
+        Security.setProperty("krb5.kdc.bad.policy", "tryLast");
+        BadKdc.go(
+                new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,3,2}, // 1, 2
+                new int[]{3,2,3,2}, // 1, 2
+                // refresh
+                new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,3,2}, // 1, 2
+                // k3 off, k2 on
+                new int[]{3,2,3,2,3,2,1,2,1,2,1,2,2,2,2,2}, // 1, 3
+                // k1 on
+                new int[]{2,2,2,2}  // 1, 3
+        );
+    }
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/jdk/test/sun/security/krb5/auto/BadKdc4.java	Thu Dec 24 13:56:19 2009 +0800
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug 6843127
+ * @run main/timeout=300 BadKdc4
+ * @summary krb5 should not try to access unavailable kdc too often
+ */
+
+import java.io.*;
+import java.security.Security;
+
+public class BadKdc4 {
+
+    public static void main(String[] args)
+            throws Exception {
+        Security.setProperty("krb5.kdc.bad.policy", "");
+        BadKdc.go(
+            new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,2,1,2,1,2,2,2,2,2,2,2,3,2},
+            new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,2,1,2,1,2,2,2,2,2,2,2,3,2},
+            // refresh
+            new int[]{1,2,1,2,1,2,2,2,2,2,2,2,3,2,1,2,1,2,1,2,2,2,2,2,2,2,3,2},
+            // k3 off, k2 on
+            new int[]{1,2,1,2,1,2,2,2,1,2,1,2,1,2,2,2},
+            // k1 on
+            new int[]{1,2,1,2}
+        );
+    }
+}
--- a/jdk/test/sun/security/krb5/auto/KDC.java	Wed Dec 23 15:57:14 2009 -0800
+++ b/jdk/test/sun/security/krb5/auto/KDC.java	Thu Dec 24 13:56:19 2009 +0800
@@ -141,6 +141,10 @@
     // Options
     private Map<Option,Object> options = new HashMap<Option,Object>();
 
+    private Thread thread1, thread2, thread3;
+    DatagramSocket u1 = null;
+    ServerSocket t1 = null;
+
     /**
      * Option names, to be expanded forever.
      */
@@ -940,8 +944,6 @@
      * @throws java.io.IOException for any communication error
      */
     protected void startServer(int port, boolean asDaemon) throws IOException {
-        DatagramSocket u1 = null;
-        ServerSocket t1 = null;
         if (port > 0) {
             u1 = new DatagramSocket(port, InetAddress.getByName("127.0.0.1"));
             t1 = new ServerSocket(port);
@@ -966,7 +968,7 @@
         this.port = port;
 
         // The UDP consumer
-        Thread thread = new Thread() {
+        thread1 = new Thread() {
             public void run() {
                 while (true) {
                     try {
@@ -982,11 +984,11 @@
                 }
             }
         };
-        thread.setDaemon(asDaemon);
-        thread.start();
+        thread1.setDaemon(asDaemon);
+        thread1.start();
 
         // The TCP consumer
-        thread = new Thread() {
+        thread2 = new Thread() {
             public void run() {
                 while (true) {
                     try {
@@ -1004,11 +1006,11 @@
                 }
             }
         };
-        thread.setDaemon(asDaemon);
-        thread.start();
+        thread2.setDaemon(asDaemon);
+        thread2.start();
 
         // The dispatcher
-        thread = new Thread() {
+        thread3 = new Thread() {
             public void run() {
                 while (true) {
                     try {
@@ -1018,10 +1020,21 @@
                 }
             }
         };
-        thread.setDaemon(true);
-        thread.start();
+        thread3.setDaemon(true);
+        thread3.start();
     }
 
+    public void terminate() {
+        try {
+            thread1.stop();
+            thread2.stop();
+            thread3.stop();
+            u1.close();
+            t1.close();
+        } catch (Exception e) {
+            // OK
+        }
+    }
     /**
      * Helper class to encapsulate a job in a KDC.
      */
--- a/jdk/test/sun/security/krb5/auto/OneKDC.java	Wed Dec 23 15:57:14 2009 -0800
+++ b/jdk/test/sun/security/krb5/auto/OneKDC.java	Thu Dec 24 13:56:19 2009 +0800
@@ -24,8 +24,6 @@
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.security.Security;
 import javax.security.auth.callback.Callback;
 import javax.security.auth.callback.CallbackHandler;