8225035: Thread stack size issue caused by large TLS size
Summary: Adjust thread stack size for static TLS on Linux when AdjustStackSizeForTLS is enabled.
Reviewed-by: dholmes, fweimer, stuefe, rriggs, martin
Contributed-by: jeremymanson@google.com, fweimer@redhat.com, jianglizhou@google.com
--- a/make/test/JtregNativeHotspot.gmk Mon Jul 08 13:29:02 2019 +0200
+++ b/make/test/JtregNativeHotspot.gmk Tue Jul 09 10:27:38 2019 -0700
@@ -862,12 +862,13 @@
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libredefineClasses := -lpthread
BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeinvoke := -ljvm -lpthread
BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-gap := -ljvm -lpthread
+ BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exestack-tls := -ljvm
BUILD_TEST_exeinvoke_exeinvoke.c_OPTIMIZATION := NONE
BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exeFPRegs := -ldl
BUILD_HOTSPOT_JTREG_LIBRARIES_LIBS_libAsyncGetCallTraceTest := -ldl
else
BUILD_HOTSPOT_JTREG_EXCLUDE += libtest-rw.c libtest-rwx.c libTestJNI.c \
- exeinvoke.c exestack-gap.c libAsyncGetCallTraceTest.cpp
+ exeinvoke.c exestack-gap.c exestack-tls.c libAsyncGetCallTraceTest.cpp
endif
BUILD_HOTSPOT_JTREG_EXECUTABLES_LIBS_exesigtest := -ljvm
--- a/src/hotspot/os/linux/globals_linux.hpp Mon Jul 08 13:29:02 2019 +0200
+++ b/src/hotspot/os/linux/globals_linux.hpp Tue Jul 09 10:27:38 2019 -0700
@@ -64,9 +64,13 @@
\
product(bool, PreferContainerQuotaForCPUCount, true, \
"Calculate the container CPU availability based on the value" \
- " of quotas (if set), when true. Otherwise, use the CPU" \
+ " of quotas (if set), when true. Otherwise, use the CPU" \
" shares value, provided it is less than quota.") \
\
+ product(bool, AdjustStackSizeForTLS, false, \
+ "Increase the thread stack size to include space for glibc " \
+ "static thread-local storage (TLS) if true") \
+ \
diagnostic(bool, DumpPrivateMappingsInCore, true, \
"If true, sets bit 2 of /proc/PID/coredump_filter, thus " \
"resulting in file-backed private mappings of the process to "\
--- a/src/hotspot/os/linux/os_linux.cpp Mon Jul 08 13:29:02 2019 +0200
+++ b/src/hotspot/os/linux/os_linux.cpp Tue Jul 09 10:27:38 2019 -0700
@@ -801,6 +801,73 @@
return 0;
}
+// On Linux, glibc places static TLS blocks (for __thread variables) on
+// the thread stack. This decreases the stack size actually available
+// to threads.
+//
+// For large static TLS sizes, this may cause threads to malfunction due
+// to insufficient stack space. This is a well-known issue in glibc:
+// http://sourceware.org/bugzilla/show_bug.cgi?id=11787.
+//
+// As a workaround, we call a private but assumed-stable glibc function,
+// __pthread_get_minstack() to obtain the minstack size and derive the
+// static TLS size from it. We then increase the user requested stack
+// size by this TLS size.
+//
+// Due to compatibility concerns, this size adjustment is opt-in and
+// controlled via AdjustStackSizeForTLS.
+typedef size_t (*GetMinStack)(const pthread_attr_t *attr);
+
+GetMinStack _get_minstack_func = NULL;
+
+static void get_minstack_init() {
+ _get_minstack_func =
+ (GetMinStack)dlsym(RTLD_DEFAULT, "__pthread_get_minstack");
+ log_info(os, thread)("Lookup of __pthread_get_minstack %s",
+ _get_minstack_func == NULL ? "failed" : "succeeded");
+}
+
+// Returns the size of the static TLS area glibc puts on thread stacks.
+// The value is cached on first use, which occurs when the first thread
+// is created during VM initialization.
+static size_t get_static_tls_area_size(const pthread_attr_t *attr) {
+ size_t tls_size = 0;
+ if (_get_minstack_func != NULL) {
+ // Obtain the pthread minstack size by calling __pthread_get_minstack.
+ size_t minstack_size = _get_minstack_func(attr);
+
+ // Remove non-TLS area size included in minstack size returned
+ // by __pthread_get_minstack() to get the static TLS size.
+ // In glibc before 2.27, minstack size includes guard_size.
+ // In glibc 2.27 and later, guard_size is automatically added
+ // to the stack size by pthread_create and is no longer included
+ // in minstack size. In both cases, the guard_size is taken into
+ // account, so there is no need to adjust the result for that.
+ //
+ // Although __pthread_get_minstack() is a private glibc function,
+ // it is expected to have a stable behavior across future glibc
+ // versions while glibc still allocates the static TLS blocks off
+ // the stack. Following is glibc 2.28 __pthread_get_minstack():
+ //
+ // size_t
+ // __pthread_get_minstack (const pthread_attr_t *attr)
+ // {
+ // return GLRO(dl_pagesize) + __static_tls_size + PTHREAD_STACK_MIN;
+ // }
+ //
+ //
+ // The following 'minstack_size > os::vm_page_size() + PTHREAD_STACK_MIN'
+ // if check is done for precaution.
+ if (minstack_size > (size_t)os::vm_page_size() + PTHREAD_STACK_MIN) {
+ tls_size = minstack_size - os::vm_page_size() - PTHREAD_STACK_MIN;
+ }
+ }
+
+ log_info(os, thread)("Stack size adjustment for TLS is " SIZE_FORMAT,
+ tls_size);
+ return tls_size;
+}
+
bool os::create_thread(Thread* thread, ThreadType thr_type,
size_t req_stack_size) {
assert(thread->osthread() == NULL, "caller responsible");
@@ -826,7 +893,7 @@
// Calculate stack size if it's not specified by caller.
size_t stack_size = os::Posix::get_initial_stack_size(thr_type, req_stack_size);
- // In the Linux NPTL pthread implementation the guard size mechanism
+ // In glibc versions prior to 2.7 the guard size mechanism
// is not implemented properly. The posix standard requires adding
// the size of the guard pages to the stack size, instead Linux
// takes the space out of 'stacksize'. Thus we adapt the requested
@@ -834,17 +901,27 @@
// behaviour. However, be careful not to end up with a size
// of zero due to overflow. Don't add the guard page in that case.
size_t guard_size = os::Linux::default_guard_size(thr_type);
- if (stack_size <= SIZE_MAX - guard_size) {
- stack_size += guard_size;
+ // Configure glibc guard page. Must happen before calling
+ // get_static_tls_area_size(), which uses the guard_size.
+ pthread_attr_setguardsize(&attr, guard_size);
+
+ size_t stack_adjust_size = 0;
+ if (AdjustStackSizeForTLS) {
+ // Adjust the stack_size for on-stack TLS - see get_static_tls_area_size().
+ stack_adjust_size += get_static_tls_area_size(&attr);
+ } else {
+ stack_adjust_size += guard_size;
+ }
+
+ stack_adjust_size = align_up(stack_adjust_size, os::vm_page_size());
+ if (stack_size <= SIZE_MAX - stack_adjust_size) {
+ stack_size += stack_adjust_size;
}
assert(is_aligned(stack_size, os::vm_page_size()), "stack_size not aligned");
int status = pthread_attr_setstacksize(&attr, stack_size);
assert_status(status == 0, status, "pthread_attr_setstacksize");
- // Configure glibc guard page.
- pthread_attr_setguardsize(&attr, os::Linux::default_guard_size(thr_type));
-
ThreadState state;
{
@@ -5145,6 +5222,10 @@
jdk_misc_signal_init();
}
+ if (AdjustStackSizeForTLS) {
+ get_minstack_init();
+ }
+
// Check and sets minimum stack sizes against command line options
if (Posix::set_minimum_stack_sizes() == JNI_ERR) {
return JNI_ERR;
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/TLS/T.java Tue Jul 09 10:27:38 2019 -0700
@@ -0,0 +1,73 @@
+/*
+ * Copyright (c) 2019, Google Inc. All rights reserved.
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+import java.lang.ProcessBuilder;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+public class T {
+ public static boolean run() {
+ boolean res = false;
+ String echoInput = "foo";
+ ProcessBuilder pb = new ProcessBuilder("echo", echoInput);
+
+ try {
+ // Starting a ProcessBuilder causes the process reaper thread to be
+ // created. The process reaper thread has small stack size. In JDK
+ // 13, the REAPER_DEFAULT_STACKSIZE is 128K. With JDK 8, it is 32K.
+ // Using the process reaper thread can demonstrate the TLS problem.
+ // The reaper thread can fail with StackOverflow in one of the
+ // failure mode with certain TLS sizes. In another observed
+ // failure mode the VM fails to create a thread with error message
+ // 'Failed to start thread - pthread_create failed'.
+ System.out.println("Starting a new process ...");
+ Process process = pb.start();
+ process.waitFor();
+ String echoOutput = output(process.getInputStream());
+ System.out.println("Echo Output: " + echoOutput);
+ if (echoOutput.equals(echoInput)) {
+ res = true;
+ } else {
+ // 'res' is false, fail
+ System.out.println("Unexpected Echo output: " + echoOutput +
+ ", expects: " + echoInput);
+ }
+ } catch (Exception e) {
+ System.out.println(e.toString());
+ e.printStackTrace();
+ }
+ return res;
+ }
+
+ private static String output(InputStream inputStream) throws IOException {
+ String s = "";
+ try (BufferedReader br =
+ new BufferedReader(new InputStreamReader(inputStream))) {
+ s = br.readLine();
+ }
+ return s;
+ }
+}
+
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/TLS/exestack-tls.c Tue Jul 09 10:27:38 2019 -0700
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2019, Google Inc. All rights reserved.
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <jni.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Declare the thread local variable(s) in the main executable. This can be
+// used to demonstrate the issues associated with the on-stack static TLS blocks
+// that may cause insufficient stack space. The dynamic TLS blocks for shared
+// objects (such as a JNI library) loaded via dlopen are not allocated on stack.
+__thread int tls[128 * 1024];
+
+JNIEnv* create_vm(JavaVM **jvm, char* argTLS) {
+ JNIEnv* env;
+ JavaVMInitArgs args;
+ JavaVMOption options[3];
+ args.version = JNI_VERSION_1_8;
+ args.nOptions = 3;
+ char classpath[4096];
+ snprintf(classpath, sizeof classpath,
+ "-Djava.class.path=%s", getenv("CLASSPATH"));
+ options[0].optionString = classpath;
+ options[1].optionString = "-Xlog:os+thread=info";
+ options[2].optionString = argTLS;
+ args.options = &options[0];
+ args.ignoreUnrecognized = 0;
+ int rv;
+ rv = JNI_CreateJavaVM(jvm, (void**)&env, &args);
+ if (rv < 0) return NULL;
+ return env;
+}
+
+int run(jboolean addTLS) {
+ JavaVM *jvm;
+ jclass testClass;
+ jmethodID runMethod;
+ char* argTLS;
+ int res = -1;
+
+ if (addTLS) {
+ argTLS = "-XX:+AdjustStackSizeForTLS";
+ } else {
+ argTLS = "-XX:-AdjustStackSizeForTLS"; // default
+ }
+ printf("Running test with %s ...\n", argTLS);
+ JNIEnv *env = create_vm(&jvm, argTLS);
+
+ // Run T.run() and check result:
+ // - Expect T.run() to return 'true' when stack size is adjusted for TLS,
+ // return 0 if so
+ // - Expect T.run() to return 'false' if stack size is not adjusted for
+ // TLS, return 0 if so
+ // Return -1 (fail) for other cases
+ testClass = (*env)->FindClass(env, "T");
+ runMethod = (*env)->GetStaticMethodID(env, testClass, "run", "()Z");
+ if ((*env)->CallStaticBooleanMethod(env, testClass, runMethod, NULL)) {
+ if (addTLS) {
+ // expect T.run() to return 'true'
+ res = 0;
+ }
+ } else {
+ if (!addTLS) {
+ // expect T.run() to return 'false'
+ res = 0;
+ }
+ }
+
+ if (res == 0) {
+ printf("Test passed with %s\n", argTLS);
+ } else {
+ printf("Test failed with %s\n", argTLS);
+ }
+ return res;
+}
+
+int main(int argc, char **argv) {
+ if (argc == 2 && strcmp(argv[1], "-add_tls") == 0) {
+ return run(JNI_TRUE);
+ } else {
+ return run(JNI_FALSE);
+ }
+}
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test/hotspot/jtreg/runtime/TLS/testtls.sh Tue Jul 09 10:27:38 2019 -0700
@@ -0,0 +1,47 @@
+# Copyright (c) 2019, Google Inc. All rights reserved.
+# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#!/bin/sh
+
+#
+# @test testtls.sh
+# @summary Test with extra TLS size.
+# @requires os.family == "linux"
+# @compile T.java
+# @run shell testtls.sh
+#
+
+if [ "${TESTSRC}" = "" ]
+then
+ TESTSRC=${PWD}
+ echo "TESTSRC not set. Using "${TESTSRC}" as default"
+fi
+echo "TESTSRC=${TESTSRC}"
+## Adding common setup Variables for running shell tests.
+. ${TESTSRC}/../../test_env.sh
+
+LD_LIBRARY_PATH=.:${TESTJAVA}/lib/${VM_TYPE}:/usr/lib:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH
+
+# Test 1) Run with stack size adjusted for TLS
+${TESTNATIVEPATH}/stack-tls -add_tls || exit $?
+# Test 2) Run with no stack size adjustment
+${TESTNATIVEPATH}/stack-tls || exit $?