8193061: Add run-test-prebuilt functionality
authorihse
Wed, 06 Dec 2017 13:45:46 +0100
changeset 48078 94a38ebbc9bb
parent 48077 2a749b997a0d
child 48079 eaa406097d93
8193061: Add run-test-prebuilt functionality Reviewed-by: erikj
make/Help.gmk
make/RunTestsPrebuilt.gmk
make/RunTestsPrebuiltSpec.gmk
make/common/MakeBase.gmk
--- a/make/Help.gmk	Wed Dec 06 02:09:08 2017 -0800
+++ b/make/Help.gmk	Wed Dec 06 13:45:46 2017 +0100
@@ -115,6 +115,13 @@
         # We need a dummy rule otherwise make will complain
 	@true
 
-ALL_GLOBAL_TARGETS := help print-configurations
+# This is not really a "help" target, but it is a global target, and those are
+# all contained in this file.
+run-test-prebuilt:
+	@( cd $(topdir) && \
+	    $(MAKE) --no-print-directory -r -R -I make/common/ -f make/RunTestsPrebuilt.gmk \
+	    run-test-prebuilt TEST="$(TEST)" )
+
+ALL_GLOBAL_TARGETS := help print-configurations run-test-prebuilt
 
 .PHONY: $(ALL_GLOBAL_TARGETS)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/make/RunTestsPrebuilt.gmk	Wed Dec 06 13:45:46 2017 +0100
@@ -0,0 +1,283 @@
+#
+# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# Initial bootstrapping, copied and stripped down from Makefile and Init.gmk
+################################################################################
+
+# In Cygwin, the MAKE variable gets prepended with the current directory if the
+# make executable is called using a Windows mixed path (c:/cygwin/bin/make.exe).
+ifneq ($(findstring :, $(MAKE)), )
+  export MAKE := $(patsubst $(CURDIR)%, %, $(patsubst $(CURDIR)/%, %, $(MAKE)))
+endif
+
+# Locate this Makefile
+ifeq ($(filter /%, $(lastword $(MAKEFILE_LIST))),)
+  makefile_path := $(CURDIR)/$(strip $(lastword $(MAKEFILE_LIST)))
+else
+  makefile_path := $(lastword $(MAKEFILE_LIST))
+endif
+TOPDIR := $(strip $(patsubst %/make/, %, $(dir $(makefile_path))))
+
+################################################################################
+# Functions
+################################################################################
+
+# Setup a required or optional variable, and/or check that it is properly
+# given.
+# Note: No spaces are allowed around the arguments.
+#
+# $1: The name of the argument
+# $2: The default value, if any, or OPTIONAL (do not provide a default but
+#     do not exit if it is missing)
+# $3: If NO_CHECK, disable checking for target file/directory existence
+define SetupVariable
+  ifeq ($$($1), )
+    ifeq ($2, )
+      $$(info Error: Prebuilt variable $1 is missing, needed for run-tests-prebuilt)
+      $$(error Cannot continue.)
+    else ifeq ($2, OPTIONAL)
+      ifneq ($$(findstring $$(LOG), info debug trace), )
+        $$(info Prebuilt variable $1 is not provided)
+      endif
+    else
+      ifneq ($$(findstring $$(LOG), info debug trace), )
+        $$(info Prebuilt variable $1=$2 (default value))
+      endif
+      $1:=$2
+    endif
+  else
+    ifneq ($$(findstring $$(LOG), info debug trace), )
+      $$(info Prebuilt variable $1=$$($1))
+    endif
+  endif
+  # If $1 has a value (is not optional), and $3 is not set (to NO_CHECK),
+  # and if wildcard is empty, then complain that the file is missing.
+  ifeq ($$(strip $$(if $$($1), , OPTIONAL) $$(wildcard $$($1)) $3), )
+    $$(info Error: Prebuilt variable $1 points to missing file/directory:)
+    $$(info '$$($1)')
+    $$(error Cannot continue.)
+  endif
+endef
+
+# Create an ephemeral spec file
+#
+# $1: The output file name
+# $2..$N: The lines to output to the file
+define CreateNewSpec
+  $(if $(strip $(26)), \
+    $(error Internal makefile error: \
+      Too many arguments to macro, please update CreateNewSpec in RunTestsPrebuilt.gmk) \
+  ) \
+  $(shell $(RM) $1) \
+  $(foreach i, $(call sequence, 2, 25), \
+    $(if $(strip $($i)), \
+      $(call AppendFile, $(strip $($i)), $1) \
+    ) \
+  )
+endef
+
+################################################################################
+# Check input and setup basic buildsystem support
+################################################################################
+
+# Verify that user has given correct additional input.
+
+# These variables are absolutely necessary
+$(eval $(call SetupVariable,OUTPUTDIR))
+$(eval $(call SetupVariable,BOOT_JDK))
+$(eval $(call SetupVariable,JT_HOME))
+
+# These can have default values based on the ones above
+$(eval $(call SetupVariable,JDK_IMAGE_DIR,$(OUTPUTDIR)/images/jdk))
+$(eval $(call SetupVariable,TEST_IMAGE_DIR,$(OUTPUTDIR)/images/test))
+
+# Provide default values for tools that we need
+$(eval $(call SetupVariable,MAKE,make,NO_CHECK))
+$(eval $(call SetupVariable,BASH,bash,NO_CHECK))
+
+# Check optional variables
+$(eval $(call SetupVariable,JIB_JAR,OPTIONAL))
+
+# Now that we have verified that we have the required variables available, we
+# can include the prebuilt spec file ourselves, without an ephemeral spec
+# wrapper. This is required so we can include MakeBase which is needed for
+# CreateNewSpec.
+HAS_SPEC :=
+include $(TOPDIR)/make/InitSupport.gmk
+
+$(eval $(call CheckDeprecatedEnvironment))
+$(eval $(call CheckInvalidMakeFlags))
+$(eval $(call ParseLogLevel))
+
+SPEC := $(TOPDIR)/make/RunTestsPrebuiltSpec.gmk
+include $(SPEC)
+include $(TOPDIR)/make/common/MakeBase.gmk
+
+################################################################################
+# Determine what platform we're running on
+################################################################################
+UNAME := uname
+
+# Get OS name from uname (Cygwin inexplicably adds _NT-x.x)
+UNAME_OS := $(shell $(UNAME) -s | $(CUT) -f1 -d_)
+
+ifeq ($(UNAME_OS), CYGWIN)
+  OPENJDK_TARGET_OS := windows
+  OPENJDK_TARGET_OS_TYPE := windows
+  OPENJDK_TARGET_OS_ENV := windows.cygwin
+else
+  OPENJDK_TARGET_OS_TYPE:=unix
+  ifeq ($(UNAME_OS), Linux)
+    OPENJDK_TARGET_OS := linux
+  else ifeq ($(UNAME_OS), Darwin)
+    OPENJDK_TARGET_OS := macosx
+  else ifeq ($(UNAME_OS), SunOS)
+    OPENJDK_TARGET_OS := solaris
+  else
+    OPENJDK_TARGET_OS := $(UNAME_OS)
+  endif
+  OPENJDK_TARGET_OS_ENV := $(OPENJDK_TARGET_OS)
+endif
+
+# Assume little endian unless otherwise specified
+OPENJDK_TARGET_CPU_ENDIAN := little
+
+ifeq ($(OPENJDK_TARGET_OS), solaris)
+  # On solaris, use uname -p
+  UNAME_CPU := $(shell $(UNAME) -p)
+  # Assume 64-bit platform
+  OPENJDK_TARGET_CPU_BITS := 64
+  ifeq ($(UNAME_CPU), i386)
+    OPENJDK_TARGET_CPU := x86_64
+  else ifeq ($(UNAME_CPU), sparc)
+    OPENJDK_TARGET_CPU := sparcv9
+    OPENJDK_TARGET_CPU_ENDIAN := big
+  else
+    OPENJDK_TARGET_CPU := $(UNAME_CPU)
+  endif
+else
+  # ... all others use uname -m
+  UNAME_CPU := $(shell $(UNAME) -m)
+  ifeq ($(UNAME_CPU), i686)
+    OPENJDK_TARGET_CPU := x86
+    OPENJDK_TARGET_CPU_BITS := 32
+  else
+    # Assume all others are 64-bit. We use the same CPU name as uname for
+    # at least x86_64 and aarch64.
+    OPENJDK_TARGET_CPU := $(UNAME_CPU)
+    OPENJDK_TARGET_CPU_BITS := 64
+  endif
+endif
+
+OPENJDK_TARGET_CPU_ARCH := $(OPENJDK_TARGET_CPU)
+ifeq ($(OPENJDK_TARGET_CPU), x86_64)
+  OPENJDK_TARGET_CPU_ARCH := x86
+else ifeq ($(OPENJDK_TARGET_CPU), sparcv9)
+  OPENJDK_TARGET_CPU_ARCH := sparc
+endif
+
+ifeq ($(OPENJDK_TARGET_OS), windows)
+  ifeq ($(wildcard $(TEST_IMAGE_DIR)/bin/fixpath.exe), )
+    $$(info Error: fixpath is missing from test image '$(TEST_IMAGE_DIR)')
+    $$(error Cannot continue.)
+  endif
+  FIXPATH := $(TEST_IMAGE_DIR)/bin/fixpath.exe -c
+  PATH_SEP:=;
+else
+  FIXPATH :=
+  PATH_SEP:=:
+endif
+
+# Check number of cores
+ifeq ($(OPENJDK_TARGET_OS), linux)
+    NUM_CORES := $(shell $(CAT) /proc/cpuinfo  | $(GREP) -c processor)
+else ifeq ($(OPENJDK_TARGET_OS), macosx)
+    NUM_CORES := $(shell /usr/sbin/sysctl -n hw.ncpu)
+else ifeq ($(OPENJDK_TARGET_OS), solaris)
+    NUM_CORES := $(shell LC_MESSAGES=C /usr/sbin/psrinfo -v | $(GREP) -c on-line)
+else ifeq ($(OPENJDK_TARGET_OS), windows)
+    NUM_CORES := $(NUMBER_OF_PROCESSORS)
+else
+    NUM_CORES := 1
+endif
+
+################################################################################
+# Generate the ephemeral spec file
+################################################################################
+
+# Now we can include additional custom support.
+# This might define CUSTOM_NEW_SPEC_LINE
+ifneq ($(CUSTOM_MAKE_DIR), )
+  include $(CUSTOM_MAKE_DIR)/RunTestsPrebuilt.gmk
+endif
+
+NEW_SPEC := $(OUTPUTDIR)/run-test-spec.gmk
+
+$(call CreateNewSpec, $(NEW_SPEC), \
+    # Generated file -- do not edit!, \
+    SPEC := $(NEW_SPEC), \
+    TOPDIR := $(TOPDIR), \
+    OUTPUTDIR := $(OUTPUTDIR), \
+    BOOT_JDK := $(BOOT_JDK), \
+    JT_HOME := $(JT_HOME), \
+    JDK_IMAGE_DIR := $(JDK_IMAGE_DIR), \
+    TEST_IMAGE_DIR := $(TEST_IMAGE_DIR), \
+    MAKE := $(MAKE), \
+    BASH := $(BASH), \
+    JIB_JAR := $(JIB_JAR), \
+    FIXPATH := $(FIXPATH), \
+    PATH_SEP := $(PATH_SEP), \
+    OPENJDK_TARGET_OS := $(OPENJDK_TARGET_OS), \
+    OPENJDK_TARGET_OS_TYPE := $(OPENJDK_TARGET_OS_TYPE), \
+    OPENJDK_TARGET_OS_ENV := $(OPENJDK_TARGET_OS_ENV), \
+    OPENJDK_TARGET_CPU := $(OPENJDK_TARGET_CPU), \
+    OPENJDK_TARGET_CPU_ARCH := $(OPENJDK_TARGET_CPU_ARCH), \
+    OPENJDK_TARGET_CPU_BITS := $(OPENJDK_TARGET_CPU_BITS), \
+    OPENJDK_TARGET_CPU_ENDIAN := $(OPENJDK_TARGET_CPU_ENDIAN), \
+    NUM_CORES := $(NUM_CORES), \
+    include $(TOPDIR)/make/RunTestsPrebuiltSpec.gmk, \
+    $(CUSTOM_NEW_SPEC_LINE), \
+)
+
+################################################################################
+# The run-test-prebuilt target
+################################################################################
+
+SPEC := $(NEW_SPEC)
+
+default: all
+
+run-test-prebuilt:
+	@$(RM) -f $(MAKESUPPORT_OUTPUTDIR)/exit-with-error
+	@cd $(TOPDIR) && $(MAKE) $(MAKE_ARGS) -f make/RunTests.gmk run-test \
+	    TEST="$(TEST)"
+	@if test -f $(MAKESUPPORT_OUTPUTDIR)/exit-with-error ; then \
+	  exit 1 ; \
+	fi
+
+all: run-test-prebuilt
+
+.PHONY: default all
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/make/RunTestsPrebuiltSpec.gmk	Wed Dec 06 13:45:46 2017 +0100
@@ -0,0 +1,175 @@
+#
+# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+################################################################################
+# Fake minimalistic spec file for RunTestsPrebuilt.gmk.
+################################################################################
+
+define VerifyVariable
+  ifeq ($$($1), )
+    $$(info Error: Variable $1 is missing, needed by RunTestPrebuiltSpec.gmk)
+    $$(error Cannot continue.)
+  else
+    ifneq ($$(findstring $$(LOG_LEVEL), debug trace), )
+      $$(info Prebuilt variable $1=$$($1))
+    endif
+  endif
+endef
+
+# It is the responsibility of the file including us to have set these up.
+# Verify that this is correct.
+$(eval $(call VerifyVariable,SPEC))
+$(eval $(call VerifyVariable,TOPDIR))
+$(eval $(call VerifyVariable,OUTPUTDIR))
+$(eval $(call VerifyVariable,BOOT_JDK))
+$(eval $(call VerifyVariable,JT_HOME))
+$(eval $(call VerifyVariable,JDK_IMAGE_DIR))
+$(eval $(call VerifyVariable,TEST_IMAGE_DIR))
+$(eval $(call VerifyVariable,MAKE))
+$(eval $(call VerifyVariable,BASH))
+
+################################################################################
+# The "human readable" name of this configuration
+CONF_NAME := run-test-prebuilt
+
+# Number of parallel jobs to use for compilation
+JOBS ?= $(NUM_CORES)
+TEST_JOBS ?= 0
+
+# Use hard-coded values for java flags (one size, fits all!)
+JAVA_FLAGS := -Duser.language=en -Duser.country=US
+JAVA_FLAGS_BIG:= -Xms64M -Xmx1600M -XX:ThreadStackSize=1536
+JAVA_FLAGS_SMALL:= -XX:+UseSerialGC -Xms32M -Xmx512M -XX:TieredStopAtLevel=1
+BUILD_JAVA_FLAGS := $(JAVA_FLAGS_BIG)
+
+################################################################################
+# Hard-coded values copied from spec.gmk.in.
+X:=
+SPACE:=$(X) $(X)
+COMMA:=,
+MAKE_ARGS = $(MAKE_LOG_FLAGS) -r -R -I $(TOPDIR)/make/common SPEC=$(SPEC) \
+    MAKE_LOG_FLAGS="$(MAKE_LOG_FLAGS)" LOG_LEVEL=$(LOG_LEVEL)
+BASH_ARGS := -o pipefail -e
+SHELL := $(BASH) $(BASH_ARGS)
+
+################################################################################
+# Set some reasonable defaults for features
+DEBUG_LEVEL := release
+HOTSPOT_DEBUG_LEVEL := release
+BUILD_GTEST := true
+BUILD_FAILURE_HANDLER := true
+
+################################################################################
+# Alias some paths (that should not really be used) to our JDK image under test.
+SUPPORT_OUTPUTDIR := $(OUTPUTDIR)/support
+BUILDTOOLS_OUTPUTDIR := $(OUTPUTDIR)/buildtools
+HOTSPOT_OUTPUTDIR := $(OUTPUTDIR)/hotspot
+JDK_OUTPUTDIR := $(OUTPUTDIR)/jdk
+IMAGES_OUTPUTDIR := $(OUTPUTDIR)/images
+BUNDLES_OUTPUTDIR := $(OUTPUTDIR)/bundles
+TESTMAKE_OUTPUTDIR := $(OUTPUTDIR)/test-make
+MAKESUPPORT_OUTPUTDIR := $(OUTPUTDIR)/make-support
+BUILDJDK_OUTPUTDIR := $(OUTPUTDIR)/buildjdk
+
+JRE_IMAGE_DIR := $(JDK_IMAGE_DIR)
+
+################################################################################
+# Assume build platform is same as target platform
+OPENJDK_BUILD_OS := $(OPENJDK_TARGET_OS)
+OPENJDK_BUILD_OS_TYPE := $(OPENJDK_TARGET_OS_TYPE)
+OPENJDK_BUILD_OS_ENV := $(OPENJDK_TARGET_OS_ENV)
+
+OPENJDK_BUILD_CPU := $(OPENJDK_TARGET_CPU)
+OPENJDK_BUILD_CPU_ARCH := $(OPENJDK_TARGET_CPU_ARCH)
+OPENJDK_BUILD_CPU_BITS := $(OPENJDK_TARGET_CPU_BITS)
+OPENJDK_BUILD_CPU_ENDIAN := $(OPENJDK_TARGET_CPU_ENDIAN)
+
+################################################################################
+# Java executable definitions
+JAVA_CMD := $(BOOT_JDK)/bin/java
+JAVAC_CMD := $(BOOT_JDK)/bin/javac
+JAVAH_CMD := $(BOOT_JDK)/bin/javah
+JAR_CMD := $(BOOT_JDK)/bin/jar
+JLINK_CMD := $(JDK_OUTPUTDIR)/bin/jlink
+JMOD_CMD := $(JDK_OUTPUTDIR)/bin/jmod
+JARSIGNER_CMD := $(BOOT_JDK)/bin/jarsigner
+
+JAVA := $(FIXPATH) $(JAVA_CMD) $(JAVA_FLAGS_BIG) $(JAVA_FLAGS)
+JAVA_SMALL := $(FIXPATH) $(JAVA_CMD) $(JAVA_FLAGS_SMALL) $(JAVA_FLAGS)
+JAVA_JAVAC := $(FIXPATH) $(JAVA_CMD) $(JAVA_FLAGS_SMALL) $(JAVA_FLAGS)
+JAVAC := $(FIXPATH) $(JAVAC_CMD)
+JAVAH := $(FIXPATH) $(JAVAH_CMD)
+JAR := $(FIXPATH) $(JAR_CMD)
+JLINK := $(FIXPATH) $(JLINK_CMD)
+JMOD := $(FIXPATH) $(JMOD_CMD)
+JARSIGNER := $(FIXPATH) $(JARSIGNER_CMD)
+
+BUILD_JAVA := $(JAVA)
+################################################################################
+# Some common tools. Assume most common name and no path.
+AWK := awk
+BASENAME := basename
+CAT := cat
+CD := cd
+CHMOD := chmod
+CP := cp
+CUT := cut
+DATE := date
+DIFF := diff
+DIRNAME := dirname
+FIND := find
+FIND_DELETE := -delete
+ECHO := echo
+EGREP := grep -E
+FGREP := grep -F
+GREP := grep
+GZIP := gzip
+HEAD := head
+LS := ls
+LN := ln
+MKDIR := mkdir
+MV := mv
+NAWK := nawk
+NICE := nice
+PATCH := patch
+PRINTF := printf
+RM := rm -f
+RMDIR := rmdir
+SED := sed
+SH := sh
+SORT := sort
+TAR := tar
+TAIL := tail
+TEE := tee
+TR := tr
+TOUCH := touch
+UNIQ := uniq
+WC := wc
+XARGS := xargs
+ZIPEXE := zip
+UNZIP := unzip
+EXPR := expr
+FILE := file
+HG := hg
--- a/make/common/MakeBase.gmk	Wed Dec 06 02:09:08 2017 -0800
+++ b/make/common/MakeBase.gmk	Wed Dec 06 13:45:46 2017 +0100
@@ -912,6 +912,17 @@
       $(shell $(PRINTF) "%s" $(call ShellQuote, $1) > $2)
 endif
 
+# Param 1 - Text to write
+# Param 2 - File to write to
+ifeq ($(HAS_FILE_FUNCTION), true)
+  AppendFile = \
+      $(file >>$2,$(strip $1))
+else
+  # Use printf to get consistent behavior on all platforms.
+  AppendFile = \
+      $(shell $(PRINTF) "%s" $(call ShellQuote, $1) >> $2)
+endif
+
 ################################################################################
 # DependOnVariable
 #