1 /*

     2  * Copyright (c) 2010, 2016, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.  Oracle designates this

     8  * particular file as subject to the "Classpath" exception as provided

     9  * by Oracle in the LICENSE file that accompanied this code.

    10  *

    11  * This code is distributed in the hope that it will be useful, but WITHOUT

    12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14  * version 2 for more details (a copy is included in the LICENSE file that

    15  * accompanied this code).

    16  *

    17  * You should have received a copy of the GNU General Public License version

    18  * 2 along with this work; if not, write to the Free Software Foundation,

    19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20  *

    21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22  * or visit www.oracle.com if you need additional information or have any

    23  * questions.

    24  */

    25 

    26 package jdk.nashorn.internal.test.framework;

    27 

    28 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_FAILED_LIST_FILE;

    29 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ENABLE_STRICT_MODE;

    30 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDES_FILE;

    31 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_EXCLUDE_LIST;

    32 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_FRAMEWORK;

    33 import static jdk.nashorn.internal.test.framework.TestConfig.TEST_JS_ROOTS;

    34 import java.io.BufferedReader;

    35 import java.io.ByteArrayOutputStream;

    36 import java.io.File;

    37 import java.io.FileInputStream;

    38 import java.io.FileOutputStream;

    39 import java.io.FileReader;

    40 import java.io.FileWriter;

    41 import java.io.IOException;

    42 import java.io.InputStreamReader;

    43 import java.io.OutputStream;

    44 import java.io.PrintStream;

    45 import java.io.PrintWriter;

    46 import java.io.StringReader;

    47 import java.nio.file.FileSystems;

    48 import java.nio.file.Files;

    49 import java.nio.file.StandardCopyOption;

    50 import java.util.ArrayList;

    51 import java.util.Collections;

    52 import java.util.Comparator;

    53 import java.util.List;

    54 import java.util.Locale;

    55 import java.util.Map;

    56 import java.util.Properties;

    57 import java.util.Set;

    58 import java.util.TreeSet;

    59 import java.util.concurrent.Callable;

    60 import java.util.concurrent.CancellationException;

    61 import java.util.concurrent.CountDownLatch;

    62 import java.util.concurrent.ExecutionException;

    63 import java.util.concurrent.ExecutorService;

    64 import java.util.concurrent.Executors;

    65 import java.util.concurrent.Future;

    66 import java.util.concurrent.TimeUnit;

    67 import java.util.regex.Matcher;

    68 import java.util.regex.Pattern;

    69 import jdk.nashorn.internal.test.framework.TestFinder.TestFactory;

    70 

    71 /**

    72  * Parallel test runner runs tests in multiple threads - but avoids any dependency

    73  * on third-party test framework library such as TestNG.

    74  */

    75 @SuppressWarnings("javadoc")

    76 public class ParallelTestRunner {

    77 

    78     // ParallelTestRunner-specific

    79     private static final String    TEST_JS_THREADS     = "test.js.threads";

    80     private static final String    TEST_JS_REPORT_FILE = "test.js.report.file";

    81     // test262 does a lot of eval's and the JVM hates multithreaded class definition, so lower thread count is usually faster.

    82     private static final int       THREADS = Integer.getInteger(TEST_JS_THREADS, Runtime.getRuntime().availableProcessors() > 4 ? 4 : 2);

    83 

    84     private final List<ScriptRunnable> tests    = new ArrayList<>();

    85     private final Set<String>      orphans  = new TreeSet<>();

    86     private final ExecutorService  executor = Executors.newFixedThreadPool(THREADS);

    87 

    88     // Ctrl-C handling

    89     private final CountDownLatch   finishedLatch = new CountDownLatch(1);

    90     private final Thread           shutdownHook  = new Thread() {

    91                                                        @Override

    92                                                        public void run() {

    93                                                            if (!executor.isTerminated()) {

    94                                                                executor.shutdownNow();

    95                                                                try {

    96                                                                    executor.awaitTermination(25, TimeUnit.SECONDS);

    97                                                                    finishedLatch.await(5, TimeUnit.SECONDS);

    98                                                                } catch (final InterruptedException e) {

    99                                                                    // empty

   100                                                                }

   101                                                            }

   102                                                        }

   103                                                    };

   104 

   105     public ParallelTestRunner() throws Exception {

   106         suite();

   107     }

   108 

   109     private static PrintStream outputStream() {

   110         final String reportFile = System.getProperty(TEST_JS_REPORT_FILE, "");

   111         PrintStream output = System.out;

   112 

   113         if (!reportFile.isEmpty()) {

   114             try {

   115                 output = new PrintStream(new OutputStreamDelegator(System.out, new FileOutputStream(reportFile)));

   116             } catch (final IOException e) {

   117                 System.err.println(e);

   118             }

   119         }

   120 

   121         return output;

   122     }

   123 

   124     public static final class ScriptRunnable extends AbstractScriptRunnable implements Callable<ScriptRunnable.Result> {

   125         private final Result                result   = new Result();

   126 

   127         public class Result {

   128             private boolean  passed = true;

   129             public String    expected;

   130             public String    out;

   131             public String    err;

   132             public Throwable exception;

   133 

   134             public ScriptRunnable getTest() {

   135                 return ScriptRunnable.this;

   136             }

   137 

   138             public boolean passed() {

   139                 return passed;

   140             }

   141 

   142             @Override

   143             public String toString() {

   144                 return getTest().toString();

   145             }

   146         }

   147 

   148         public ScriptRunnable(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> scriptArguments) {

   149             super(framework, testFile, engineOptions, testOptions, scriptArguments);

   150         }

   151 

   152         @Override

   153         protected void log(final String msg) {

   154             System.err.println(msg);

   155         }

   156 

   157         @Override

   158         protected void fail(final String message) {

   159             throw new TestFailedError(message);

   160         }

   161 

   162         @Override

   163         protected void compile() throws IOException {

   164             final ByteArrayOutputStream out = new ByteArrayOutputStream();

   165             final ByteArrayOutputStream err = new ByteArrayOutputStream();

   166             final List<String> args = getCompilerArgs();

   167             int errors;

   168             try {

   169                 errors = evaluateScript(out, err, args.toArray(new String[0]));

   170             } catch (final AssertionError e) {

   171                 final PrintWriter writer = new PrintWriter(err);

   172                 e.printStackTrace(writer);

   173                 writer.flush();

   174                 errors = 1;

   175             }

   176             if (errors != 0 || checkCompilerMsg) {

   177                 result.err = err.toString();

   178                 if (expectCompileFailure || checkCompilerMsg) {

   179                     final PrintStream outputDest = new PrintStream(new FileOutputStream(getErrorFileName()));

   180                     TestHelper.dumpFile(outputDest, new StringReader(new String(err.toByteArray())));

   181                     outputDest.println("--");

   182                 }

   183                 if (errors != 0 && !expectCompileFailure) {

   184                     fail(String.format("%d errors compiling %s", errors, testFile));

   185                 }

   186                 if (checkCompilerMsg) {

   187                     compare(getErrorFileName(), expectedFileName, true);

   188                 }

   189             }

   190             if (expectCompileFailure && errors == 0) {

   191                 fail(String.format("No errors encountered compiling negative test %s", testFile));

   192             }

   193         }

   194 

   195         @Override

   196         protected void execute() {

   197             final List<String> args = getRuntimeArgs();

   198             final ByteArrayOutputStream out = new ByteArrayOutputStream();

   199             final ByteArrayOutputStream err = new ByteArrayOutputStream();

   200 

   201             try {

   202                 final int errors = evaluateScript(out, err, args.toArray(new String[0]));

   203 

   204                 if (errors != 0 || err.size() > 0) {

   205                     if (expectRunFailure) {

   206                         return;

   207                     }

   208                     if (!ignoreStdError) {

   209 

   210                         try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {

   211                             outputFile.write(out.toByteArray());

   212                             errorFile.write(err.toByteArray());

   213                         }

   214 

   215                         result.out = out.toString();

   216                         result.err = err.toString();

   217                         fail(err.toString());

   218                     }

   219                 }

   220 

   221                 if (compare) {

   222                     final File expectedFile = new File(expectedFileName);

   223                     try {

   224                         BufferedReader expected;

   225                         if (expectedFile.exists()) {

   226                             expected = new BufferedReader(new FileReader(expectedFile));

   227                         } else {

   228                             expected = new BufferedReader(new StringReader(""));

   229                         }

   230                         compare(new BufferedReader(new StringReader(out.toString())), expected, false);

   231                     } catch (final Throwable ex) {

   232                         if (expectedFile.exists()) {

   233                             copyExpectedFile();

   234                         }

   235                         try (OutputStream outputFile = new FileOutputStream(getOutputFileName()); OutputStream errorFile = new FileOutputStream(getErrorFileName())) {

   236                             outputFile.write(out.toByteArray());

   237                             errorFile.write(err.toByteArray());

   238                         }

   239                         ex.printStackTrace();

   240                         throw ex;

   241                     }

   242                 }

   243             } catch (final IOException e) {

   244                 if (!expectRunFailure) {

   245                     fail("Failure running test " + testFile + ": " + e.getMessage());

   246                 } // else success

   247             }

   248         }

   249 

   250         private void compare(final String fileName, final String expected, final boolean compareCompilerMsg) throws IOException {

   251             final File expectedFile = new File(expected);

   252 

   253             BufferedReader expectedReader;

   254             if (expectedFile.exists()) {

   255                 expectedReader = new BufferedReader(new InputStreamReader(new FileInputStream(expectedFileName)));

   256             } else {

   257                 expectedReader = new BufferedReader(new StringReader(""));

   258             }

   259 

   260             final BufferedReader actual = new BufferedReader(new InputStreamReader(new FileInputStream(fileName)));

   261 

   262             compare(actual, expectedReader, compareCompilerMsg);

   263         }

   264 

   265         private void copyExpectedFile() {

   266             if (!new File(expectedFileName).exists()) {

   267                 return;

   268             }

   269             // copy expected file overwriting existing file and preserving last

   270             // modified time of source

   271             try {

   272                 Files.copy(FileSystems.getDefault().getPath(expectedFileName), FileSystems.getDefault().getPath(getCopyExpectedFileName()), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);

   273             } catch (final IOException ex) {

   274                 fail("failed to copy expected " + expectedFileName + " to " + getCopyExpectedFileName() + ": " + ex.getMessage());

   275             }

   276         }

   277 

   278         @Override

   279         public Result call() {

   280             try {

   281                 runTest();

   282             } catch (final Throwable ex) {

   283                 result.exception = ex;

   284                 result.passed = false;

   285                 ex.printStackTrace();

   286             }

   287             return result;

   288         }

   289 

   290         private String getOutputFileName() {

   291             buildDir.mkdirs();

   292             return outputFileName;

   293         }

   294 

   295         private String getErrorFileName() {

   296             buildDir.mkdirs();

   297             return errorFileName;

   298         }

   299 

   300         private String getCopyExpectedFileName() {

   301             buildDir.mkdirs();

   302             return copyExpectedFileName;

   303         }

   304     }

   305 

   306     private void suite() throws Exception {

   307         Locale.setDefault(new Locale(""));

   308         System.setOut(outputStream());

   309 

   310         final TestFactory<ScriptRunnable> testFactory = new TestFactory<ScriptRunnable>() {

   311             @Override

   312             public ScriptRunnable createTest(final String framework, final File testFile, final List<String> engineOptions, final Map<String, String> testOptions, final List<String> arguments) {

   313                 return new ScriptRunnable(framework, testFile, engineOptions, testOptions, arguments);

   314             }

   315 

   316             @Override

   317             public void log(final String msg) {

   318                 System.err.println(msg);

   319             }

   320         };

   321 

   322         TestFinder.findAllTests(tests, orphans, testFactory);

   323 

   324         Collections.sort(tests, new Comparator<ScriptRunnable>() {

   325             @Override

   326             public int compare(final ScriptRunnable o1, final ScriptRunnable o2) {

   327                 return o1.testFile.compareTo(o2.testFile);

   328             }

   329         });

   330     }

   331 

   332     @SuppressWarnings("resource")

   333     public boolean run() throws IOException {

   334         final int testCount = tests.size();

   335         int passCount = 0;

   336         int doneCount = 0;

   337         System.out.printf("Found %d tests.\n", testCount);

   338         final long startTime = System.nanoTime();

   339 

   340         Runtime.getRuntime().addShutdownHook(shutdownHook);

   341 

   342         final List<Future<ScriptRunnable.Result>> futures = new ArrayList<>();

   343         for (final ScriptRunnable test : tests) {

   344             futures.add(executor.submit(test));

   345         }

   346 

   347         executor.shutdown();

   348         try {

   349             executor.awaitTermination(60, TimeUnit.MINUTES);

   350         } catch (final InterruptedException ex) {

   351             // empty

   352         }

   353 

   354         final List<ScriptRunnable.Result> results = new ArrayList<>();

   355         for (final Future<ScriptRunnable.Result> future : futures) {

   356             if (future.isDone()) {

   357                 try {

   358                     final ScriptRunnable.Result result = future.get();

   359                     results.add(result);

   360                     doneCount++;

   361                     if (result.passed()) {

   362                         passCount++;

   363                     }

   364                 } catch (CancellationException | ExecutionException ex) {

   365                     ex.printStackTrace();

   366                 } catch (final InterruptedException ex) {

   367                     assert false : "should not reach here";

   368                 }

   369             }

   370         }

   371 

   372         Collections.sort(results, new Comparator<ScriptRunnable.Result>() {

   373             @Override

   374             public int compare(final ScriptRunnable.Result o1, final ScriptRunnable.Result o2) {

   375                 return o1.getTest().testFile.compareTo(o2.getTest().testFile);

   376             }

   377         });

   378 

   379         boolean hasFailed = false;

   380         final String failedList = System.getProperty(TEST_FAILED_LIST_FILE);

   381         final boolean hasFailedList = failedList != null;

   382         final boolean hadPreviouslyFailingTests = hasFailedList && new File(failedList).length() > 0;

   383         final FileWriter failedFileWriter = hasFailedList ? new FileWriter(failedList) : null;

   384         try {

   385             final PrintWriter failedListWriter = failedFileWriter == null ? null : new PrintWriter(failedFileWriter);

   386             for (final ScriptRunnable.Result result : results) {

   387                 if (!result.passed()) {

   388                     if (hasFailed == false) {

   389                         hasFailed = true;

   390                         System.out.println();

   391                         System.out.println("FAILED TESTS");

   392                     }

   393 

   394                     System.out.println(result.getTest());

   395                     if(failedFileWriter != null) {

   396                         failedListWriter.println(result.getTest().testFile.getPath());

   397                     }

   398                     if (result.exception != null) {

   399                         final String exceptionString = result.exception instanceof TestFailedError ? result.exception.getMessage() : result.exception.toString();

   400                         System.out.print(exceptionString.endsWith("\n") ? exceptionString : exceptionString + "\n");

   401                         System.out.print(result.out != null ? result.out : "");

   402                     }

   403                 }

   404             }

   405         } finally {

   406             if(failedFileWriter != null) {

   407                 failedFileWriter.close();

   408             }

   409         }

   410         final double timeElapsed = (System.nanoTime() - startTime) / 1e9; // [s]

   411         System.out.printf("Tests run: %d/%d tests, passed: %d (%.2f%%), failed: %d. Time elapsed: %.0fmin %.0fs.\n", doneCount, testCount, passCount, 100d * passCount / doneCount, doneCount - passCount, timeElapsed / 60, timeElapsed % 60);

   412         System.out.flush();

   413 

   414         finishedLatch.countDown();

   415 

   416         if (hasFailed) {

   417             throw new AssertionError("TEST FAILED");

   418         }

   419 

   420         if(hasFailedList) {

   421             new File(failedList).delete();

   422         }

   423 

   424         if(hadPreviouslyFailingTests) {

   425             System.out.println();

   426             System.out.println("Good job on getting all your previously failing tests pass!");

   427             System.out.println("NOW re-running all tests to make sure you haven't caused any NEW test failures.");

   428             System.out.println();

   429         }

   430 

   431         return hadPreviouslyFailingTests;

   432     }

   433 

   434     public static void main(final String[] args) throws Exception {

   435         parseArgs(args);

   436 

   437         while (new ParallelTestRunner().run()) {

   438             //empty

   439         }

   440     }

   441 

   442     private static void parseArgs(final String[] args) {

   443         if (args.length > 0) {

   444             String roots = "";

   445             String reportFile = "";

   446             for (int i = 0; i < args.length; i++) {

   447                 if (args[i].equals("--roots") && i != args.length - 1) {

   448                     roots += args[++i] + " ";

   449                 } else if (args[i].equals("--report-file") && i != args.length - 1) {

   450                     reportFile = args[++i];

   451                 } else if (args[i].equals("--test262")) {

   452                     try {

   453                         setTest262Properties();

   454                     } catch (final IOException ex) {

   455                         System.err.println(ex);

   456                     }

   457                 }

   458             }

   459             if (!roots.isEmpty()) {

   460                 System.setProperty(TEST_JS_ROOTS, roots.trim());

   461             }

   462             if (!reportFile.isEmpty()) {

   463                 System.setProperty(TEST_JS_REPORT_FILE, reportFile);

   464             }

   465         }

   466     }

   467 

   468     private static void setTest262Properties() throws IOException {

   469         System.setProperty(TEST_JS_ROOTS, "test/test262/test/suite/");

   470         System.setProperty(TEST_JS_FRAMEWORK, "test/script/test262.js test/test262/test/harness/framework.js test/test262/test/harness/sta.js");

   471         System.setProperty(TEST_JS_EXCLUDES_FILE, "test/test262/test/config/excludelist.xml");

   472         System.setProperty(TEST_JS_ENABLE_STRICT_MODE, "true");

   473 

   474         final Properties projectProperties = new Properties();

   475         projectProperties.load(new FileInputStream("project.properties"));

   476         String excludeList = projectProperties.getProperty("test262-test-sys-prop.test.js.exclude.list", "");

   477         final Pattern pattern = Pattern.compile("\\$\\{([^}]+)}");

   478         for (;;) {

   479             final Matcher matcher = pattern.matcher(excludeList);

   480             if (!matcher.find()) {

   481                 break;

   482             }

   483             final String propertyValue = projectProperties.getProperty(matcher.group(1), "");

   484             excludeList = excludeList.substring(0, matcher.start()) + propertyValue + excludeList.substring(matcher.end());

   485         }

   486         System.setProperty(TEST_JS_EXCLUDE_LIST, excludeList);

   487     }

   488 

   489     public static final class OutputStreamDelegator extends OutputStream {

   490         private final OutputStream[] streams;

   491 

   492         public OutputStreamDelegator(final OutputStream... streams) {

   493             this.streams = streams;

   494         }

   495 

   496         @Override

   497         public void write(final int b) throws IOException {

   498             for (final OutputStream stream : streams) {

   499                 stream.write(b);

   500             }

   501         }

   502 

   503         @Override

   504         public void flush() throws IOException {

   505             for (final OutputStream stream : streams) {

   506                 stream.flush();

   507             }

   508         }

   509     }

   510 }

   511 

   512 final class TestFailedError extends Error {

   513     private static final long serialVersionUID = 1L;

   514 

   515     public TestFailedError(final String message) {

   516         super(message);

   517     }

   518 

   519     public TestFailedError(final String message, final Throwable cause) {

   520         super(message, cause);

   521     }

   522 

   523     public TestFailedError(final Throwable cause) {

   524         super(cause);

   525     }

   526 }