langtools/test/com/sun/javadoc/lib/JavadocTester.java
changeset 24399 af1a0220d0fa
parent 24217 25b12d4d4192
child 25004 b33effe4f252
--- a/langtools/test/com/sun/javadoc/lib/JavadocTester.java	Fri May 09 23:31:05 2014 +0200
+++ b/langtools/test/com/sun/javadoc/lib/JavadocTester.java	Fri May 09 15:37:12 2014 -0700
@@ -21,102 +21,163 @@
  * questions.
  */
 
-import java.io.*;
+import java.io.BufferedWriter;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileWriter;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.lang.annotation.Annotation;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.ref.SoftReference;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 
 /**
- * Runs javadoc and then runs regression tests on the resulting output.
- * This class currently contains three tests:
- * <ul>
- * <li> String search: Reads each file, complete with newlines,
- *      into a string.  Lets you search for strings that contain
- *      newlines.  String matching is case-sensitive.
- *      You can run javadoc multiple times with different arguments,
- *      generating output into different destination directories, and
- *      then perform a different array of tests on each one.
- *      To do this, the run method accepts a test array for testing
- *      that a string is found, and a negated test array for testing
- *      that a string is not found.
- * <li> Run diffs: Iterate through the list of given file pairs
- *      and diff the pairs.
- * <li> Check exit code: Check the exit code of Javadoc and
- *      record whether the test passed or failed.
- * </ul>
+ * Test framework for running javadoc and performing tests on the resulting output.
+ *
+ * <p>
+ * Tests are typically written as subtypes of JavadocTester, with a main
+ * method that creates an instance of the test class and calls the runTests()
+ * method. The runTests() methods calls all the test methods declared in the class,
+ * and then calls a method to print a summary, and throw an exception if
+ * any of the test methods reported a failure.
+ *
+ * <p>
+ * Test methods are identified with a @Test annotation. They have no parameters.
+ * The name of the method is not important, but if you have more than one, it is
+ * recommended that the names be meaningful and suggestive of the test case
+ * contained therein.
+ *
+ * <p>
+ * Typically, a test method will invoke javadoc, and then perform various
+ * checks on the results. The standard checks are:
+ *
+ * <dl>
+ * <dt>checkExitCode
+ * <dd>Check the exit code returned from javadoc.
+ * <dt>checkOutput
+ * <dd>Perform a series of checks on the contents on a file or output stream
+ *     generated by javadoc.
+ *     The checks can be either that a series of strings are found or are not found.
+ * <dt>checkFiles
+ * <dd>Perform a series of checks on the files generated by javadoc.
+ *     The checks can be that a series of files are found or are not found.
+ * </dl>
+ *
+ * <pre><code>
+ *  public class MyTester extends JavadocTester {
+ *      public static void main(String... args) throws Exception {
+ *          MyTester tester = new MyTester();
+ *          tester.runTests();
+ *      }
+ *
+ *      // test methods...
+ *      @Test
+ *      void test() {
+ *          javadoc(<i>args</i>);
+ *          checkExit(Exit.OK);
+ *          checkOutput(<i>file</i>, true,
+ *              <i>strings-to-find</i>);
+ *          checkOutput(<i>file</i>, false,
+ *              <i>strings-to-not-find</i>);
+ *      }
+ *  }
+ * </code></pre>
+ *
+ * <p>
+ * If javadoc is run more than once in a test method, you can compare the
+ * results that are generated with the diff method. Since files written by
+ * javadoc typically contain a timestamp, you may want to use the -notimestamp
+ * option if you are going to compare the results from two runs of javadoc.
+ *
+ * <p>
+ * If you have many calls of checkOutput that are very similar, you can write
+ * your own check... method to reduce the amount of duplication. For example,
+ * if you want to check that many files contain the same string, you could
+ * write a method that takes a varargs list of files and calls checkOutput
+ * on each file in turn with the string to be checked.
+ *
+ * <p>
+ * You can also write you own custom check methods, which can use
+ * readFile to get the contents of a file generated by javadoc,
+ * and then use pass(...) or fail(...) to report whether the check
+ * succeeded or not.
+ *
+ * <p>
+ * You can have many separate test methods, each identified with a @Test
+ * annotation. However, you should <b>not</b> assume they will be called
+ * in the order declared in your source file.  If the order of a series
+ * of javadoc invocations is important, do that within a single method.
+ * If the invocations are independent, for better clarity, use separate
+ * test methods, each with their own set of checks on the results.
  *
  * @author Doug Kramer
  * @author Jamie Ho
- * @since 1.4.2
+ * @author Jonathan Gibbons (rewrite)
  */
 public abstract class JavadocTester {
 
-    protected static final String NL = System.getProperty("line.separator");
-    protected static final String FS = System.getProperty("file.separator");
+    public static final String FS = System.getProperty("file.separator");
+    public static final String PS = System.getProperty("path.separator");
+    public static final String NL = System.getProperty("line.separator");
 
-    protected static final String SRC_DIR = System.getProperty("test.src", ".");
-    protected static final String JAVA_VERSION = System.getProperty("java.version");
-    protected static final String OUTPUT_DIR = "out";
-    protected static final String[][] NO_TEST = new String[][] {};
-    protected static final String[] NO_FILE_TEST = new String[] {};
+    public enum Output {
+        /** The name for error output from javadoc. */
+        ERROR,
+        /** The name for the notice output from javadoc. */
+        NOTICE,
+        /** The name for the warning output  from javadoc. */
+        WARNING,
+        /** The name for any output written to System.out. */
+        STDOUT,
+        /** The name for any output written to System.err. */
+        STDERR
+    }
 
-    /**
-     * Use this as the file name in the test array when you want to search
-     * for a string in the error output.
-     */
-    public static final String ERROR_OUTPUT = "ERROR_OUTPUT";
+    /** The output directory used in the most recent call of javadoc. */
+    protected File outputDir;
+
+    /** The exit code of the most recent call of javadoc. */
+    private int exitCode;
+
+    /** The output generated by javadoc to the various writers and streams. */
+    private final Map<Output, String> outputMap = new EnumMap<>(Output.class);
 
-    /**
-     * Use this as the file name in the test array when you want to search
-     * for a string in the notice output.
-     */
-    public static final String NOTICE_OUTPUT = "NOTICE_OUTPUT";
+    /** A cache of file content, to avoid reading files unnecessarily. */
+    private final Map<File,SoftReference<String>> fileContentCache = new HashMap<>();
 
-    /**
-     * Use this as the file name in the test array when you want to search
-     * for a string in the warning output.
-     */
-    public static final String WARNING_OUTPUT = "WARNING_OUTPUT";
+    /** Stream used for logging messages. */
+    private final PrintStream out = System.out;
+
+    /** The directory containing the source code for the test. */
+    public static final String testSrc = System.getProperty("test.src");
 
     /**
-     * Use this as the file name in the test array when you want to search
-     * for a string in standard output.
-     */
-    public static final String STANDARD_OUTPUT = "STANDARD_OUTPUT";
-
-    /**
-     * The default doclet.
-     */
-    public static final String DEFAULT_DOCLET_CLASS = "com.sun.tools.doclets.formats.html.HtmlDoclet";
-    public static final String DEFAULT_DOCLET_CLASS_OLD = "com.sun.tools.doclets.standard.Standard";
-
-    /**
-     * The writer to write error messages.
+     * Get the path for a source file in the test source directory.
+     * @param path the path of a file or directory in the source directory
+     * @return the full path of the specified file
      */
-    public StringWriter errors;
-
-    /**
-     * The writer to write notices.
-     */
-    public StringWriter notices;
-
-    /**
-     * The writer to write warnings.
-     */
-    public StringWriter warnings;
-
-    /**
-     * The buffer of warning output.
-     */
-    public StringBuffer standardOut;
-
-    /**
-     * The output directory.
-     */
-    private File outputDir;
+    public static String testSrc(String path) {
+        return new File(testSrc, path).getPath();
+    }
 
     /**
      * Alternatives for checking the contents of a directory.
      */
-    enum DirectoryCheck {
+    public enum DirectoryCheck {
         /**
          * Check that the directory is empty.
          */
@@ -153,91 +214,75 @@
 
     private DirectoryCheck outputDirectoryCheck = DirectoryCheck.EMPTY;
 
-    /**
-     * The current subtest number.
-     */
-    private static int numTestsRun = 0;
+    /** The current subtest number. Incremented when checking(...) is called. */
+    private int numTestsRun = 0;
+
+    /** The number of subtests passed. Incremented when passed(...) is called. */
+    private int numTestsPassed = 0;
+
+    /** The current run of javadoc. Incremented when javadoc is called. */
+    private int javadocRunNum = 0;
 
-    /**
-     * The number of subtests passed.
-     */
-    private static int numTestsPassed = 0;
+    /** The name of the standard doclet. */
+    // This ought not to be necessary; there ought to be a javadoc entry point
+    // that does not require this to be know externally.
+    private static final String standardDocletClassName =
+            "com.sun.tools.doclets.standard.Standard";
 
-    /**
-     * The current run of javadoc
-     */
-    private static int javadocRunNum = 0;
+    /** Marker annotation for test methods to be invoked by runTests. */
+    @Retention(RetentionPolicy.RUNTIME)
+    @interface Test { }
 
     /**
-     * Construct a JavadocTester.
+     * Run all methods annotated with @Test, followed by printSummary.
+     * Typically called on a tester object in main()
+     * @throws Exception if any errors occurred
      */
-    public JavadocTester() {
-    }
-
-    /**
-     * Execute the tests.
-     *
-     * @param args             the arguments to pass to Javadoc
-     * @param testArray        the array of tests
-     * @param negatedTestArray the array of negated tests
-     * @return                 the return code for the execution of Javadoc
-     */
-    public int run(String[] args,
-            String[][] testArray, String[][] negatedTestArray) {
-        int returnCode = runJavadoc(args);
-        runTestsOnHTML(testArray, negatedTestArray);
-        return returnCode;
+    public void runTests() throws Exception {
+        for (Method m: getClass().getDeclaredMethods()) {
+            Annotation a = m.getAnnotation(Test.class);
+            if (a != null) {
+                try {
+                    out.println("Running test " + m.getName());
+                    m.invoke(this, new Object[] { });
+                } catch (InvocationTargetException e) {
+                    Throwable cause = e.getCause();
+                    throw (cause instanceof Exception) ? ((Exception) cause) : e;
+                }
+                out.println();
+            }
+        }
+        printSummary();
     }
 
     /**
-     * Execute the tests.
+     * Run javadoc.
+     * The output directory used by this call and the final exit code
+     * will be saved for later use.
+     * To aid the reader, it is recommended that calls to this method
+     * put each option and the arguments it takes on a separate line.
      *
-     * @param args                 the arguments to pass to Javadoc
-     * @param testArray            the array of tests
-     * @param negatedTestArray     the array of negated tests
-     * @param fileTestArray        the array of file tests
-     * @param negatedFileTestArray the array of negated file tests
-     * @return                     the return code for the execution of Javadoc
+     * Example:
+     * <pre><code>
+     *  javadoc("-d", "out",
+     *          "-sourcepath", testSrc,
+     *          "-notimestamp",
+     *          "pkg1", "pkg2", "pkg3/C.java");
+     * </code></pre>
+     *
+     * @param args the arguments to pass to javadoc
      */
-    public int run(String[] args,
-            String[][] testArray, String[][] negatedTestArray,
-            String[] fileTestArray, String[] negatedFileTestArray) {
-        int returnCode = runJavadoc(args);
-        runTestsOnHTML(testArray, negatedTestArray);
-        runTestsOnFile(fileTestArray, negatedFileTestArray);
-        return returnCode;
-    }
+    public void javadoc(String... args) {
+        outputMap.clear();
+        fileContentCache.clear();
 
-    /**
-     * Execute Javadoc using the default doclet.
-     *
-     * @param args  the arguments to pass to Javadoc
-     * @return      the return code from the execution of Javadoc
-     */
-    public int runJavadoc(String[] args) {
-        float javaVersion = Float.parseFloat(JAVA_VERSION.substring(0,3));
-        String docletClass = javaVersion < 1.5 ?
-            DEFAULT_DOCLET_CLASS_OLD : DEFAULT_DOCLET_CLASS;
-        return runJavadoc(docletClass, args);
-    }
-
-
-    /**
-     * Execute Javadoc.
-     *
-     * @param docletClass the doclet being tested.
-     * @param args  the arguments to pass to Javadoc
-     * @return      the return code from the execution of Javadoc
-     */
-    public int runJavadoc(String docletClass, String[] args) {
         javadocRunNum++;
         if (javadocRunNum == 1) {
-            System.out.println("\n" + "Running javadoc...");
+            out.println("Running javadoc...");
         } else {
-            System.out.println("\n" + "Running javadoc (run "
+            out.println("Running javadoc (run "
                                     + javadocRunNum + ")...");
         }
-        initOutputBuffers();
         outputDir = new File(".");
         for (int i = 0; i < args.length - 2; i++) {
             if (args[i].equals("-d")) {
@@ -245,363 +290,199 @@
                 break;
             }
         }
+//        log.setOutDir(outputDir);
 
         outputDirectoryCheck.check(outputDir);
 
-        ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-        PrintStream prevOut = System.out;
-        System.setOut(new PrintStream(stdout));
-
-        ByteArrayOutputStream stderr = new ByteArrayOutputStream();
-        PrintStream prevErr = System.err;
-        System.setErr(new PrintStream(stderr));
-
-        int returnCode = com.sun.tools.javadoc.Main.execute(
-                "javadoc",
-                new PrintWriter(errors, true),
-                new PrintWriter(warnings, true),
-                new PrintWriter(notices, true),
-                docletClass,
-                getClass().getClassLoader(),
-                args);
-        System.setOut(prevOut);
-        standardOut = new StringBuffer(stdout.toString());
-        System.setErr(prevErr);
-        errors.write(NL + stderr.toString());
+        // These are the primary streams used by javadoc
+        WriterOutput errOut = new WriterOutput();
+        WriterOutput warnOut = new WriterOutput();
+        WriterOutput noticeOut = new WriterOutput();
+        // These are to catch output to System.out and System.err,
+        // in case these are used instead of the primary streams
+        StreamOutput sysOut = new StreamOutput(System.out, System::setOut);
+        StreamOutput sysErr = new StreamOutput(System.err, System::setErr);
 
-        printJavadocOutput();
-        return returnCode;
-    }
+        try {
+            exitCode = com.sun.tools.javadoc.Main.execute(
+                    "javadoc",
+                    errOut.pw, warnOut.pw, noticeOut.pw,
+                    standardDocletClassName,
+                    args);
+        } finally {
+            outputMap.put(Output.STDOUT, sysOut.close());
+            outputMap.put(Output.STDERR, sysErr.close());
+            outputMap.put(Output.ERROR, errOut.close());
+            outputMap.put(Output.WARNING, warnOut.close());
+            outputMap.put(Output.NOTICE, noticeOut.close());
+        }
 
-    /**
-     * Set a filter to check the initial contents of the output directory
-     * before javadoc is run.
-     * The filter should return true for files that should <b>not</b> appear.
-     */
-    public void setCheckOutputDirectoryCheck(DirectoryCheck c) {
-        outputDirectoryCheck = c;
-    }
-
-    /**
-     * Create new string writer buffers
-     */
-    private void initOutputBuffers() {
-        errors   = new StringWriter();
-        notices  = new StringWriter();
-        warnings = new StringWriter();
+        outputMap.forEach((name, text) -> {
+            if (!text.isEmpty()) {
+                out.println("javadoc " + name + ":");
+                out.println(text);
+            }
+        });
     }
 
     /**
-     * Run array of tests on the resulting HTML.
-     * This method accepts a testArray for testing that a string is found
-     * and a negatedTestArray for testing that a string is not found.
-     *
-     * @param testArray         the array of tests
-     * @param negatedTestArray  the array of negated tests
+     * Set the kind of check for the initial contents of the output directory
+     * before javadoc is run.
+     * The filter should return true for files that should <b>not</b> appear.
+     * @param c the kind of check to perform
      */
-    public void runTestsOnHTML(String[][] testArray, String[][] negatedTestArray) {
-        runTestsOnHTML(testArray, false);
-        runTestsOnHTML(negatedTestArray, true);
+    public void setOutputDirectoryCheck(DirectoryCheck c) {
+        outputDirectoryCheck = c;
     }
 
-    /**
-     * Run array of tests on the generated files.
-     * This method accepts a fileTestArray for testing if a file is generated
-     * and a negatedFileTestArray for testing if a file is not found.
-     * The files are relative to the most recent output directory specified
-     * with -d.
-     *
-     * @param fileTestArray         the array of file tests
-     * @param negatedFileTestArray  the array of negated file tests
-     */
-    public void runTestsOnFile(String[] fileTestArray, String[] negatedFileTestArray) {
-        runTestsOnFile(outputDir, fileTestArray, false);
-        runTestsOnFile(outputDir, negatedFileTestArray, true);
+    public enum Exit {
+        OK(0),
+        FAILED(1);
+
+        Exit(int code) {
+            this.code = code;
+        }
+
+        final int code;
     }
 
     /**
-     * Run the array of tests on the resulting HTML.
-     * The files are relative to the most recent output directory specified
-     * with -d.
+     * Check the exit code of the most recent call of javadoc.
      *
-     * @param testArray the array of tests
-     * @param isNegated true if test is negated; false otherwise
+     * @param expected the exit code that is required for the test
+     * to pass.
      */
-    private void runTestsOnHTML(String[][] testArray , boolean isNegated) {
-        for (String[] test : testArray) {
-            numTestsRun++;
-            System.out.print("Running subtest #" + numTestsRun + "... ");
-            // Get string to find
-            String stringToFind = test[1];
-            // Read contents of file into a string
-            String fileString;
-            try {
-                fileString = readFileToString(outputDir, test[0]);
-            } catch (Error e) {
-                if (isNegated) {
-                    System.out.println( "FAILED, due to " + e + "\n");
-                    continue;
-                }
-                throw e;
-            }
-            // Find string in file's contents
-            boolean isFound = findString(fileString, stringToFind);
-            if ((isNegated && !isFound) || (!isNegated && isFound)) {
-                numTestsPassed += 1;
-                System.out.println("Passed" + "\n"
-                        + (isNegated ? "not found:" : "found:") + "\n"
-                        + stringToFind + " in " + test[0] + "\n");
-            } else {
-                System.out.println("FAILED, when searching for:" + "\n"
-                        + stringToFind
-                        + " in " + test[0] + "\n");
-            }
+    public void checkExit(Exit expected) {
+        checking("check exit code");
+        if (exitCode == expected.code) {
+            passed("return code " + exitCode);
+        } else {
+            failed("return code " + exitCode +"; expected " + expected.code + " (" + expected + ")");
         }
     }
 
     /**
-     * Run the array of file tests on the generated files.
-     *
-     * @param testArray the array of file tests
-     * @param isNegated true if test is negated; false otherwise
+     * Check for content in (or not in) the generated output.
+     * Within the search strings, the newline character \n
+     * will be translated to the platform newline character sequence.
+     * @param path a path within the most recent output directory
+     *  or the name of one of the output buffers, identifying
+     *  where to look for the search strings.
+     * @param expectedFound true if all of the search strings are expected
+     *  to be found, or false if all of the strings are expected to be
+     *  not found
+     * @param strings the strings to be searched for
      */
-    private void runTestsOnFile(File baseDir, String[] testArray, boolean isNegated) {
-        for (String fileName : testArray) {
-            numTestsRun++;
-            String failedString = "FAILED: file (" + fileName + ") found" + "\n";
-            String passedString = "Passed" + "\n" +
-                    "file (" + fileName + ") not found" + "\n";
-            System.out.print("Running subtest #" + numTestsRun + "... ");
-            try {
-                File file = new File(baseDir, fileName);
-                if ((file.exists() && !isNegated) || (!file.exists() && isNegated)) {
-                    numTestsPassed += 1;
-                    System.out.println(passedString);
-                } else {
-                    System.out.println(failedString);
-                }
-            } catch (Error e) {
-                System.err.println(e);
+    public void checkOutput(String path, boolean expectedFound, String... strings) {
+        // Read contents of file
+        String fileString;
+        try {
+            fileString = readFile(outputDir, path);
+        } catch (Error e) {
+            if (!expectedFound) {
+                failed("Error reading file: " + e);
+                return;
+            }
+            throw e;
+        }
+        checkOutput(path, fileString, expectedFound, strings);
+    }
+
+    /**
+     * Check for content in (or not in) the one of the output streams written by
+     * javadoc. Within the search strings, the newline character \n
+     * will be translated to the platform newline character sequence.
+     * @param output the output stream to check
+     * @param expectedFound true if all of the search strings are expected
+     *  to be found, or false if all of the strings are expected to be
+     *  not found
+     * @param strings the strings to be searched for
+     */
+    public void checkOutput(Output output, boolean expectedFound, String... strings) {
+        checkOutput(output.toString(), outputMap.get(output), expectedFound, strings);
+    }
+
+    private void checkOutput(String path, String fileString, boolean expectedFound, String... strings) {
+        for (String stringToFind : strings) {
+//            log.logCheckOutput(path, expectedFound, stringToFind);
+            checking("checkOutput");
+            // Find string in file's contents
+            boolean isFound = findString(fileString, stringToFind);
+            if (isFound == expectedFound) {
+                passed(path + ": " + (isFound ? "found:" : "not found:") + "\n"
+                        + stringToFind + "\n");
+            } else {
+                failed(path + ": " + (isFound ? "found:" : "not found:") + "\n"
+                        + stringToFind + "\n");
             }
         }
     }
 
     /**
-     * Iterate through the list of given file pairs and diff each file.
-     *
-     * @param baseDir1 the directory containing the first set of files
-     * @param baseDir2 the directory containing the second set of files
-     * @param files the set of files to be compared
-     * @throws Error if any differences are found between
-     * file pairs.
-     */
-    public void runDiffs(String baseDir1, String baseDir2, String[] files) throws Error {
-        runDiffs(baseDir1, baseDir2, files, true);
-    }
-
-    /**
-     * Iterate through the list of given file pairs and diff each file.
-     *
-     * @param baseDir1 the directory containing the first set of files
-     * @param baseDir2 the directory containing the second set of files
-     * @param files the set of files to be compared
-     * @param throwErrorIfNoMatch flag to indicate whether or not to throw
-     * an error if the files do not match.
-     *
-     * @throws Error if any differences are found between
-     * file pairs and throwErrorIfNoMatch is true.
-     */
-    public void runDiffs(String baseDir1, String baseDir2, String[] files, boolean throwErrorIfNoMatch) throws Error {
-        File bd1 = new File(baseDir1);
-        File bd2 = new File(baseDir2);
-        for (String file : files) {
-            diff(bd1, bd2, file, throwErrorIfNoMatch);
-        }
-    }
-
-    /**
-     * Check the exit code of Javadoc and record whether the test passed
-     * or failed.
-     *
-     * @param expectedExitCode The exit code that is required for the test
-     * to pass.
-     * @param actualExitCode The actual exit code from the previous run of
-     * Javadoc.
-     */
-    public void checkExitCode(int expectedExitCode, int actualExitCode) {
-        numTestsRun++;
-        if (expectedExitCode == actualExitCode) {
-            System.out.println( "Passed" + "\n" + " got return code " +
-                actualExitCode);
-            numTestsPassed++;
-        } else {
-            System.out.println( "FAILED: expected return code " +
-                expectedExitCode + " but got " + actualExitCode);
-        }
-    }
-
-    /**
-     * Print a summary of the test results.
-     */
-    protected void printSummary() {
-        if ( numTestsRun != 0 && numTestsPassed == numTestsRun ) {
-            // Test passed
-            System.out.println("\n" + "All " + numTestsPassed
-                                             + " subtests passed");
-        } else {
-            // Test failed
-            throw new Error("\n" + (numTestsRun - numTestsPassed)
-                                    + " of " + (numTestsRun)
-                                    + " subtests failed\n");
+     * Check for files in (or not in) the generated output.
+     * @param expectedFound true if all of the files are expected
+     *  to be found, or false if all of the files are expected to be
+     *  not found
+     * @param paths the files to check, within the most recent output directory.
+     * */
+    public void checkFiles(boolean expectedFound, String... paths) {
+        for (String path: paths) {
+//            log.logCheckFile(path, expectedFound);
+            checking("checkFile");
+            File file = new File(outputDir, path);
+            boolean isFound = file.exists();
+            if (isFound == expectedFound) {
+                passed(path + ": " + (isFound ? "found:" : "not found:") + "\n");
+            } else {
+                failed(path + ": " + (isFound ? "found:" : "not found:") + "\n");
+            }
         }
     }
 
     /**
-     * Print the output stored in the buffers.
-     */
-    protected void printJavadocOutput() {
-        System.out.println(STANDARD_OUTPUT + " : \n" + getStandardOutput());
-        System.err.println(ERROR_OUTPUT + " : \n" + getErrorOutput());
-        System.err.println(WARNING_OUTPUT + " : \n" + getWarningOutput());
-        System.out.println(NOTICE_OUTPUT + " : \n" + getNoticeOutput());
-    }
-
-    /**
-     * Read the file and return it as a string.
-     *
-     * @param fileName  the name of the file to read
-     * @return          the file in string format
-     */
-    public String readFileToString(String fileName) throws Error {
-        return readFileToString(outputDir, fileName);
-    }
-
-    /**
-     * Read the file and return it as a string.
-     *
-     * @param baseDir   the directory in which to locate the file
-     * @param fileName  the name of the file to read
-     * @return          the file in string format
+     * Check that a series of strings are found in order in a file in
+     * the generated output.
+     * @param path the file to check
+     * @param strings  the strings whose order to check
      */
-    private String readFileToString(File baseDir, String fileName) throws Error {
-        switch (fileName) {
-            case ERROR_OUTPUT:
-                return getErrorOutput();
-            case NOTICE_OUTPUT:
-                return getNoticeOutput();
-            case WARNING_OUTPUT:
-                return getWarningOutput();
-            case STANDARD_OUTPUT:
-                return getStandardOutput();
-        }
-        try {
-            File file = new File(baseDir, fileName);
-            if ( !file.exists() ) {
-                System.out.println("\n" + "FILE DOES NOT EXIST: " + fileName);
+    public void checkOrder(String path, String... strings) {
+        String fileString = readOutputFile(path);
+        int prevIndex = -1;
+        for (String s : strings) {
+            int currentIndex = fileString.indexOf(s);
+            checking(s + " at index " + currentIndex);
+            if (currentIndex >= prevIndex) {
+                passed(s + "is in the correct order");
+            } else {
+                failed(s + " is in the wrong order.");
             }
-            char[] allChars;
-            try (BufferedReader in = new BufferedReader(new FileReader(file))) {
-                // Create an array of characters the size of the file
-                allChars = new char[(int)file.length()];
-                // Read the characters into the allChars array
-                in.read(allChars, 0, (int)file.length());
-            }
-
-            // Convert to a string
-            String allCharsString = new String(allChars);
-            return allCharsString;
-        } catch (FileNotFoundException e) {
-            System.err.println(e);
-            throw new Error("File not found: " + fileName);
-        } catch (IOException e) {
-            System.err.println(e);
-            throw new Error("Error reading file: " + fileName);
+            prevIndex = currentIndex;
         }
     }
 
     /**
-     * Compare the two given files.
+     * Compare a set of files in each of two directories.
      *
-     * @param baseDir1 the directory in which to locate the first file
-     * @param baseDir2 the directory in which to locate the second file
-     * @param file the file to compare in the two base directories
-     * @param throwErrorIFNoMatch flag to indicate whether or not to throw
-     * an error if the files do not match.
-     * @return true if the files are the same and false otherwise.
+     * @param baseDir1 the directory containing the first set of files
+     * @param baseDir2 the directory containing the second set of files
+     * @param files the set of files to be compared
      */
-    private boolean diff(File baseDir1, File baseDir2, String file,
-            boolean throwErrorIFNoMatch) throws Error {
-        String file1Contents = readFileToString(baseDir1, file);
-        String file2Contents = readFileToString(baseDir2, file);
-        numTestsRun++;
-        if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
-            System.out.println("Diff successful: " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
-            numTestsPassed++;
-            return true;
-        } else if (throwErrorIFNoMatch) {
-            throw new Error("Diff failed: " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
-        } else {
-            return false;
+    public void diff(String baseDir1, String baseDir2, String... files) {
+        File bd1 = new File(baseDir1);
+        File bd2 = new File(baseDir2);
+        for (String file : files) {
+            diff(bd1, bd2, file);
         }
     }
 
     /**
-     * Search for the string in the given file and return true
-     * if the string was found.
-     *
-     * @param fileString    the contents of the file to search through
-     * @param stringToFind  the string to search for
-     * @return              true if the string was found
-     */
-    private boolean findString(String fileString, String stringToFind) {
-        return fileString.contains(stringToFind.replace("\n", NL));
-    }
-
-
-    /**
-     * Return the standard output.
-     * @return the standard output
-     */
-    public String getStandardOutput() {
-        return standardOut.toString();
-    }
-
-    /**
-     * Return the error output.
-     * @return the error output
-     */
-    public String getErrorOutput() {
-        return errors.getBuffer().toString();
-    }
-
-    /**
-     * Return the notice output.
-     * @return the notice output
-     */
-    public String getNoticeOutput() {
-        return notices.getBuffer().toString();
-    }
-
-    /**
-     * Return the warning output.
-     * @return the warning output
-     */
-    public String getWarningOutput() {
-        return warnings.getBuffer().toString();
-    }
-
-    /**
      * A utility to copy a directory from one place to another.
-     * We may possibly want to move this to our doclet toolkit in
-     * the near future and maintain it from there.
      *
      * @param targetDir the directory to copy.
      * @param destDir the destination to copy the directory to.
      */
-    public static void copyDir(String targetDir, String destDir) {
-        if (targetDir.endsWith("SCCS")) {
-            return;
-        }
+    // TODO: convert to using java.nio.Files.walkFileTree
+    public void copyDir(String targetDir, String destDir) {
         try {
             File targetDirObj = new File(targetDir);
             File destDirParentObj = new File(destDir);
@@ -617,7 +498,7 @@
                 File srcFile = new File(targetDirObj, file);
                 File destFile = new File(destDirObj, file);
                 if (srcFile.isFile()) {
-                    System.out.println("Copying " + srcFile + " to " + destFile);
+                    out.println("Copying " + srcFile + " to " + destFile);
                     copyFile(destFile, srcFile);
                 } else if(srcFile.isDirectory()) {
                     copyDir(srcFile.getAbsolutePath(), destDirObj.getAbsolutePath());
@@ -633,25 +514,235 @@
      *
      * @param destfile the destination file
      * @param srcfile the source file
-     * @throws SecurityException
      * @throws IOException
      */
-    public static void copyFile(File destfile, File srcfile)
-        throws IOException {
-        byte[] bytearr = new byte[512];
-        int len;
-        FileInputStream input = new FileInputStream(srcfile);
-        File destDir = destfile.getParentFile();
-        destDir.mkdirs();
-        FileOutputStream output = new FileOutputStream(destfile);
+    public void copyFile(File destfile, File srcfile) throws IOException {
+        Files.copy(srcfile.toPath(), destfile.toPath());
+    }
+
+    /**
+     * Read a file from the output directory.
+     *
+     * @param fileName  the name of the file to read
+     * @return          the file in string format
+     */
+    public String readOutputFile(String fileName) throws Error {
+        return readFile(outputDir, fileName);
+    }
+
+    protected String readFile(String fileName) throws Error {
+        return readFile(outputDir, fileName);
+    }
+
+    protected String readFile(String baseDir, String fileName) throws Error {
+        return readFile(new File(baseDir), fileName);
+    }
+
+    /**
+     * Read the file and return it as a string.
+     *
+     * @param baseDir   the directory in which to locate the file
+     * @param fileName  the name of the file to read
+     * @return          the file in string format
+     */
+    private String readFile(File baseDir, String fileName) throws Error {
         try {
-            while ((len = input.read(bytearr)) != -1) {
-                output.write(bytearr, 0, len);
+            File file = new File(baseDir, fileName);
+            SoftReference<String> ref = fileContentCache.get(file);
+            String content = (ref == null) ? null : ref.get();
+            if (content != null)
+                return content;
+
+            content = new String(Files.readAllBytes(file.toPath()));
+            fileContentCache.put(file, new SoftReference(content));
+            return content;
+        } catch (FileNotFoundException e) {
+            System.err.println(e);
+            throw new Error("File not found: " + fileName);
+        } catch (IOException e) {
+            System.err.println(e);
+            throw new Error("Error reading file: " + fileName);
+        }
+    }
+
+    protected void checking(String message) {
+        numTestsRun++;
+        print("Starting subtest " + numTestsRun, message);
+    }
+
+    protected void passed(String message) {
+        numTestsPassed++;
+        print("Passed", message);
+    }
+
+    protected void failed(String message) {
+        print("FAILED", message);
+    }
+
+    private void print(String prefix, String message) {
+        if (message.isEmpty())
+            out.println(prefix);
+        else {
+            out.print(prefix);
+            out.print(": ");
+            out.println(message.replace("\n", NL));
+        }
+    }
+
+    /**
+     * Print a summary of the test results.
+     */
+    protected void printSummary() {
+//        log.write();
+        if (numTestsRun != 0 && numTestsPassed == numTestsRun) {
+            // Test passed
+            out.println();
+            out.println("All " + numTestsPassed + " subtests passed");
+        } else {
+            // Test failed
+            throw new Error((numTestsRun - numTestsPassed)
+                    + " of " + (numTestsRun)
+                    + " subtests failed");
+        }
+    }
+
+    /**
+     * Search for the string in the given file and return true
+     * if the string was found.
+     *
+     * @param fileString    the contents of the file to search through
+     * @param stringToFind  the string to search for
+     * @return              true if the string was found
+     */
+    private boolean findString(String fileString, String stringToFind) {
+        // javadoc (should) always use the platform newline sequence,
+        // but in the strings to find it is more convenient to use the Java
+        // newline character. So we translate \n to NL before we search.
+        stringToFind = stringToFind.replace("\n", NL);
+        return fileString.contains(stringToFind);
+    }
+
+    /**
+     * Compare the two given files.
+     *
+     * @param baseDir1 the directory in which to locate the first file
+     * @param baseDir2 the directory in which to locate the second file
+     * @param file the file to compare in the two base directories
+     * @param throwErrorIFNoMatch flag to indicate whether or not to throw
+     * an error if the files do not match.
+     * @return true if the files are the same and false otherwise.
+     */
+    private void diff(File baseDir1, File baseDir2, String file) {
+        String file1Contents = readFile(baseDir1, file);
+        String file2Contents = readFile(baseDir2, file);
+        checking("diff " + new File(baseDir1, file) + ", " + new File(baseDir2, file));
+        if (file1Contents.trim().compareTo(file2Contents.trim()) == 0) {
+            passed("files are equal");
+        } else {
+            failed("files differ");
+        }
+    }
+
+    /**
+     * Utility class to simplify the handling of temporarily setting a
+     * new stream for System.out or System.err.
+     */
+    private static class StreamOutput {
+        // functional interface to set a stream.
+        private interface Initializer {
+            void set(PrintStream s);
+        }
+
+        private final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        private final PrintStream ps = new PrintStream(baos);
+        private final PrintStream prev;
+        private final Initializer init;
+
+        StreamOutput(PrintStream s, Initializer init) {
+            prev = s;
+            init.set(ps);
+            this.init = init;
+        }
+
+        String close() {
+            init.set(prev);
+            ps.close();
+            return baos.toString();
+        }
+    }
+
+    /**
+     * Utility class to simplify the handling of creating an in-memory PrintWriter.
+     */
+    private static class WriterOutput {
+        private final StringWriter sw = new StringWriter();
+        final PrintWriter pw = new PrintWriter(sw);
+        String close() {
+            pw.close();
+            return sw.toString();
+        }
+    }
+
+
+//    private final Logger log = new Logger();
+
+    //--------- Logging --------------------------------------------------------
+    //
+    // This class writes out the details of calls to checkOutput and checkFile
+    // in a canonical way, so that the resulting file can be checked against
+    // similar files from other versions of JavadocTester using the same logging
+    // facilities.
+
+    static class Logger {
+        private static final int PREFIX = 40;
+        private static final int SUFFIX = 20;
+        private static final int MAX = PREFIX + SUFFIX;
+        List<String> tests = new ArrayList<>();
+        String outDir;
+        String rootDir = rootDir();
+
+        static String rootDir() {
+            File f = new File(".").getAbsoluteFile();
+            while (!new File(f, ".hg").exists())
+                f = f.getParentFile();
+            return f.getPath();
+        }
+
+        void setOutDir(File outDir) {
+            this.outDir = outDir.getPath();
+        }
+
+        void logCheckFile(String file, boolean positive) {
+            // Strip the outdir because that will typically not be the same
+            if (file.startsWith(outDir + "/"))
+                file = file.substring(outDir.length() + 1);
+            tests.add(file + " " + positive);
+        }
+
+        void logCheckOutput(String file, boolean positive, String text) {
+            // Compress the string to be displayed in the log file
+            String simpleText = text.replaceAll("\\s+", " ").replace(rootDir, "[ROOT]");
+            if (simpleText.length() > MAX)
+                simpleText = simpleText.substring(0, PREFIX)
+                        + "..." + simpleText.substring(simpleText.length() - SUFFIX);
+            // Strip the outdir because that will typically not be the same
+            if (file.startsWith(outDir + "/"))
+                file = file.substring(outDir.length() + 1);
+            // The use of text.hashCode ensure that all of "text" is taken into account
+            tests.add(file + " " + positive + " " + text.hashCode() + " " + simpleText);
+        }
+
+        void write() {
+            // sort the log entries because the subtests may not be executed in the same order
+            tests.sort((a, b) -> a.compareTo(b));
+            try (BufferedWriter bw = new BufferedWriter(new FileWriter("tester.log"))) {
+                for (String t: tests) {
+                    bw.write(t);
+                    bw.newLine();
+                }
+            } catch (IOException e) {
+                throw new Error("problem writing log: " + e);
             }
-        } catch (FileNotFoundException | SecurityException exc) {
-        } finally {
-            input.close();
-            output.close();
         }
     }
 }