1 /*

     2  * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved.

     3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     4  *

     5  * This code is free software; you can redistribute it and/or modify it

     6  * under the terms of the GNU General Public License version 2 only, as

     7  * published by the Free Software Foundation.

     8  *

     9  * This code is distributed in the hope that it will be useful, but WITHOUT

    10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    12  * version 2 for more details (a copy is included in the LICENSE file that

    13  * accompanied this code).

    14  *

    15  * You should have received a copy of the GNU General Public License version

    16  * 2 along with this work; if not, write to the Free Software Foundation,

    17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    18  *

    19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    20  * or visit www.oracle.com if you need additional information or have any

    21  * questions.

    22  */

    23 

    24 package combo;

    25 

    26 import javax.tools.JavaCompiler;

    27 import javax.tools.StandardJavaFileManager;

    28 import javax.tools.ToolProvider;

    29 

    30 import java.io.IOException;

    31 import java.util.ArrayList;

    32 import java.util.HashMap;

    33 import java.util.List;

    34 import java.util.Map;

    35 import java.util.Optional;

    36 import java.util.Stack;

    37 import java.util.function.Consumer;

    38 import java.util.function.Predicate;

    39 import java.util.function.Supplier;

    40 

    41 

    42 /**

    43  * An helper class for defining combinatorial (aka "combo" tests). A combo test is made up of one

    44  * or more 'dimensions' - each of which represent a different axis of the test space. For instance,

    45  * if we wanted to test class/interface declaration, one dimension could be the keyword used for

    46  * the declaration (i.e. 'class' vs. 'interface') while another dimension could be the class/interface

    47  * modifiers (i.e. 'public', 'pachake-private' etc.). A combo test consists in running a test instance

    48  * for each point in the test space; that is, for any combination of the combo test dimension:

    49  * <p>

    50  * 'public' 'class'

    51  * 'public' interface'

    52  * 'package-private' 'class'

    53  * 'package-private' 'interface'

    54  * ...

    55  * <p>

    56  * A new test instance {@link ComboInstance} is created, and executed, after its dimensions have been

    57  * initialized accordingly. Each instance can either pass, fail or throw an unexpected error; this helper

    58  * class defines several policies for how failures should be handled during a combo test execution

    59  * (i.e. should errors be ignored? Do we want the first failure to result in a failure of the whole

    60  * combo test?).

    61  * <p>

    62  * Additionally, this helper class allows to specify filter methods that can be used to throw out

    63  * illegal combinations of dimensions - for instance, in the example above, we might want to exclude

    64  * all combinations involving 'protected' and 'private' modifiers, which are disallowed for toplevel

    65  * declarations.

    66  * <p>

    67  * While combo tests can be used for a variety of workloads, typically their main task will consist

    68  * in performing some kind of javac compilation. For this purpose, this framework defines an optimized

    69  * javac context {@link ReusableContext} which can be shared across multiple combo instances,

    70  * when the framework detects it's safe to do so. This allows to reduce the overhead associated with

    71  * compiler initialization when the test space is big.

    72  */

    73 public class ComboTestHelper<X extends ComboInstance<X>> {

    74 

    75     /** Failure mode. */

    76     FailMode failMode = FailMode.FAIL_FAST;

    77 

    78     /** Ignore mode. */

    79     IgnoreMode ignoreMode = IgnoreMode.IGNORE_NONE;

    80 

    81     /** Combo test instance filter. */

    82     Optional<Predicate<X>> optFilter = Optional.empty();

    83 

    84     /** Combo test dimensions. */

    85     List<DimensionInfo<?>> dimensionInfos = new ArrayList<>();

    86 

    87     /** Combo test stats. */

    88     Info info = new Info();

    89 

    90     /** Shared JavaCompiler used across all combo test instances. */

    91     JavaCompiler comp = ToolProvider.getSystemJavaCompiler();

    92 

    93     /** Shared file manager used across all combo test instances. */

    94     StandardJavaFileManager fm = comp.getStandardFileManager(null, null, null);

    95 

    96     /** Shared context used across all combo instances. */

    97     ReusableContext context = new ReusableContext();

    98 

    99     /**

   100      * Set failure mode for this combo test.

   101      */

   102     public ComboTestHelper<X> withFailMode(FailMode failMode) {

   103         this.failMode = failMode;

   104         return this;

   105     }

   106 

   107     /**

   108      * Set ignore mode for this combo test.

   109      */

   110     public ComboTestHelper<X> withIgnoreMode(IgnoreMode ignoreMode) {

   111         this.ignoreMode = ignoreMode;

   112         return this;

   113     }

   114 

   115     /**

   116      * Set a filter for combo test instances to be ignored.

   117      */

   118     public ComboTestHelper<X> withFilter(Predicate<X> filter) {

   119         optFilter = Optional.of(optFilter.map(filter::and).orElse(filter));

   120         return this;

   121     }

   122 

   123     /**

   124      * Adds a new dimension to this combo test, with a given name an array of values.

   125      */

   126     @SafeVarargs

   127     public final <D> ComboTestHelper<X> withDimension(String name, D... dims) {

   128         return withDimension(name, null, dims);

   129     }

   130 

   131     /**

   132      * Adds a new dimension to this combo test, with a given name, an array of values and a

   133      * coresponding setter to be called in order to set the dimension value on the combo test instance

   134      * (before test execution).

   135      */

   136     @SuppressWarnings("unchecked")

   137     @SafeVarargs

   138     public final <D> ComboTestHelper<X> withDimension(String name, DimensionSetter<X, D> setter, D... dims) {

   139         dimensionInfos.add(new DimensionInfo<>(name, dims, setter));

   140         return this;

   141     }

   142 

   143     /**

   144      * Adds a new array dimension to this combo test, with a given base name. This allows to specify

   145      * multiple dimensions at once; the names of the underlying dimensions will be generated from the

   146      * base name, using standard array bracket notation - i.e. "DIM[0]", "DIM[1]", etc.

   147      */

   148     @SafeVarargs

   149     public final <D> ComboTestHelper<X> withArrayDimension(String name, int size, D... dims) {

   150         return withArrayDimension(name, null, size, dims);

   151     }

   152 

   153     /**

   154      * Adds a new array dimension to this combo test, with a given base name, an array of values and a

   155      * coresponding array setter to be called in order to set the dimension value on the combo test

   156      * instance (before test execution). This allows to specify multiple dimensions at once; the names

   157      * of the underlying dimensions will be generated from the base name, using standard array bracket

   158      * notation - i.e. "DIM[0]", "DIM[1]", etc.

   159      */

   160     @SafeVarargs

   161     public final <D> ComboTestHelper<X> withArrayDimension(String name, ArrayDimensionSetter<X, D> setter, int size, D... dims) {

   162         for (int i = 0 ; i < size ; i++) {

   163             dimensionInfos.add(new ArrayDimensionInfo<>(name, dims, i, setter));

   164         }

   165         return this;

   166     }

   167 

   168     /**

   169      * Returns the stat object associated with this combo test.

   170      */

   171     public Info info() {

   172         return info;

   173     }

   174 

   175     /**

   176      * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and

   177      * execute a new test instance (built using given supplier) for each such combination.

   178      */

   179     public void run(Supplier<X> instanceBuilder) {

   180         run(instanceBuilder, null);

   181     }

   182 

   183     /**

   184      * Runs this combo test. This will generate the combinatorial explosion of all dimensions, and

   185      * execute a new test instance (built using given supplier) for each such combination. Before

   186      * executing the test instance entry point, the supplied initialization method is called on

   187      * the test instance; this is useful for ad-hoc test instance initialization once all the dimension

   188      * values have been set.

   189      */

   190     public void run(Supplier<X> instanceBuilder, Consumer<X> initAction) {

   191         runInternal(0, new Stack<>(), instanceBuilder, Optional.ofNullable(initAction));

   192         end();

   193     }

   194 

   195     /**

   196      * Generate combinatorial explosion of all dimension values and create a new test instance

   197      * for each combination.

   198      */

   199     @SuppressWarnings({"unchecked", "rawtypes"})

   200     private void runInternal(int index, Stack<DimensionBinding<?>> bindings, Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction) {

   201         if (index == dimensionInfos.size()) {

   202             runCombo(instanceBuilder, initAction, bindings);

   203         } else {

   204             DimensionInfo<?> dinfo = dimensionInfos.get(index);

   205             for (Object d : dinfo.dims) {

   206                 bindings.push(new DimensionBinding(d, dinfo));

   207                 runInternal(index + 1, bindings, instanceBuilder, initAction);

   208                 bindings.pop();

   209             }

   210         }

   211     }

   212 

   213     /**

   214      * Run a new test instance using supplied dimension bindings. All required setters and initialization

   215      * method are executed before calling the instance main entry point. Also checks if the instance

   216      * is compatible with the specified test filters; if not, the test is simply skipped.

   217      */

   218     @SuppressWarnings("unchecked")

   219     private void runCombo(Supplier<X> instanceBuilder, Optional<Consumer<X>> initAction, List<DimensionBinding<?>> bindings) {

   220         X x = instanceBuilder.get();

   221         for (DimensionBinding<?> binding : bindings) {

   222             binding.init(x);

   223         }

   224         initAction.ifPresent(action -> action.accept(x));

   225         info.comboCount++;

   226         if (!optFilter.isPresent() || optFilter.get().test(x)) {

   227             x.run(new Env(bindings));

   228             if (failMode.shouldStop(ignoreMode, info)) {

   229                 end();

   230             }

   231         } else {

   232             info.skippedCount++;

   233         }

   234     }

   235 

   236     /**

   237      * This method is executed upon combo test completion (either normal or erroneous). Closes down

   238      * all pending resources and dumps useful stats info.

   239      */

   240     private void end() {

   241         try {

   242             fm.close();

   243             if (info.hasFailures()) {

   244                 throw new AssertionError("Failure when executing combo:" + info.lastFailure.orElse(""));

   245             } else if (info.hasErrors()) {

   246                 throw new AssertionError("Unexpected exception while executing combo", info.lastError.get());

   247             }

   248         } catch (IOException ex) {

   249             throw new AssertionError("Failure when closing down shared file manager; ", ex);

   250         } finally {

   251             info.dump();

   252         }

   253     }

   254 

   255     /**

   256      * Functional interface for specifying combo test instance setters.

   257      */

   258     public interface DimensionSetter<X extends ComboInstance<X>, D> {

   259         void set(X x, D d);

   260     }

   261 

   262     /**

   263      * Functional interface for specifying combo test instance array setters. The setter method

   264      * receives an extra argument for the index of the array element to be set.

   265      */

   266     public interface ArrayDimensionSetter<X extends ComboInstance<X>, D> {

   267         void set(X x, D d, int index);

   268     }

   269 

   270     /**

   271      * Dimension descriptor; each dimension has a name, an array of value and an optional setter

   272      * to be called on the associated combo test instance.

   273      */

   274     class DimensionInfo<D> {

   275         String name;

   276         D[] dims;

   277         boolean isParameter;

   278         Optional<DimensionSetter<X, D>> optSetter;

   279 

   280         DimensionInfo(String name, D[] dims, DimensionSetter<X, D> setter) {

   281             this.name = name;

   282             this.dims = dims;

   283             this.optSetter = Optional.ofNullable(setter);

   284             this.isParameter = dims[0] instanceof ComboParameter;

   285         }

   286     }

   287 

   288     /**

   289      * Array dimension descriptor. The dimension name is derived from a base name and an index using

   290      * standard bracket notation; ; the setter accepts an additional 'index' argument to point

   291      * to the array element to be initialized.

   292      */

   293     class ArrayDimensionInfo<D> extends DimensionInfo<D> {

   294         public ArrayDimensionInfo(String name, D[] dims, int index, ArrayDimensionSetter<X, D> setter) {

   295             super(String.format("%s[%d]", name, index), dims,

   296                     setter != null ? (x, d) -> setter.set(x, d, index) : null);

   297         }

   298     }

   299 

   300     /**

   301      * Failure policies for a combo test run.

   302      */

   303     public enum FailMode {

   304         /** Combo test fails when first failure is detected. */

   305         FAIL_FAST,

   306         /** Combo test fails after all instances have been executed. */

   307         FAIL_AFTER;

   308 

   309         boolean shouldStop(IgnoreMode ignoreMode, Info info) {

   310             switch (this) {

   311                 case FAIL_FAST:

   312                     return !ignoreMode.canIgnore(info);

   313                 default:

   314                     return false;

   315             }

   316         }

   317     }

   318 

   319     /**

   320      * Ignore policies for a combo test run.

   321      */

   322     public enum IgnoreMode {

   323         /** No error or failure is ignored. */

   324         IGNORE_NONE,

   325         /** Only errors are ignored. */

   326         IGNORE_ERRORS,

   327         /** Only failures are ignored. */

   328         IGNORE_FAILURES,

   329         /** Both errors and failures are ignored. */

   330         IGNORE_ALL;

   331 

   332         boolean canIgnore(Info info) {

   333             switch (this) {

   334                 case IGNORE_ERRORS:

   335                     return info.failCount == 0;

   336                 case IGNORE_FAILURES:

   337                     return info.errCount == 0;

   338                 case IGNORE_ALL:

   339                     return true;

   340                 default:

   341                     return info.failCount == 0 && info.errCount == 0;

   342             }

   343         }

   344     }

   345 

   346     /**

   347      * A dimension binding. This is essentially a pair of a dimension value and its corresponding

   348      * dimension info.

   349      */

   350     class DimensionBinding<D> {

   351         D d;

   352         DimensionInfo<D> info;

   353 

   354         DimensionBinding(D d, DimensionInfo<D> info) {

   355             this.d = d;

   356             this.info = info;

   357         }

   358 

   359         void init(X x) {

   360             info.optSetter.ifPresent(setter -> setter.set(x, d));

   361         }

   362 

   363         public String toString() {

   364             return String.format("(%s -> %s)", info.name, d);

   365         }

   366     }

   367 

   368     /**

   369      * This class is used to keep track of combo tests stats; info such as numbero of failures/errors,

   370      * number of times a context has been shared/dropped are all recorder here.

   371      */

   372     public static class Info {

   373         int failCount;

   374         int errCount;

   375         int passCount;

   376         int comboCount;

   377         int skippedCount;

   378         int ctxReusedCount;

   379         int ctxDroppedCount;

   380         Optional<String> lastFailure = Optional.empty();

   381         Optional<Throwable> lastError = Optional.empty();

   382 

   383         void dump() {

   384             System.err.println(String.format("%d total checks executed", comboCount));

   385             System.err.println(String.format("%d successes found", passCount));

   386             System.err.println(String.format("%d failures found", failCount));

   387             System.err.println(String.format("%d errors found", errCount));

   388             System.err.println(String.format("%d skips found", skippedCount));

   389             System.err.println(String.format("%d contexts shared", ctxReusedCount));

   390             System.err.println(String.format("%d contexts dropped", ctxDroppedCount));

   391         }

   392 

   393         public boolean hasFailures() {

   394             return failCount != 0;

   395         }

   396 

   397         public boolean hasErrors() {

   398             return errCount != 0;

   399         }

   400     }

   401 

   402     /**

   403      * THe execution environment for a given combo test instance. An environment contains the

   404      * bindings for all the dimensions, along with the combo parameter cache (this is non-empty

   405      * only if one or more dimensions are subclasses of the {@code ComboParameter} interface).

   406      */

   407     class Env {

   408         List<DimensionBinding<?>> bindings;

   409         Map<String, ComboParameter> parametersCache = new HashMap<>();

   410 

   411         @SuppressWarnings({"Unchecked", "rawtypes"})

   412         Env(List<DimensionBinding<?>> bindings) {

   413             this.bindings = bindings;

   414             for (DimensionBinding<?> binding : bindings) {

   415                 if (binding.info.isParameter) {

   416                     parametersCache.put(binding.info.name, (ComboParameter)binding.d);

   417                 };

   418             }

   419         }

   420 

   421         Info info() {

   422             return ComboTestHelper.this.info();

   423         }

   424 

   425         StandardJavaFileManager fileManager() {

   426             return fm;

   427         }

   428 

   429         JavaCompiler javaCompiler() {

   430             return comp;

   431         }

   432 

   433         ReusableContext context() {

   434             return context;

   435         }

   436 

   437         ReusableContext setContext(ReusableContext context) {

   438             return ComboTestHelper.this.context = context;

   439         }

   440     }

   441 }

   442 

   443 

   444