1 #

     2 # ----------------------------------------------------------------------------------------------------

     3 #

     4 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.

     5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

     6 #

     7 # This code is free software; you can redistribute it and/or modify it

     8 # under the terms of the GNU General Public License version 2 only, as

     9 # published by the Free Software Foundation.

    10 #

    11 # This code is distributed in the hope that it will be useful, but WITHOUT

    12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

    13 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License

    14 # version 2 for more details (a copy is included in the LICENSE file that

    15 # accompanied this code).

    16 #

    17 # You should have received a copy of the GNU General Public License version

    18 # 2 along with this work; if not, write to the Free Software Foundation,

    19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

    20 #

    21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

    22 # or visit www.oracle.com if you need additional information or have any

    23 # questions.

    24 #

    25 # ----------------------------------------------------------------------------------------------------

    26 

    27 import sanitycheck

    28 import itertools

    29 import json

    30 

    31 import mx

    32 import mx_graal

    33 

    34 def _run_benchmark(args, availableBenchmarks, runBenchmark):

    35 

    36     vmOpts, benchmarksAndOptions = mx.extract_VM_args(args, useDoubleDash=availableBenchmarks is None)

    37 

    38     if availableBenchmarks is None:

    39         harnessArgs = benchmarksAndOptions

    40         return runBenchmark(None, harnessArgs, vmOpts)

    41 

    42     if len(benchmarksAndOptions) == 0:

    43         mx.abort('at least one benchmark name or "all" must be specified')

    44     benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions))

    45     harnessArgs = benchmarksAndOptions[len(benchmarks):]

    46 

    47     if 'all' in benchmarks:

    48         benchmarks = availableBenchmarks

    49     else:

    50         for bm in benchmarks:

    51             if bm not in availableBenchmarks:

    52                 mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks))

    53 

    54     failed = []

    55     for bm in benchmarks:

    56         if not runBenchmark(bm, harnessArgs, vmOpts):

    57             failed.append(bm)

    58 

    59     if len(failed) != 0:

    60         mx.abort('Benchmark failures: ' + str(failed))

    61 

    62 def deoptalot(args):

    63     """bootstrap a VM with DeoptimizeALot and VerifyOops on

    64 

    65     If the first argument is a number, the process will be repeated

    66     this number of times. All other arguments are passed to the VM."""

    67     count = 1

    68     if len(args) > 0 and args[0].isdigit():

    69         count = int(args[0])

    70         del args[0]

    71 

    72     for _ in range(count):

    73         if not mx_graal.run_vm(['-XX:-TieredCompilation', '-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version']) == 0:

    74             mx.abort("Failed")

    75 

    76 def longtests(args):

    77 

    78     deoptalot(['15', '-Xmx48m'])

    79 

    80     dacapo(['100', 'eclipse', '-esa'])

    81 

    82 def dacapo(args):

    83     """run one or more DaCapo benchmarks"""

    84 

    85     def launcher(bm, harnessArgs, extraVmOpts):

    86         return sanitycheck.getDacapo(bm, harnessArgs).test(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

    87 

    88     _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher)

    89 

    90 def scaladacapo(args):

    91     """run one or more Scala DaCapo benchmarks"""

    92 

    93     def launcher(bm, harnessArgs, extraVmOpts):

    94         return sanitycheck.getScalaDacapo(bm, harnessArgs).test(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

    95 

    96     _run_benchmark(args, sanitycheck.dacapoScalaSanityWarmup.keys(), launcher)

    97 

    98 

    99 """

   100 Extra benchmarks to run from 'bench()'.

   101 """

   102 extraBenchmarks = []

   103 

   104 def bench(args):

   105     """run benchmarks and parse their output for results

   106 

   107     Results are JSON formated : {group : {benchmark : score}}."""

   108     resultFile = None

   109     if '-resultfile' in args:

   110         index = args.index('-resultfile')

   111         if index + 1 < len(args):

   112             resultFile = args[index + 1]

   113             del args[index]

   114             del args[index]

   115         else:

   116             mx.abort('-resultfile must be followed by a file name')

   117     resultFileCSV = None

   118     if '-resultfilecsv' in args:

   119         index = args.index('-resultfilecsv')

   120         if index + 1 < len(args):

   121             resultFileCSV = args[index + 1]

   122             del args[index]

   123             del args[index]

   124         else:

   125             mx.abort('-resultfilecsv must be followed by a file name')

   126     vm = mx_graal.get_vm()

   127     if len(args) is 0:

   128         args = ['all']

   129 

   130     vmArgs = [arg for arg in args if arg.startswith('-')]

   131 

   132     def benchmarks_in_group(group):

   133         prefix = group + ':'

   134         return [a[len(prefix):] for a in args if a.startswith(prefix)]

   135 

   136     results = {}

   137     benchmarks = []

   138     # DaCapo

   139     if 'dacapo' in args or 'all' in args:

   140         benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)

   141     else:

   142         dacapos = benchmarks_in_group('dacapo')

   143         for dacapo in dacapos:

   144             if dacapo not in sanitycheck.dacapoSanityWarmup.keys():

   145                 mx.abort('Unknown DaCapo : ' + dacapo)

   146             iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark]

   147             if iterations > 0:

   148                 benchmarks += [sanitycheck.getDacapo(dacapo, ['-n', str(iterations)])]

   149 

   150     if 'scaladacapo' in args or 'all' in args:

   151         benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)

   152     else:

   153         scaladacapos = benchmarks_in_group('scaladacapo')

   154         for scaladacapo in scaladacapos:

   155             if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():

   156                 mx.abort('Unknown Scala DaCapo : ' + scaladacapo)

   157             iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark]

   158             if iterations > 0:

   159                 benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])]

   160 

   161     # Bootstrap

   162     if 'bootstrap' in args or 'all' in args:

   163         benchmarks += sanitycheck.getBootstraps()

   164     # SPECjvm2008

   165     if 'specjvm2008' in args or 'all' in args:

   166         benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])]

   167     else:

   168         specjvms = benchmarks_in_group('specjvm2008')

   169         for specjvm in specjvms:

   170             benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])]

   171 

   172     if 'specjbb2005' in args or 'all' in args:

   173         benchmarks += [sanitycheck.getSPECjbb2005()]

   174 

   175     if 'specjbb2013' in args:  # or 'all' in args //currently not in default set

   176         benchmarks += [sanitycheck.getSPECjbb2013()]

   177 

   178     if 'ctw-full' in args:

   179         benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))

   180     if 'ctw-noinline' in args:

   181         benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))

   182 

   183     for f in extraBenchmarks:

   184         f(args, vm, benchmarks)

   185 

   186     for test in benchmarks:

   187         for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():

   188             group = results.setdefault(groupName, {})

   189             group.update(res)

   190     mx.log(json.dumps(results))

   191     if resultFile:

   192         with open(resultFile, 'w') as f:

   193             f.write(json.dumps(results))

   194     if resultFileCSV:

   195         with open(resultFileCSV, 'w') as f:

   196             for key1, value1 in results.iteritems():

   197                 f.write('%s;\n' % (str(key1)))

   198                 for key2, value2 in sorted(value1.iteritems()):

   199                     f.write('%s; %s;\n' % (str(key2), str(value2)))

   200 

   201 def specjvm2008(args):

   202     """run one or more SPECjvm2008 benchmarks"""

   203 

   204     def launcher(bm, harnessArgs, extraVmOpts):

   205         return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

   206 

   207     availableBenchmarks = set(sanitycheck.specjvm2008Names)

   208     if "all" not in args:

   209         # only add benchmark groups if we are not running "all"

   210         for name in sanitycheck.specjvm2008Names:

   211             parts = name.rsplit('.', 1)

   212             if len(parts) > 1:

   213                 assert len(parts) == 2

   214                 group = parts[0]

   215                 availableBenchmarks.add(group)

   216 

   217     _run_benchmark(args, sorted(availableBenchmarks), launcher)

   218 

   219 def specjbb2013(args):

   220     """run the composite SPECjbb2013 benchmark"""

   221 

   222     def launcher(bm, harnessArgs, extraVmOpts):

   223         assert bm is None

   224         return sanitycheck.getSPECjbb2013(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

   225 

   226     _run_benchmark(args, None, launcher)

   227 

   228 def specjbb2015(args):

   229     """run the composite SPECjbb2015 benchmark"""

   230 

   231     def launcher(bm, harnessArgs, extraVmOpts):

   232         assert bm is None

   233         return sanitycheck.getSPECjbb2015(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

   234 

   235     _run_benchmark(args, None, launcher)

   236 

   237 def specjbb2005(args):

   238     """run the composite SPECjbb2005 benchmark"""

   239 

   240     def launcher(bm, harnessArgs, extraVmOpts):

   241         assert bm is None

   242         return sanitycheck.getSPECjbb2005(harnessArgs).bench(mx_graal.get_vm(), extraVmOpts=extraVmOpts)

   243 

   244     _run_benchmark(args, None, launcher)

   245 

   246 mx.update_commands(mx.suite('graal'), {

   247     'dacapo': [dacapo, '[VM options] benchmarks...|"all" [DaCapo options]'],

   248     'scaladacapo': [scaladacapo, '[VM options] benchmarks...|"all" [Scala DaCapo options]'],

   249     'specjvm2008': [specjvm2008, '[VM options] benchmarks...|"all" [SPECjvm2008 options]'],

   250     'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],

   251     'specjbb2015': [specjbb2015, '[VM options] [-- [SPECjbb2015 options]]'],

   252     'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],

   253     'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],

   254     'deoptalot' : [deoptalot, '[n]'],

   255     'longtests' : [longtests, ''],

   256 })