# HG changeset patch # User lagergren # Date 1369653073 -7200 # Node ID 0407501fa563e5c218f86f8af3dacc4bd4fb7752 # Parent 9acc52342786ed853f7f4a0779c278ba94d179b5 8014219: Make the run-octane harness more deterministic by not measuring elapsed time every iteration. Also got rid of most of the run logic in base.js and call benchmarks directly for the same purpose Reviewed-by: jlaskey, attila diff -r 9acc52342786 -r 0407501fa563 nashorn/make/build-benchmark.xml --- a/nashorn/make/build-benchmark.xml Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/make/build-benchmark.xml Mon May 27 13:11:13 2013 +0200 @@ -32,250 +32,263 @@ + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + + - + - + - + diff -r 9acc52342786 -r 0407501fa563 nashorn/src/jdk/nashorn/internal/runtime/AccessorProperty.java --- a/nashorn/src/jdk/nashorn/internal/runtime/AccessorProperty.java Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/src/jdk/nashorn/internal/runtime/AccessorProperty.java Mon May 27 13:11:13 2013 +0200 @@ -306,14 +306,14 @@ protected Object getObjectValue(final ScriptObject self, final ScriptObject owner) { if (isSpill()) { return self.spill[getSlot()]; - } else { - try { - return getGetter(Object.class).invokeExact((Object)self); - } catch (final Error|RuntimeException e) { - throw e; - } catch (final Throwable e) { - throw new RuntimeException(e); - } + } + + try { + return getGetter(Object.class).invokeExact((Object)self); + } catch (final Error|RuntimeException e) { + throw e; + } catch (final Throwable e) { + throw new RuntimeException(e); } } diff -r 9acc52342786 -r 0407501fa563 nashorn/src/jdk/nashorn/internal/runtime/Property.java --- a/nashorn/src/jdk/nashorn/internal/runtime/Property.java Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/src/jdk/nashorn/internal/runtime/Property.java Mon May 27 13:11:13 2013 +0200 @@ -52,6 +52,7 @@ * we can use leave flag byte initialized with (the default) zero value. */ + /** Mask for property being both writable, enumerable and configurable */ public static final int WRITABLE_ENUMERABLE_CONFIGURABLE = 0b0000_0000_0000; /** ECMA 8.6.1 - Is this property not writable? */ @@ -360,6 +361,7 @@ * @param self the this object * @param owner the owner object * @param value the new property value + * @param strict is this a strict setter? */ protected abstract void setObjectValue(ScriptObject self, ScriptObject owner, Object value, boolean strict); diff -r 9acc52342786 -r 0407501fa563 nashorn/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java --- a/nashorn/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/src/jdk/nashorn/internal/runtime/UserAccessorProperty.java Mon May 27 13:11:13 2013 +0200 @@ -193,7 +193,6 @@ // getter/setter may be inherited. If so, proto is bound during lookup. In either // inherited or self case, slot is also bound during lookup. Actual ScriptFunction // to be called is retrieved everytime and applied. - @SuppressWarnings("unused") static Object userAccessorGetter(final ScriptObject proto, final int slot, final Object self) { final ScriptObject container = (proto != null) ? proto : (ScriptObject)self; final Object func = container.getSpill(slot); @@ -211,7 +210,6 @@ return UNDEFINED; } - @SuppressWarnings("unused") static void userAccessorSetter(final ScriptObject proto, final int slot, final String name, final Object self, final Object value) { final ScriptObject container = (proto != null) ? proto : (ScriptObject)self; final Object func = container.getSpill(slot); diff -r 9acc52342786 -r 0407501fa563 nashorn/test/script/basic/compile-octane.js.EXPECTED --- a/nashorn/test/script/basic/compile-octane.js.EXPECTED Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/test/script/basic/compile-octane.js.EXPECTED Mon May 27 13:11:13 2013 +0200 @@ -1,39 +1,13 @@ -Compiling... box2d.js -Compiled OK: box2d.js - -Compiling... code-load.js -Compiled OK: code-load.js - -Compiling... crypto.js -Compiled OK: crypto.js - -Compiling... deltablue.js -Compiled OK: deltablue.js - -Compiling... earley-boyer.js -Compiled OK: earley-boyer.js - -Compiling... gbemu.js -Compiled OK: gbemu.js - -Compiling... mandreel.js -Compiled OK: mandreel.js - -Compiling... navier-stokes.js -Compiled OK: navier-stokes.js - -Compiling... pdfjs.js -Compiled OK: pdfjs.js - -Compiling... raytrace.js -Compiled OK: raytrace.js - -Compiling... regexp.js -Compiled OK: regexp.js - -Compiling... richards.js -Compiled OK: richards.js - -Compiling... splay.js -Compiled OK: splay.js - +Compiled OK: box2d +Compiled OK: code-load +Compiled OK: crypto +Compiled OK: deltablue +Compiled OK: earley-boyer +Compiled OK: gbemu +Compiled OK: mandreel +Compiled OK: navier-stokes +Compiled OK: pdfjs +Compiled OK: raytrace +Compiled OK: regexp +Compiled OK: richards +Compiled OK: splay diff -r 9acc52342786 -r 0407501fa563 nashorn/test/script/basic/run-octane.js --- a/nashorn/test/script/basic/run-octane.js Fri May 24 23:27:52 2013 +0530 +++ b/nashorn/test/script/basic/run-octane.js Mon May 27 13:11:13 2013 +0200 @@ -26,36 +26,20 @@ */ var tests = [ - "box2d.js", - "code-load.js", - "crypto.js", - "deltablue.js", - "earley-boyer.js", - "gbemu.js", - "mandreel.js", - "navier-stokes.js", - "pdfjs.js", - "raytrace.js", - "regexp.js", - "richards.js", - "splay.js" + {file:"box2d",suite:"Box2DBenchmark"}, + {file:"code-load",suite:"CodeLoad"}, + {file:"crypto",suite:"Crypto"}, + {file:"deltablue",suite:"DeltaBlue"}, + {file:"earley-boyer", suite:"EarleyBoyer"}, + {file:"gbemu", suite:"GameboyBenchmark"}, + {file:"mandreel", suite:"MandreelBenchmark"}, + {file:"navier-stokes", suite:"NavierStokes"}, + {file:"pdfjs", suite:"PdfJS"}, + {file:"raytrace", suite:"RayTrace"}, + {file:"regexp", suite:"RegExpSuite"}, + {file:"richards", suite:"Richards"}, + {file:"splay", suite:"Splay"} ]; - -// hack, teardown breaks things defined in the global space, making it impossible -// to do multiple consecutive benchmark runs with the same harness. I think it's a bug -// that the setup and teardown aren't each others constructor and destructor but rather -// that the benchmarks rely on partial global state. For shame, Octane! -var ignoreTeardown = [ - { name: "box2d.js" }, - { name: "gbemu.js" }, -]; - - -//TODO mandreel can be compiled as a test, but not run multiple times unless modified to not have global state -var compileOnly = { - "mandreel.js" : true -}; - var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__; // TODO: why is this path hard coded when it's defined in project properties? @@ -71,110 +55,96 @@ } function should_compile_only(name) { - return (typeof compile_only !== 'undefined') || compileOnly[name] === true; + return (typeof compile_only !== 'undefined') } function run_one_benchmark(arg, iters) { - var file_name; - var file = arg.split('/'); - if (file.length == 1) { - file = arg.split('\\'); - } - - //trim off trailing path separators - while (file[file.length - 1].indexOf(".js") == -1) { - file.pop(); - } - file_name = file[file.length - 1]; - + var file = (arg.file + ".js").split('/'); + + file_name = path + file[file.length - 1]; + var compile_and_return = should_compile_only(file_name); if (compile_and_return) { if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them return; } - print("Compiling... " + file_name); } - - load(path + 'base.js'); - load(arg); + + print_verbose("Loading... " + file_name); + load(file_name); if (compile_and_return) { - print("Compiled OK: " + file_name); - print(""); + print_always("Compiled OK: " + arg.file); return; } var success = true; - var hiscore = 0; - var loscore = 10e8; var current_name; - function PrintResult(name, result) { - current_name = name; - } - - function PrintError(name, error) { - current_name = name; - PrintResult(name, error); - success = false; - } - - function PrintScore(score) { - if (success) { - if (+score >= hiscore) { - hiscore = +score; - } - if (+score <= loscore) { - loscore = +score; - } - } - - if (verbose) { - print("Score: " + score); - } - } - if (iters == undefined) { iters = numberOfIterations; } else { numberOfIterations = iters; } - - print(runtime + ": running " + file_name + "..."); - - for (var i = 0; i < numberOfIterations; i++) { - var callbacks = - { NotifyResult: PrintResult, - NotifyError: PrintError, - NotifyScore: PrintScore }; - - for (j in ignoreTeardown) { - var ignore = ignoreTeardown[j]; - if (endsWith(arg, ignore.name)) { - var teardownOverride = ignore.teardown; - if (!teardownOverride) { - teardownOverride = function() {}; - } + + var benchmarks = eval(arg.suite + ".benchmarks"); + for (var x = 0; x < benchmarks.length ; x++) { + benchmarks[x].Setup(); + } + print_verbose("Running '" + arg.file + "' for " + iters + " iterations of no less than " + min_time + " seconds (" + runtime + ")"); + + var scores = []; + + var min_time_ms = min_time * 1000; + var len = benchmarks.length; + + for (var it = 0; it < iters + 1; it++) { + //every iteration must take a minimum of 10 secs + var ops = 0; + var elapsed = 0; + var start = new Date; + do { + for (var i = 0; i < len; i++) { + benchmarks[i].run(); + } + ops += len; + elapsed = new Date - start; + } while (elapsed < min_time * 1000); + + var score = ops / elapsed * 1000 * 60; + scores.push(score); + var name = it == 0 ? "warmup" : "iteration " + it; + print_verbose("[" + arg.file + "] " + name + " finished " + score.toFixed(0) + " ops/minute"); + } + for (var x = 0; x < benchmarks.length ; x++) { + benchmarks[x].TearDown(); + } - for (k in BenchmarkSuite.suites) { - var benchmarks = BenchmarkSuite.suites[k].benchmarks; - for (l in benchmarks) { - benchmarks[l].TearDown = teardownOverride; - } - } - break; - } - } - - BenchmarkSuite.RunSuites(callbacks); + var min_score = 1e9; + var max_score = 0; + var mean_score = 0; + for (var x = 1; x < iters + 1 ; x++) { + mean_score += scores[x]; + min_score = Math.min(min_score, scores[x]); + max_score = Math.max(max_score, scores[x]); } - - var start = "Score: "; - if (runtime != "") { - start = runtime + ": "; - } - print(start + current_name + ' (version ' + BenchmarkSuite.version + '): ' + loscore + '-' + hiscore); + mean_score /= iters; + var res = "[" + arg.file + "] " + mean_score.toFixed(0); + if (verbose) { + res += " ops/minute (" + min_score.toFixed(0) + "-" + max_score.toFixed(0) + "), warmup=" + scores[0].toFixed(0); + } + print_always(res); +} + +function print_always(x) { + print(x); +} + +function print_verbose(x) { + if (verbose) { + print(x); + } } function run_suite(tests, iters) { @@ -211,6 +181,7 @@ var tests_found = []; var iters = undefined; +var min_time = 5; for (var i = 0; i < args.length; i++) { arg = args[i]; @@ -220,21 +191,41 @@ runtime = args[++i]; } else if (arg == "--verbose") { verbose = true; + } else if (arg == "--min-time") { + min_time = +args[++i]; } else if (arg == "") { continue; //skip } else { - tests_found.push(arg); + var found = false; + for (j in tests) { + if (tests[j].file === arg) { + tests_found.push(tests[j]); + found = true; + break; + } + } + if (!found) { + var str = "unknown test name: '" + arg + "' -- valid names are: "; + for (j in tests) { + if (j != 0) { + str += ", "; + } + str += "'" + tests[j].file + "'"; + } + throw str; + } } } if (tests_found.length == 0) { for (i in tests) { - tests_found.push(path + tests[i]); + tests_found.push(tests[i]); } } tests_found.sort(); +load(path + 'base.js'); run_suite(tests_found, iters);