24 /** |
24 /** |
25 * @subtest |
25 * @subtest |
26 */ |
26 */ |
27 |
27 |
28 var tests = [ |
28 var tests = [ |
29 "box2d.js", |
29 {file:"box2d",suite:"Box2DBenchmark"}, |
30 "code-load.js", |
30 {file:"code-load",suite:"CodeLoad"}, |
31 "crypto.js", |
31 {file:"crypto",suite:"Crypto"}, |
32 "deltablue.js", |
32 {file:"deltablue",suite:"DeltaBlue"}, |
33 "earley-boyer.js", |
33 {file:"earley-boyer", suite:"EarleyBoyer"}, |
34 "gbemu.js", |
34 {file:"gbemu", suite:"GameboyBenchmark"}, |
35 "mandreel.js", |
35 {file:"mandreel", suite:"MandreelBenchmark"}, |
36 "navier-stokes.js", |
36 {file:"navier-stokes", suite:"NavierStokes"}, |
37 "pdfjs.js", |
37 {file:"pdfjs", suite:"PdfJS"}, |
38 "raytrace.js", |
38 {file:"raytrace", suite:"RayTrace"}, |
39 "regexp.js", |
39 {file:"regexp", suite:"RegExpSuite"}, |
40 "richards.js", |
40 {file:"richards", suite:"Richards"}, |
41 "splay.js" |
41 {file:"splay", suite:"Splay"} |
42 ]; |
42 ]; |
43 |
|
44 // hack, teardown breaks things defined in the global space, making it impossible |
|
45 // to do multiple consecutive benchmark runs with the same harness. I think it's a bug |
|
46 // that the setup and teardown aren't each others constructor and destructor but rather |
|
47 // that the benchmarks rely on partial global state. For shame, Octane! |
|
48 var ignoreTeardown = [ |
|
49 { name: "box2d.js" }, |
|
50 { name: "gbemu.js" }, |
|
51 ]; |
|
52 |
|
53 |
|
54 //TODO mandreel can be compiled as a test, but not run multiple times unless modified to not have global state |
|
55 var compileOnly = { |
|
56 "mandreel.js" : true |
|
57 }; |
|
58 |
|
59 var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__; |
43 var dir = (typeof(__DIR__) == 'undefined') ? "test/script/basic/" : __DIR__; |
60 |
44 |
61 // TODO: why is this path hard coded when it's defined in project properties? |
45 // TODO: why is this path hard coded when it's defined in project properties? |
62 var path = dir + "../external/octane/"; |
46 var path = dir + "../external/octane/"; |
63 |
47 |
69 function endsWith(str, suffix) { |
53 function endsWith(str, suffix) { |
70 return str.indexOf(suffix, str.length - suffix.length) !== -1; |
54 return str.indexOf(suffix, str.length - suffix.length) !== -1; |
71 } |
55 } |
72 |
56 |
73 function should_compile_only(name) { |
57 function should_compile_only(name) { |
74 return (typeof compile_only !== 'undefined') || compileOnly[name] === true; |
58 return (typeof compile_only !== 'undefined') |
75 } |
59 } |
76 |
60 |
77 function run_one_benchmark(arg, iters) { |
61 function run_one_benchmark(arg, iters) { |
78 |
|
79 var file_name; |
62 var file_name; |
80 var file = arg.split('/'); |
63 var file = (arg.file + ".js").split('/'); |
81 if (file.length == 1) { |
64 |
82 file = arg.split('\\'); |
65 file_name = path + file[file.length - 1]; |
83 } |
66 |
84 |
|
85 //trim off trailing path separators |
|
86 while (file[file.length - 1].indexOf(".js") == -1) { |
|
87 file.pop(); |
|
88 } |
|
89 file_name = file[file.length - 1]; |
|
90 |
|
91 var compile_and_return = should_compile_only(file_name); |
67 var compile_and_return = should_compile_only(file_name); |
92 if (compile_and_return) { |
68 if (compile_and_return) { |
93 if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them |
69 if (typeof compile_only === 'undefined') { //for a run, skip compile onlies, don't even compile them |
94 return; |
70 return; |
95 } |
71 } |
96 print("Compiling... " + file_name); |
72 } |
97 } |
73 |
98 |
74 print_verbose("Loading... " + file_name); |
99 load(path + 'base.js'); |
75 load(file_name); |
100 load(arg); |
|
101 |
76 |
102 if (compile_and_return) { |
77 if (compile_and_return) { |
103 print("Compiled OK: " + file_name); |
78 print_always("Compiled OK: " + arg.file); |
104 print(""); |
|
105 return; |
79 return; |
106 } |
80 } |
107 |
81 |
108 var success = true; |
82 var success = true; |
109 var hiscore = 0; |
|
110 var loscore = 10e8; |
|
111 var current_name; |
83 var current_name; |
112 |
|
113 function PrintResult(name, result) { |
|
114 current_name = name; |
|
115 } |
|
116 |
|
117 function PrintError(name, error) { |
|
118 current_name = name; |
|
119 PrintResult(name, error); |
|
120 success = false; |
|
121 } |
|
122 |
|
123 function PrintScore(score) { |
|
124 if (success) { |
|
125 if (+score >= hiscore) { |
|
126 hiscore = +score; |
|
127 } |
|
128 if (+score <= loscore) { |
|
129 loscore = +score; |
|
130 } |
|
131 } |
|
132 |
|
133 if (verbose) { |
|
134 print("Score: " + score); |
|
135 } |
|
136 } |
|
137 |
84 |
138 if (iters == undefined) { |
85 if (iters == undefined) { |
139 iters = numberOfIterations; |
86 iters = numberOfIterations; |
140 } else { |
87 } else { |
141 numberOfIterations = iters; |
88 numberOfIterations = iters; |
142 } |
89 } |
143 |
90 |
144 print(runtime + ": running " + file_name + "..."); |
91 var benchmarks = eval(arg.suite + ".benchmarks"); |
145 |
92 for (var x = 0; x < benchmarks.length ; x++) { |
146 for (var i = 0; i < numberOfIterations; i++) { |
93 benchmarks[x].Setup(); |
147 var callbacks = |
94 } |
148 { NotifyResult: PrintResult, |
95 print_verbose("Running '" + arg.file + "' for " + iters + " iterations of no less than " + min_time + " seconds (" + runtime + ")"); |
149 NotifyError: PrintError, |
96 |
150 NotifyScore: PrintScore }; |
97 var scores = []; |
151 |
98 |
152 for (j in ignoreTeardown) { |
99 var min_time_ms = min_time * 1000; |
153 var ignore = ignoreTeardown[j]; |
100 var len = benchmarks.length; |
154 if (endsWith(arg, ignore.name)) { |
101 |
155 var teardownOverride = ignore.teardown; |
102 for (var it = 0; it < iters + 1; it++) { |
156 if (!teardownOverride) { |
103 //every iteration must take a minimum of 10 secs |
157 teardownOverride = function() {}; |
104 var ops = 0; |
158 } |
105 var elapsed = 0; |
159 |
106 var start = new Date; |
160 for (k in BenchmarkSuite.suites) { |
107 do { |
161 var benchmarks = BenchmarkSuite.suites[k].benchmarks; |
108 for (var i = 0; i < len; i++) { |
162 for (l in benchmarks) { |
109 benchmarks[i].run(); |
163 benchmarks[l].TearDown = teardownOverride; |
110 } |
164 } |
111 ops += len; |
165 } |
112 elapsed = new Date - start; |
166 break; |
113 } while (elapsed < min_time * 1000); |
167 } |
|
168 } |
|
169 |
114 |
170 BenchmarkSuite.RunSuites(callbacks); |
115 var score = ops / elapsed * 1000 * 60; |
171 } |
116 scores.push(score); |
172 |
117 var name = it == 0 ? "warmup" : "iteration " + it; |
173 var start = "Score: "; |
118 print_verbose("[" + arg.file + "] " + name + " finished " + score.toFixed(0) + " ops/minute"); |
174 if (runtime != "") { |
119 } |
175 start = runtime + ": "; |
120 for (var x = 0; x < benchmarks.length ; x++) { |
176 } |
121 benchmarks[x].TearDown(); |
177 print(start + current_name + ' (version ' + BenchmarkSuite.version + '): ' + loscore + '-' + hiscore); |
122 } |
|
123 |
|
124 var min_score = 1e9; |
|
125 var max_score = 0; |
|
126 var mean_score = 0; |
|
127 for (var x = 1; x < iters + 1 ; x++) { |
|
128 mean_score += scores[x]; |
|
129 min_score = Math.min(min_score, scores[x]); |
|
130 max_score = Math.max(max_score, scores[x]); |
|
131 } |
|
132 mean_score /= iters; |
|
133 var res = "[" + arg.file + "] " + mean_score.toFixed(0); |
|
134 if (verbose) { |
|
135 res += " ops/minute (" + min_score.toFixed(0) + "-" + max_score.toFixed(0) + "), warmup=" + scores[0].toFixed(0); |
|
136 } |
|
137 print_always(res); |
|
138 } |
|
139 |
|
140 function print_always(x) { |
|
141 print(x); |
|
142 } |
|
143 |
|
144 function print_verbose(x) { |
|
145 if (verbose) { |
|
146 print(x); |
|
147 } |
178 } |
148 } |
179 |
149 |
180 function run_suite(tests, iters) { |
150 function run_suite(tests, iters) { |
181 for (var idx = 0; idx < tests.length; idx++) { |
151 for (var idx = 0; idx < tests.length; idx++) { |
182 run_one_benchmark(tests[idx], iters); |
152 run_one_benchmark(tests[idx], iters); |
209 args = new_args; |
179 args = new_args; |
210 } |
180 } |
211 |
181 |
212 var tests_found = []; |
182 var tests_found = []; |
213 var iters = undefined; |
183 var iters = undefined; |
|
184 var min_time = 5; |
214 |
185 |
215 for (var i = 0; i < args.length; i++) { |
186 for (var i = 0; i < args.length; i++) { |
216 arg = args[i]; |
187 arg = args[i]; |
217 if (arg == "--iterations") { |
188 if (arg == "--iterations") { |
218 iters = +args[++i]; |
189 iters = +args[++i]; |
219 } else if (arg == "--runtime") { |
190 } else if (arg == "--runtime") { |
220 runtime = args[++i]; |
191 runtime = args[++i]; |
221 } else if (arg == "--verbose") { |
192 } else if (arg == "--verbose") { |
222 verbose = true; |
193 verbose = true; |
|
194 } else if (arg == "--min-time") { |
|
195 min_time = +args[++i]; |
223 } else if (arg == "") { |
196 } else if (arg == "") { |
224 continue; //skip |
197 continue; //skip |
225 } else { |
198 } else { |
226 tests_found.push(arg); |
199 var found = false; |
|
200 for (j in tests) { |
|
201 if (tests[j].file === arg) { |
|
202 tests_found.push(tests[j]); |
|
203 found = true; |
|
204 break; |
|
205 } |
|
206 } |
|
207 if (!found) { |
|
208 var str = "unknown test name: '" + arg + "' -- valid names are: "; |
|
209 for (j in tests) { |
|
210 if (j != 0) { |
|
211 str += ", "; |
|
212 } |
|
213 str += "'" + tests[j].file + "'"; |
|
214 } |
|
215 throw str; |
|
216 } |
227 } |
217 } |
228 } |
218 } |
229 |
219 |
230 if (tests_found.length == 0) { |
220 if (tests_found.length == 0) { |
231 for (i in tests) { |
221 for (i in tests) { |
232 tests_found.push(path + tests[i]); |
222 tests_found.push(tests[i]); |
233 } |
223 } |
234 } |
224 } |
235 |
225 |
236 tests_found.sort(); |
226 tests_found.sort(); |
237 |
227 |
|
228 load(path + 'base.js'); |
238 run_suite(tests_found, iters); |
229 run_suite(tests_found, iters); |
239 |
230 |
240 |
231 |
241 |
232 |