63 parser = ArgumentParser(description=HELP)
65 parser.add_argument(
'outdir', type=str, help=
"output directory")
66 parser.add_argument(
'a', type=str, help=
"name of A")
67 parser.add_argument(
'b', type=str, help=
"name of B")
68 parser.add_argument(
'nano_a', type=str, help=
"path to A's nanobench binary")
69 parser.add_argument(
'nano_b', type=str, help=
"path to B's nanobench binary")
70 parser.add_argument(
'arg_a', type=str, help=
"args for A's nanobench run")
71 parser.add_argument(
'arg_b', type=str, help=
"args for B's nanobench run")
72 parser.add_argument(
'repeat', type=int, help=
"number of initial runs")
73 parser.add_argument(
'skip_b', type=str, help=(
"whether to skip running B"
74 " ('true' or 'false')"))
75 parser.add_argument(
'config', type=str, help=
"nanobenh config")
76 parser.add_argument(
'threads', type=int, help=
"number of threads to run")
77 parser.add_argument(
'noinit', type=str, help=(
"whether to skip running B"
78 " ('true' or 'false')"))
80 parser.add_argument(
'--concise', dest=
'concise', action=
"store_true",
81 help=
"If set, no verbose thread info will be printed.")
82 parser.set_defaults(concise=
False)
85 BHELP =
"bot specific options"
86 parser.add_argument(
'--githash', type=str, default=
"", help=BHELP)
87 parser.add_argument(
'--keys', type=str, default=[], nargs=
'+', help=BHELP)
89 args = parser.parse_args()
90 args.skip_b = args.skip_b ==
"true"
91 args.noinit = args.noinit ==
"true"
93 if args.threads == -1:
95 if args.config
in [
"8888",
"565"]:
96 args.threads =
max(1, multiprocessing.cpu_count() / 2)
108 normalized_t = t * 1000 ** UNITS.index(unit);
109 if name.startswith(args.a):
116 with open(filename)
as f:
117 lines = f.readlines()
122 matches = re.search(
"([+-]?\d*.?\d+)(s|ms|µs|ns)", items[3])
123 if (
not matches
or items[9] != args.config):
125 time_num = matches.group(1)
126 time_unit = matches.group(2)
127 add_time(args, name, bench, float(time_num), time_unit)
208def run(args, threadRunner, name, nano, arg, i):
210 file_i =
"%s/%s.out%d" % (args.outdir, name, i)
212 should_run =
not args.noinit
and not (name == args.b
and args.skip_b)
219 print(
"Init run %d for %s..." % (i, name))
221 subprocess.check_call([
"touch", file_i])
222 with open(file_i,
'w')
as f:
223 subprocess.check_call([nano] +
split_arg(arg) +
224 [
"--config", args.config], stderr=f, stdout=f)
230 threadRunner.add(args, task)
235 for i
in range(1,
max(args.repeat, args.threads / 2) + 1):
236 run(args, threadRunner, args.a, args.nano_a, args.arg_a, i)
237 run(args, threadRunner, args.b, args.nano_b, args.arg_b, i)
308 last_unchanged_iter = 0
309 last_suspect_number = -1
312 while tryCnt < MAXTRY:
315 if len(suspects) != last_suspect_number:
316 last_suspect_number = len(suspects)
317 last_unchanged_iter = it
318 if (len(suspects) == 0
or it - last_unchanged_iter >= TERM):
321 print(
"Number of suspects at iteration %d: %d" % (it, len(suspects)))
323 for j
in range(1,
max(1, args.threads / 2) + 1):
324 run(args, threadRunner, args.a, args.nano_a,
326 run(args, threadRunner, args.b, args.nano_b,
332 if len(suspects) == 0:
333 print((
"%s and %s does not seem to have significant " + \
334 "performance differences.") % (args.a, args.b))
336 suspects.sort(key = regression)
337 print(
"%s (compared to %s) is likely" % (args.a, args.b))
338 for suspect
in suspects:
341 print(
"\033[31m %s slower in %s\033[0m" % (
format_r(1/r), suspect))
343 print(
"\033[32m %s faster in %s\033[0m" % (
format_r(r), suspect))
345 with open(
"%s/bench_%s_%s.json" % (args.outdir, args.a, args.b),
'w')
as f:
348 r =
regression(bench)
if bench
in suspects
else 1.0
365 output = {
"results": results}
367 output[
"gitHash"] = args.githash
370 for i
in range(len(args.keys) / 2):
371 keys[args.keys[i * 2]] = args.keys[i * 2 + 1]
373 f.write(json.dumps(output, indent=4))
374 print((
"\033[36mJSON results available in %s\033[0m" % f.name))
376 with open(
"%s/bench_%s_%s.csv" % (args.outdir, args.a, args.b),
'w')
as out:
377 out.write((
"bench, significant?, raw regresion, " +
378 "%(A)s quantile (ns), %(B)s quantile (ns), " +
379 "%(A)s (ns), %(B)s (ns)\n") % {
'A': args.a,
'B': args.b})
380 for bench
in suspects + timesA.keys():
381 if (bench
not in timesA
or bench
not in timesB):
386 "%s, %s, %f, " % (bench, bench
in suspects,
regression(bench)) +
389 (
"%s, %s\n" % (
' '.join(map(str, ta)),
' '.join(map(str, tb))))
392 "Compared %d benches. " +
393 "%d of them seem to be significantly differrent." +
395 (len([x
for x
in timesA
if x
in timesB]), len(suspects)))
396 print(
"\033[36mPlease see detailed bench results in %s\033[0m" % out.name)