ab Namespace Reference

Classes
class	ThreadRunner
class	ThreadWithException

Functions
	parse_args ()
	append_dict_sorted_array (dict_array, key, value)
	add_time (args, name, bench, t, unit)
	append_times_from_file (args, name, filename)
	split_arg (arg)
	run (args, threadRunner, name, nano, arg, i)
	init_run (args)
	get_lower_upper (values)
	different_enough (lower1, upper2)
	get_suspects ()
	process_bench_pattern (s)
	suspects_arg (suspects)
	median (array)
	regression (bench)
	percentage (x)
	format_r (r)
	normalize_r (r)
	test ()

Variables
str	HELP
int	FACTOR = 3
float	DIFF_T = 0.99
int	TERM = 10
int	MAXTRY = 30
str	UNITS = "ns µs ms s".split()
	timesLock = Lock()
dict	timesA = {}
dict	timesB = {}

Function Documentation

add_time()

ab.add_time	(		args,
			name,
			bench,
			t,
			unit
	)

Definition at line 107 of file ab.py.

def add_time(args, name, bench, t, unit):
  normalized_t = t * 1000 ** UNITS.index(unit);
  if name.startswith(args.a):
    append_dict_sorted_array(timesA, bench, normalized_t)
  else:
    append_dict_sorted_array(timesB, bench, normalized_t)
 
 

append_dict_sorted_array()

ab.append_dict_sorted_array	(		dict_array,
			key,
			value
	)

Definition at line 100 of file ab.py.

def append_dict_sorted_array(dict_array, key, value):
  if key not in dict_array:
    dict_array[key] = []
  dict_array[key].append(value)
  dict_array[key].sort()
 
 

append_times_from_file()

ab.append_times_from_file	(		args,
			name,
			filename
	)

Definition at line 115 of file ab.py.

def append_times_from_file(args, name, filename):
  with open(filename) as f:
    lines = f.readlines()
  for line in lines:
    items = line.split()
    if len(items) > 10:
      bench = items[10]
      matches = re.search("([+-]?\d*.?\d+)(s|ms|µs|ns)", items[3])
      if (not matches or items[9] != args.config):
        continue
      time_num = matches.group(1)
      time_unit = matches.group(2)
      add_time(args, name, bench, float(time_num), time_unit)
 
 

different_enough()

ab.different_enough	(		lower1,
			upper2
	)

Definition at line 246 of file ab.py.

def different_enough(lower1, upper2):
  return upper2 < DIFF_T * lower1
 
 
# TODO(liyuqian): we used this hacky criteria mainly because that I didn't have
# time to study more rigorous statistical tests. We should adopt a more rigorous
# test in the future.

format_r()

ab.format_r

(

r

)

Definition at line 293 of file ab.py.

def format_r(r):
  return ('%6.2f' % percentage(r)) + "%"
 
 

get_lower_upper()

ab.get_lower_upper

(

values

)

Definition at line 241 of file ab.py.

def get_lower_upper(values):
  i = max(0, (len(values) - 1) / FACTOR)
  return values[i], values[-i - 1]
 
 

get_suspects()

ab.get_suspects

(

)

Definition at line 253 of file ab.py.

def get_suspects():
  suspects = []
  for bench in timesA.keys():
    if bench not in timesB:
      continue
    lowerA, upperA = get_lower_upper(timesA[bench])
    lowerB, upperB = get_lower_upper(timesB[bench])
    if different_enough(lowerA, upperB) or different_enough(lowerB, upperA):
      suspects.append(bench)
  return suspects
 
 

init_run()

ab.init_run

(

args

)

Definition at line 233 of file ab.py.

def init_run(args):
  threadRunner = ThreadRunner(args)
  for i in range(1, max(args.repeat, args.threads / 2) + 1):
    run(args, threadRunner, args.a, args.nano_a, args.arg_a, i)
    run(args, threadRunner, args.b, args.nano_b, args.arg_b, i)
  threadRunner.wait()
 
 

median()

ab.median

(

array

)

Definition at line 277 of file ab.py.

def median(array):
  return array[len(array) / 2]
 
 

normalize_r()

ab.normalize_r

(

r

)

Definition at line 297 of file ab.py.

def normalize_r(r):
  if r > 1.0:
    return r - 1.0
  else:
    return 1.0 - 1/r
 
 

parse_args()

ab.parse_args

(

)

Definition at line 62 of file ab.py.

def parse_args():
  parser = ArgumentParser(description=HELP)
 
  parser.add_argument('outdir', type=str, help="output directory")
  parser.add_argument('a', type=str, help="name of A")
  parser.add_argument('b', type=str, help="name of B")
  parser.add_argument('nano_a', type=str, help="path to A's nanobench binary")
  parser.add_argument('nano_b', type=str, help="path to B's nanobench binary")
  parser.add_argument('arg_a', type=str, help="args for A's nanobench run")
  parser.add_argument('arg_b', type=str, help="args for B's nanobench run")
  parser.add_argument('repeat', type=int, help="number of initial runs")
  parser.add_argument('skip_b', type=str, help=("whether to skip running B"
                                                " ('true' or 'false')"))
  parser.add_argument('config', type=str, help="nanobenh config")
  parser.add_argument('threads', type=int, help="number of threads to run")
  parser.add_argument('noinit', type=str, help=("whether to skip running B"
                                                " ('true' or 'false')"))
 
  parser.add_argument('--concise', dest='concise', action="store_true",
      help="If set, no verbose thread info will be printed.")
  parser.set_defaults(concise=False)
 
  # Additional args for bots
  BHELP = "bot specific options"
  parser.add_argument('--githash', type=str, default="", help=BHELP)
  parser.add_argument('--keys', type=str, default=[], nargs='+', help=BHELP)
 
  args = parser.parse_args()
  args.skip_b = args.skip_b == "true"
  args.noinit = args.noinit == "true"
 
  if args.threads == -1:
    args.threads = 1
    if args.config in ["8888", "565"]: # multi-thread for CPU only
        args.threads = max(1, multiprocessing.cpu_count() / 2)
 
  return args
 

percentage()

ab.percentage

(

x

)

Definition at line 289 of file ab.py.

def percentage(x):
  return (x - 1) * 100
 
 

process_bench_pattern()

ab.process_bench_pattern

(

s

)

Definition at line 265 of file ab.py.

def process_bench_pattern(s):
  if ".skp" in s: # skp bench won't match their exact names...
    return "^\"" + s[0:(s.index(".skp") + 3)] + "\""
  else:
    return "^\"" + s + "\"$"
 
 

regression()

ab.regression

(

bench

)

Definition at line 281 of file ab.py.

def regression(bench):
  a = median(timesA[bench])
  b = median(timesB[bench])
  if (a == 0): # bad bench, just return no regression
    return 1
  return b / a
 
 

run()

ab.run	(		args,
			threadRunner,
			name,
			nano,
			arg,
			i
	)

Definition at line 208 of file ab.py.

def run(args, threadRunner, name, nano, arg, i):
  def task():
    file_i = "%s/%s.out%d" % (args.outdir, name, i)
 
    should_run = not args.noinit and not (name == args.b and args.skip_b)
    if i <= 0:
      should_run = True # always run for suspects
 
    if should_run:
      if i > 0:
        timesLock.acquire()
        print("Init run %d for %s..." % (i, name))
        timesLock.release()
      subprocess.check_call(["touch", file_i])
      with open(file_i, 'w') as f:
        subprocess.check_call([nano] + split_arg(arg) +
                              ["--config", args.config], stderr=f, stdout=f)
 
    timesLock.acquire()
    append_times_from_file(args, name, file_i)
    timesLock.release()
 
  threadRunner.add(args, task)
 
 

split_arg()

ab.split_arg

(

arg

)

Definition at line 197 of file ab.py.

def split_arg(arg):
  raw = shlex.split(arg)
  result = []
  for r in raw:
    if '~' in r:
      result.append(os.path.expanduser(r))
    else:
      result.append(r)
  return result
 
 

suspects_arg()

ab.suspects_arg

(

suspects

)

Definition at line 272 of file ab.py.

def suspects_arg(suspects):
  patterns = map(process_bench_pattern, suspects)
  return " --match " + (" ".join(patterns))
 
 

test()

ab.test

(

)

Definition at line 304 of file ab.py.

def test():
  args = parse_args()
 
  init_run(args)
  last_unchanged_iter = 0
  last_suspect_number = -1
  tryCnt = 0
  it = 0
  while tryCnt < MAXTRY:
    it += 1
    suspects = get_suspects()
    if len(suspects) != last_suspect_number:
      last_suspect_number = len(suspects)
      last_unchanged_iter = it
    if (len(suspects) == 0 or it - last_unchanged_iter >= TERM):
      break
 
    print("Number of suspects at iteration %d: %d" % (it, len(suspects)))
    threadRunner = ThreadRunner(args)
    for j in range(1, max(1, args.threads / 2) + 1):
      run(args, threadRunner, args.a, args.nano_a,
          args.arg_a + suspects_arg(suspects), -j)
      run(args, threadRunner, args.b, args.nano_b,
          args.arg_b + suspects_arg(suspects), -j)
      tryCnt += 1
    threadRunner.wait()
 
  suspects = get_suspects()
  if len(suspects) == 0:
    print(("%s and %s does not seem to have significant " + \
           "performance differences.") % (args.a, args.b))
  else:
    suspects.sort(key = regression)
    print("%s (compared to %s) is likely" % (args.a, args.b))
    for suspect in suspects:
      r = regression(suspect)
      if r < 1:
        print("\033[31m  %s slower in %s\033[0m" % (format_r(1/r), suspect))
      else:
        print("\033[32m  %s faster in %s\033[0m" % (format_r(r), suspect))
 
  with open("%s/bench_%s_%s.json" % (args.outdir, args.a, args.b), 'w') as f:
    results = {}
    for bench in timesA:
      r = regression(bench) if bench in suspects else 1.0
      results[bench] = {
        args.config: {
          "signed_regression": normalize_r(r),
          "lower_quantile_ms": get_lower_upper(timesA[bench])[0] * 1e-6,
          "upper_quantile_ms": get_lower_upper(timesA[bench])[1] * 1e-6,
          "options": {
            # TODO(liyuqian): let ab.py call nanobench with --outResultsFile so
            # nanobench could generate the json for us that's exactly the same
            # as that being used by perf bots. Currently, we cannot guarantee
            # that bench is the name (e.g., bench may have additional resolution
            # information appended after name).
            "name": bench
          }
        }
      }
 
    output = {"results": results}
    if args.githash:
      output["gitHash"] = args.githash
    if args.keys:
      keys = {}
      for i in range(len(args.keys) / 2):
        keys[args.keys[i * 2]] = args.keys[i * 2 + 1]
      output["key"] = keys
    f.write(json.dumps(output, indent=4))
    print(("\033[36mJSON results available in %s\033[0m" % f.name))
 
  with open("%s/bench_%s_%s.csv" % (args.outdir, args.a, args.b), 'w') as out:
    out.write(("bench, significant?, raw regresion, " +
                   "%(A)s quantile (ns), %(B)s quantile (ns), " +
                   "%(A)s (ns), %(B)s (ns)\n") % {'A': args.a, 'B': args.b})
    for bench in suspects + timesA.keys():
      if (bench not in timesA or bench not in timesB):
        continue
      ta = timesA[bench]
      tb = timesB[bench]
      out.write(
          "%s, %s, %f, " % (bench, bench in suspects, regression(bench)) +
          ' '.join(map(str, get_lower_upper(ta))) + ", " +
          ' '.join(map(str, get_lower_upper(tb))) + ", " +
          ("%s, %s\n" % (' '.join(map(str, ta)), ' '.join(map(str, tb))))
      )
    print(("\033[36m" +
           "Compared %d benches. " +
           "%d of them seem to be significantly differrent." +
           "\033[0m") %
           (len([x for x in timesA if x in timesB]), len(suspects)))
    print("\033[36mPlease see detailed bench results in %s\033[0m" % out.name)
 
 

Variable Documentation

DIFF_T

float ab.DIFF_T = 0.99

Definition at line 50 of file ab.py.

FACTOR

int ab.FACTOR = 3

Definition at line 49 of file ab.py.

HELP

str ab.HELP

Initial value:

=  """
\033[31mPlease call calmbench.py to drive this script if you're not doing so.
This script is not supposed to be used by itself. (At least, it's not easy to
use by itself. The calmbench bots may use this script directly.)
\033[0m
"""

Definition at line 42 of file ab.py.

MAXTRY

int ab.MAXTRY = 30

Definition at line 52 of file ab.py.

TERM

int ab.TERM = 10

Definition at line 51 of file ab.py.

timesA

dict ab.timesA = {}

Definition at line 58 of file ab.py.

timesB

dict ab.timesB = {}

Definition at line 59 of file ab.py.

timesLock

ab.timesLock = Lock()

Definition at line 57 of file ab.py.

UNITS

str ab.UNITS = "ns µs ms s".split()

Definition at line 54 of file ab.py.