• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3
2""" NNAPI benchmark output parser (statistics aggregator)
3
4Reads a file with output from multiple runs of
5  adb shell am instrument
6    -w com.android.nn.benchmark.app/androidx.test.runner.AndroidJUnitRunner
7
8and provides either raw measurements or aggregated statistics of the runs.
9
10Usage:
11  parse_benchmark --format=[json|table] --output=[full|stats] [adb output filename]
12
13"""
14import argparse
15import json
16import statistics
17
18FLAG_FORMAT_JSON = "json"
19FLAG_FORMAT_TABLE = "table"
20
21FLAG_OUTPUT_FULL = "full"
22FLAG_OUTPUT_STATS = "stats"
23
24
25def main():
26  parser = argparse.ArgumentParser()
27  parser.add_argument("input", help="input filename")
28  parser.add_argument("--format", help="output format ({0}|{1})"
29                      .format(FLAG_FORMAT_TABLE, FLAG_FORMAT_JSON),
30                      default=FLAG_FORMAT_TABLE)
31  parser.add_argument("--output", help="output data ({0}|{1})"
32                      .format(FLAG_OUTPUT_FULL, FLAG_OUTPUT_STATS),
33                      default=FLAG_OUTPUT_STATS)
34  args = parser.parse_args()
35
36  data = read_data(args.input)
37
38  if args.output == FLAG_OUTPUT_STATS:
39    stats = compute_stats(data)
40    print_stats(stats, args.format)
41  else:
42    print_data(data, args.format)
43
44
45def read_data(input_filename):
46  data = dict()
47
48  with open(input_filename) as f:
49    for line in f:
50      if "INSTRUMENTATION_STATUS:" in line and "_avg" in line:
51        sample = line.split(": ")[1]
52        name, value = sample.split("=")
53        name = name[:-4]
54        data[name] = data.get(name, []) + [float(value)]
55
56  return data
57
58
59def compute_stats(data):
60  stats = list()
61
62  for name in sorted(data):
63    values = data[name]
64    stat_mean = statistics.mean(values)
65    stat_stdev = statistics.stdev(values)
66    stat_min = min(values)
67    stat_max = max(values)
68    stat_n = len(values)
69    stats.append({"benchmark": name, "mean": stat_mean, "stddev": stat_stdev,
70                  "min": stat_min, "max": stat_max, "n": stat_n})
71
72  return stats
73
74
75def print_stats(stats, print_format):
76  if print_format == FLAG_FORMAT_TABLE:
77    print("{0:<34}{1:>10}{2:>10}{3:>10}{4:>10}{5:>10}".format(
78        "Benchmark", "mean", "stddev", "min", "max", "n"))
79    for line in stats:
80      print("{0:<34}{1:>10.2f}{2:>10.2f}{3:>10.2f}{4:>10.2f}{5:>10d}".format(
81          line["benchmark"], line["mean"], line["stddev"], line["min"],
82          line["max"], line["n"]))
83  else:
84    print(json.dumps(stats))
85
86
87def print_data(data, print_format):
88  if print_format == FLAG_FORMAT_TABLE:
89    print("{0:<34}{1:>10}".format(
90        "Benchmark", "sample"))
91
92  for name in data:
93    for sample in data[name]:
94      if print_format == FLAG_FORMAT_TABLE:
95        print("{0:<34}{1:>10}".format(name, sample))
96      else:
97        print(json.dumps({"benchmark": name, "sample": sample}))
98
99if __name__ == "__main__":
100  main()
101