• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# This import depends on the automake rule protoc_middleman, please make sure
2# protoc_middleman has been built before run this file.
3import json
4import re
5import os.path
6# BEGIN OPENSOURCE
7import sys
8sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
9# END OPENSOURCE
10import tmp.benchmarks_pb2 as benchmarks_pb2
11
12__file_size_map = {}
13
14def __get_data_size(filename):
15  if filename[0] != '/':
16    filename = os.path.dirname(os.path.abspath(__file__)) + "/../" + filename
17  if filename in __file_size_map:
18    return __file_size_map[filename]
19  benchmark_dataset = benchmarks_pb2.BenchmarkDataset()
20  benchmark_dataset.ParseFromString(
21      open(filename, "rb").read())
22  size = 0
23  count = 0
24  for payload in benchmark_dataset.payload:
25    size += len(payload)
26    count += 1
27  __file_size_map[filename] = (size, 1.0 * size / count)
28  return size, 1.0 * size / count
29
30
31def __extract_file_name(file_name):
32  name_list = re.split(r"[/\.]", file_name)
33  short_file_name = ""
34  for name in name_list:
35    if name[:14] == "google_message":
36      short_file_name = name
37  return short_file_name
38
39
40__results = []
41
42
43# CPP results example:
44# [
45#   "benchmarks": [
46#     {
47#       "bytes_per_second": int,
48#       "cpu_time_ns": double,
49#       "iterations": int,
50#       "name: string,
51#       "real_time_ns: double,
52#       ...
53#     },
54#     ...
55#   ],
56#   ...
57# ]
58def __parse_cpp_result(filename):
59  if filename == "":
60    return
61  if filename[0] != '/':
62    filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
63  with open(filename, "rb") as f:
64    results = json.loads(f.read())
65    for benchmark in results["benchmarks"]:
66      data_filename = "".join(
67          re.split("(_parse_|_serialize)", benchmark["name"])[0])
68      behavior = benchmark["name"][len(data_filename) + 1:]
69      if data_filename[:2] == "BM":
70        data_filename = data_filename[3:]
71      __results.append({
72        "language": "cpp",
73        "dataFilename": data_filename,
74        "behavior": behavior,
75        "throughput": benchmark["bytes_per_second"] / 2.0 ** 20
76      })
77
78
79# Synthetic benchmark results example:
80# [
81#   "benchmarks": [
82#     {
83#       "cpu_time_ns": double,
84#       "iterations": int,
85#       "name: string,
86#       "real_time_ns: double,
87#       ...
88#     },
89#     ...
90#   ],
91#   ...
92# ]
93def __parse_synthetic_result(filename):
94  if filename == "":
95    return
96  if filename[0] != "/":
97    filename = os.path.dirname(os.path.abspath(__file__)) + "/" + filename
98  with open(filename, "rb") as f:
99    results = json.loads(f.read())
100    for benchmark in results["benchmarks"]:
101      __results.append({
102          "language": "cpp",
103          "dataFilename": "",
104          "behavior": "synthetic",
105          "throughput": 10.0**9 / benchmark["cpu_time_ns"]
106      })
107
108
109# Python results example:
110# [
111#   [
112#     {
113#       "filename": string,
114#       "benchmarks": {
115#         behavior: results,
116#         ...
117#       },
118#     },
119#     ...
120#   ], #pure-python
121#   ...
122# ]
123def __parse_python_result(filename):
124  if filename == "":
125    return
126  if filename[0] != '/':
127    filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
128  with open(filename, "rb") as f:
129    results_list = json.loads(f.read())
130    for results in results_list:
131      for result in results:
132        _, avg_size = __get_data_size(result["filename"])
133        for behavior in result["benchmarks"]:
134          __results.append({
135            "language": "python",
136            "dataFilename": __extract_file_name(result["filename"]),
137            "behavior": behavior,
138            "throughput": result["benchmarks"][behavior]
139          })
140
141
142# Java results example:
143# [
144#   {
145#     "id": string,
146#     "instrumentSpec": {...},
147#     "measurements": [
148#       {
149#         "weight": float,
150#         "value": {
151#           "magnitude": float,
152#           "unit": string
153#         },
154#         ...
155#       },
156#       ...
157#     ],
158#     "run": {...},
159#     "scenario": {
160#       "benchmarkSpec": {
161#         "methodName": string,
162#         "parameters": {
163#            defined parameters in the benchmark: parameters value
164#         },
165#         ...
166#       },
167#       ...
168#     }
169#
170#   },
171#   ...
172# ]
173def __parse_java_result(filename):
174  if filename == "":
175    return
176  if filename[0] != '/':
177    filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
178  with open(filename, "rb") as f:
179    results = json.loads(f.read())
180    for result in results:
181      total_weight = 0
182      total_value = 0
183      for measurement in result["measurements"]:
184        total_weight += measurement["weight"]
185        total_value += measurement["value"]["magnitude"]
186      avg_time = total_value * 1.0 / total_weight
187      total_size, _ = __get_data_size(
188          result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"])
189      __results.append({
190        "language": "java",
191        "throughput": total_size / avg_time * 1e9 / 2 ** 20,
192        "behavior": result["scenario"]["benchmarkSpec"]["methodName"],
193        "dataFilename": __extract_file_name(
194            result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"])
195      })
196
197
198# Go benchmark results:
199#
200# goos: linux
201# goarch: amd64
202# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Unmarshal-12               3000      705784 ns/op
203# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Marshal-12                 2000      634648 ns/op
204# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Size-12                    5000      244174 ns/op
205# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Clone-12                    300     4120954 ns/op
206# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Merge-12                    300     4108632 ns/op
207# PASS
208# ok    _/usr/local/google/home/yilunchong/mygit/protobuf/benchmarks  124.173s
209def __parse_go_result(filename):
210  if filename == "":
211    return
212  if filename[0] != '/':
213    filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
214  with open(filename, "rb") as f:
215    for line in f:
216      result_list = re.split(r"[\ \t]+", line)
217      if result_list[0][:9] != "Benchmark":
218        continue
219      first_slash_index = result_list[0].find('/')
220      last_slash_index = result_list[0].rfind('/')
221      full_filename = result_list[0][first_slash_index+1:last_slash_index]
222      total_bytes, _ = __get_data_size(full_filename)
223      behavior_with_suffix = result_list[0][last_slash_index+1:]
224      last_dash = behavior_with_suffix.rfind("-")
225      if last_dash == -1:
226        behavior = behavior_with_suffix
227      else:
228        behavior = behavior_with_suffix[:last_dash]
229      __results.append({
230        "dataFilename": __extract_file_name(full_filename),
231        "throughput": total_bytes / float(result_list[2]) * 1e9 / 2 ** 20,
232        "behavior": behavior,
233        "language": "go"
234      })
235
236
237# Self built json results example:
238#
239# [
240#   {
241#     "filename": string,
242#     "benchmarks": {
243#       behavior: results,
244#       ...
245#     },
246#   },
247#   ...
248# ]
249def __parse_custom_result(filename, language):
250  if filename == "":
251    return
252  if filename[0] != '/':
253    filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename
254  with open(filename, "rb") as f:
255    results = json.loads(f.read())
256    for result in results:
257      _, avg_size = __get_data_size(result["filename"])
258      for behavior in result["benchmarks"]:
259        __results.append({
260          "language": language,
261          "dataFilename": __extract_file_name(result["filename"]),
262          "behavior": behavior,
263          "throughput": result["benchmarks"][behavior]
264        })
265
266
267def __parse_js_result(filename, language):
268  return __parse_custom_result(filename, language)
269
270def __parse_php_result(filename, language):
271  return __parse_custom_result(filename, language)
272
273
274def get_result_from_file(cpp_file="",
275                         java_file="",
276                         python_file="",
277                         go_file="",
278                         synthetic_file="",
279                         node_file="",
280                         php_c_file="",
281                         php_file=""):
282  results = {}
283  if cpp_file != "":
284    __parse_cpp_result(cpp_file)
285  if java_file != "":
286    __parse_java_result(java_file)
287  if python_file != "":
288    __parse_python_result(python_file)
289  if go_file != "":
290    __parse_go_result(go_file)
291  if synthetic_file != "":
292    __parse_synthetic_result(synthetic_file)
293  if node_file != "":
294    __parse_js_result(node_file, "node")
295  if php_file != "":
296    __parse_php_result(php_file, "php")
297  if php_c_file != "":
298    __parse_php_result(php_c_file, "php")
299
300  return __results
301