1# This import depends on the automake rule protoc_middleman, please make sure 2# protoc_middleman has been built before run this file. 3import json 4import re 5import os.path 6# BEGIN OPENSOURCE 7import sys 8sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) 9# END OPENSOURCE 10import tmp.benchmarks_pb2 as benchmarks_pb2 11 12__file_size_map = {} 13 14def __get_data_size(filename): 15 if filename[0] != '/': 16 filename = os.path.dirname(os.path.abspath(__file__)) + "/../" + filename 17 if filename in __file_size_map: 18 return __file_size_map[filename] 19 benchmark_dataset = benchmarks_pb2.BenchmarkDataset() 20 benchmark_dataset.ParseFromString( 21 open(filename, "rb").read()) 22 size = 0 23 count = 0 24 for payload in benchmark_dataset.payload: 25 size += len(payload) 26 count += 1 27 __file_size_map[filename] = (size, 1.0 * size / count) 28 return size, 1.0 * size / count 29 30 31def __extract_file_name(file_name): 32 name_list = re.split(r"[/\.]", file_name) 33 short_file_name = "" 34 for name in name_list: 35 if name[:14] == "google_message": 36 short_file_name = name 37 return short_file_name 38 39 40__results = [] 41 42 43# CPP results example: 44# [ 45# "benchmarks": [ 46# { 47# "bytes_per_second": int, 48# "cpu_time_ns": double, 49# "iterations": int, 50# "name: string, 51# "real_time_ns: double, 52# ... 53# }, 54# ... 55# ], 56# ... 57# ] 58def __parse_cpp_result(filename): 59 if filename == "": 60 return 61 if filename[0] != '/': 62 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 63 with open(filename, "rb") as f: 64 results = json.loads(f.read()) 65 for benchmark in results["benchmarks"]: 66 data_filename = "".join( 67 re.split("(_parse_|_serialize)", benchmark["name"])[0]) 68 behavior = benchmark["name"][len(data_filename) + 1:] 69 if data_filename[:2] == "BM": 70 data_filename = data_filename[3:] 71 __results.append({ 72 "language": "cpp", 73 "dataFilename": data_filename, 74 "behavior": behavior, 75 "throughput": benchmark["bytes_per_second"] / 2.0 ** 20 76 }) 77 78 79# Synthetic benchmark results example: 80# [ 81# "benchmarks": [ 82# { 83# "cpu_time_ns": double, 84# "iterations": int, 85# "name: string, 86# "real_time_ns: double, 87# ... 88# }, 89# ... 90# ], 91# ... 92# ] 93def __parse_synthetic_result(filename): 94 if filename == "": 95 return 96 if filename[0] != "/": 97 filename = os.path.dirname(os.path.abspath(__file__)) + "/" + filename 98 with open(filename, "rb") as f: 99 results = json.loads(f.read()) 100 for benchmark in results["benchmarks"]: 101 __results.append({ 102 "language": "cpp", 103 "dataFilename": "", 104 "behavior": "synthetic", 105 "throughput": 10.0**9 / benchmark["cpu_time_ns"] 106 }) 107 108 109# Python results example: 110# [ 111# [ 112# { 113# "filename": string, 114# "benchmarks": { 115# behavior: results, 116# ... 117# }, 118# }, 119# ... 120# ], #pure-python 121# ... 122# ] 123def __parse_python_result(filename): 124 if filename == "": 125 return 126 if filename[0] != '/': 127 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 128 with open(filename, "rb") as f: 129 results_list = json.loads(f.read()) 130 for results in results_list: 131 for result in results: 132 _, avg_size = __get_data_size(result["filename"]) 133 for behavior in result["benchmarks"]: 134 __results.append({ 135 "language": "python", 136 "dataFilename": __extract_file_name(result["filename"]), 137 "behavior": behavior, 138 "throughput": result["benchmarks"][behavior] 139 }) 140 141 142# Java results example: 143# [ 144# { 145# "id": string, 146# "instrumentSpec": {...}, 147# "measurements": [ 148# { 149# "weight": float, 150# "value": { 151# "magnitude": float, 152# "unit": string 153# }, 154# ... 155# }, 156# ... 157# ], 158# "run": {...}, 159# "scenario": { 160# "benchmarkSpec": { 161# "methodName": string, 162# "parameters": { 163# defined parameters in the benchmark: parameters value 164# }, 165# ... 166# }, 167# ... 168# } 169# 170# }, 171# ... 172# ] 173def __parse_java_result(filename): 174 if filename == "": 175 return 176 if filename[0] != '/': 177 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 178 with open(filename, "rb") as f: 179 results = json.loads(f.read()) 180 for result in results: 181 total_weight = 0 182 total_value = 0 183 for measurement in result["measurements"]: 184 total_weight += measurement["weight"] 185 total_value += measurement["value"]["magnitude"] 186 avg_time = total_value * 1.0 / total_weight 187 total_size, _ = __get_data_size( 188 result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) 189 __results.append({ 190 "language": "java", 191 "throughput": total_size / avg_time * 1e9 / 2 ** 20, 192 "behavior": result["scenario"]["benchmarkSpec"]["methodName"], 193 "dataFilename": __extract_file_name( 194 result["scenario"]["benchmarkSpec"]["parameters"]["dataFile"]) 195 }) 196 197 198# Go benchmark results: 199# 200# goos: linux 201# goarch: amd64 202# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Unmarshal-12 3000 705784 ns/op 203# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Marshal-12 2000 634648 ns/op 204# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Size-12 5000 244174 ns/op 205# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Clone-12 300 4120954 ns/op 206# Benchmark/.././datasets/google_message2/dataset.google_message2.pb/Merge-12 300 4108632 ns/op 207# PASS 208# ok _/usr/local/google/home/yilunchong/mygit/protobuf/benchmarks 124.173s 209def __parse_go_result(filename): 210 if filename == "": 211 return 212 if filename[0] != '/': 213 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 214 with open(filename, "rb") as f: 215 for line in f: 216 result_list = re.split(r"[\ \t]+", line) 217 if result_list[0][:9] != "Benchmark": 218 continue 219 first_slash_index = result_list[0].find('/') 220 last_slash_index = result_list[0].rfind('/') 221 full_filename = result_list[0][first_slash_index+1:last_slash_index] 222 total_bytes, _ = __get_data_size(full_filename) 223 behavior_with_suffix = result_list[0][last_slash_index+1:] 224 last_dash = behavior_with_suffix.rfind("-") 225 if last_dash == -1: 226 behavior = behavior_with_suffix 227 else: 228 behavior = behavior_with_suffix[:last_dash] 229 __results.append({ 230 "dataFilename": __extract_file_name(full_filename), 231 "throughput": total_bytes / float(result_list[2]) * 1e9 / 2 ** 20, 232 "behavior": behavior, 233 "language": "go" 234 }) 235 236 237# Self built json results example: 238# 239# [ 240# { 241# "filename": string, 242# "benchmarks": { 243# behavior: results, 244# ... 245# }, 246# }, 247# ... 248# ] 249def __parse_custom_result(filename, language): 250 if filename == "": 251 return 252 if filename[0] != '/': 253 filename = os.path.dirname(os.path.abspath(__file__)) + '/' + filename 254 with open(filename, "rb") as f: 255 results = json.loads(f.read()) 256 for result in results: 257 _, avg_size = __get_data_size(result["filename"]) 258 for behavior in result["benchmarks"]: 259 __results.append({ 260 "language": language, 261 "dataFilename": __extract_file_name(result["filename"]), 262 "behavior": behavior, 263 "throughput": result["benchmarks"][behavior] 264 }) 265 266 267def __parse_js_result(filename, language): 268 return __parse_custom_result(filename, language) 269 270def __parse_php_result(filename, language): 271 return __parse_custom_result(filename, language) 272 273 274def get_result_from_file(cpp_file="", 275 java_file="", 276 python_file="", 277 go_file="", 278 synthetic_file="", 279 node_file="", 280 php_c_file="", 281 php_file=""): 282 results = {} 283 if cpp_file != "": 284 __parse_cpp_result(cpp_file) 285 if java_file != "": 286 __parse_java_result(java_file) 287 if python_file != "": 288 __parse_python_result(python_file) 289 if go_file != "": 290 __parse_go_result(go_file) 291 if synthetic_file != "": 292 __parse_synthetic_result(synthetic_file) 293 if node_file != "": 294 __parse_js_result(node_file, "node") 295 if php_file != "": 296 __parse_php_result(php_file, "php") 297 if php_c_file != "": 298 __parse_php_result(php_c_file, "php") 299 300 return __results 301