1#!/usr/bin/env python3 2# coding=utf-8 3 4# 5# Copyright (c) 2020 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19import json 20import os 21import shutil 22import sys 23 24SETTING_RED_STYLE = """\033[33;31m%s\033[0m""" 25 26def load_json_data(json_file_path): 27 json_data = {} 28 if os.path.isfile(json_file_path): 29 try: 30 with open(json_file_path, 'r') as file_read: 31 json_data = json.load(file_read) 32 if not json_data: 33 print("Loading file \"%s\" error" % json_file_path) 34 return {} 35 except(IOError, ValueError) as err_msg: 36 print("Error for load_json_data: \"%s\"" % 37 json_file_path, err_msg) 38 else: 39 print("Info: \"%s\" not exist." % json_file_path) 40 return json_data 41 42 43def get_file_list(find_path, postfix): 44 file_names = os.listdir(find_path) 45 file_list = [] 46 if len(file_names) > 0: 47 for name in file_names: 48 if name.find(postfix) != -1 and name[-len(postfix):] == postfix: 49 file_list.append(name) 50 return file_list 51 52 53def get_file_list_by_postfix(path, postfix, filter_jar=""): 54 file_list = [] 55 for dirs in os.walk(path): 56 files = get_file_list(find_path=dirs[0], postfix=postfix) 57 for file_path in files: 58 if "" != file_path and -1 == file_path.find(__file__): 59 pos = file_path.rfind(os.sep) 60 file_name = file_path[pos + 1:] 61 file_path = os.path.join(dirs[0], file_path) 62 if filter_jar != "" and file_name == filter_jar: 63 print(SETTING_RED_STYLE % ("Skipped %s" % file_path)) 64 continue 65 file_list.append(file_path) 66 return file_list 67 68class BenchmarkReport(object): 69 SUBSYSTEM_SUMMARY = "OHOS_SUBSYSTEM_SUMMARY" 70 ENABLE_LINK = "OHOS_ENABLE_PASSCASE_LINK" 71 REPORT_SUMMARY = "OHOS_REPORT_SUMMARY" 72 LEGEND_DATA = "OHOS_LEGEND_DATA" 73 XAXIS_DATA = "OHOS_XAXIS_DATA" 74 SERIES_DATA = "OHOS_TITLE_DATA" 75 TITLE_TEXT = "OHOS_TITLE_TEST" 76 YAXIS_FORMATTER = "OHOS_YAXIS_FORMATTER" 77 78 79 def __init__(self): 80 self.index = 0 81 self.filtered = ["detail", "id", "pm", "owner", 82 "Count", "ScoreUnit", "Variance"] 83 self.default_item = [] 84 self.max_index = 1000 85 self.sbs_mdl_summary_list = [] 86 self.benchmark_list = [] 87 self._init_default_item() 88 89 def _init_default_item(self): 90 self.default_item.append("Subsystem") 91 self.default_item.append("Module") 92 self.default_item.append("Testsuit") 93 self.default_item.append("Benchmark") 94 self.default_item.append("Mode") 95 self.default_item.append("RunType") 96 self.default_item.append("TestTargetName") 97 self.default_item.append("TestTargetMethod") 98 self.default_item.append("Repetitions") 99 self.default_item.append("RepetitionIndex") 100 self.default_item.append("Threads") 101 self.default_item.append("Iterations") 102 self.default_item.append("Score") 103 self.default_item.append("CpuTime") 104 self.max_index = len(self.default_item) + 1000 105 106 def generate_benchmark(self, args): 107 if args is None or len(args) <= 2: 108 print(SETTING_RED_STYLE % 109 "Error: source_dir and report_dir can't be empty") 110 return 111 112 src_path = sys.argv[1] 113 dest_path = os.path.abspath(sys.argv[2]) 114 115 print("source_dir: %s" % src_path) 116 print("report_dir: %s" % dest_path) 117 118 if not os.path.exists(src_path): 119 print("%s not exists" % src_path) 120 return 121 122 if os.path.exists(dest_path): 123 shutil.rmtree(dest_path) 124 125 self._get_benchmark_result_data(src_path) 126 self._generate_benchmark_summary_report(os.path.abspath(dest_path)) 127 self._generate_all_benchmark_detail(os.path.abspath(dest_path)) 128 129 def _remove_iterations(self, mdl_summary_list): 130 final_mdl_summary = [] 131 for item_info in mdl_summary_list: 132 copy_item = item_info.copy() 133 copy_item.pop("Iterations") 134 final_mdl_summary.append(copy_item) 135 return final_mdl_summary 136 137 def _get_benchmark_result_data(self, src_path): 138 self.benchmark_list = [] 139 self.sbs_mdl_summary_list = [] 140 system_summary_dic = {} 141 json_files = get_file_list_by_postfix(src_path, ".json") 142 print("json_files %s" % json_files) 143 for json_file in json_files: 144 pos = json_file.find(src_path) 145 subsystem_root = json_file[pos + len(src_path):] 146 dir_list = subsystem_root.split(os.sep) 147 sbs_name = dir_list[1] 148 module_name = dir_list[2] 149 testsuit_name = dir_list[len(dir_list) - 2] 150 151 print(SETTING_RED_STYLE % ( 152 "subsystem_root: %s \n\n" 153 "subsystem_name: %s \n\n" 154 "module_name: %s \n\n" 155 "testsuit_name: %s \n\n" % 156 (subsystem_root, str(sbs_name), 157 str(module_name), str(testsuit_name)))) 158 159 mdl_summary_list = self._get_subsystem_cxx_benchmark(sbs_name, 160 module_name, testsuit_name, json_file) 161 self.benchmark_list.extend(mdl_summary_list) 162 163 if sbs_name in system_summary_dic.keys() \ 164 and testsuit_name in system_summary_dic[sbs_name].keys(): 165 subsystem_summary_dic = \ 166 system_summary_dic[sbs_name][testsuit_name] 167 subsystem_summary_dic["children"] += \ 168 self._remove_iterations(mdl_summary_list) 169 else: 170 self.index += 1 171 subsystem_summary_dic = dict() 172 subsystem_summary_dic["id"] = self.index 173 subsystem_summary_dic["Subsystem"] = sbs_name 174 subsystem_summary_dic["Testsuit"] = testsuit_name 175 subsystem_summary_dic["Module"] = "---" 176 subsystem_summary_dic["Detail"] = "" 177 subsystem_summary_dic["TestTargetName"] = "---" 178 subsystem_summary_dic["TestTargetMethod"] = "---" 179 subsystem_summary_dic["RunType"] = "---" 180 subsystem_summary_dic["Benchmark"] = "---" 181 subsystem_summary_dic["Mode"] = "---" 182 subsystem_summary_dic["Count"] = "---" 183 subsystem_summary_dic["Score"] = "---" 184 subsystem_summary_dic["ScoreUnit"] = "---" 185 subsystem_summary_dic["children"] = [] 186 subsystem_summary_dic["children"] += \ 187 self._remove_iterations(mdl_summary_list) 188 self.sbs_mdl_summary_list.append(subsystem_summary_dic) 189 system_summary_dic[sbs_name] = {} 190 system_summary_dic[sbs_name][testsuit_name] = \ 191 subsystem_summary_dic 192 subsystem_summary_dic["pm"] = "unknown" 193 subsystem_summary_dic["owner"] = "unknown" 194 195 def _get_subsystem_cxx_benchmark(self, sbs_name, module_name, 196 testsuit_name, json_file): 197 sbs_mdl_summary_list = list() 198 json_data_dic = load_json_data(json_file) 199 for json_data in json_data_dic.get("benchmarks", []): 200 self.index += 1 201 sbs_mdl_summary = dict() 202 sbs_mdl_summary["id"] = self.index 203 sbs_mdl_summary["Subsystem"] = sbs_name 204 sbs_mdl_summary["Module"] = module_name 205 sbs_mdl_summary["Testsuit"] = testsuit_name 206 sbs_mdl_summary["pm"] = "unknown" 207 sbs_mdl_summary["owner"] = "unknown" 208 209 benchmark_name = json_data.get("name", "").replace("/", "_"). \ 210 replace(":", "_") 211 test_target = benchmark_name.split("_")[0] 212 sbs_mdl_summary["TestTargetName"] = test_target 213 sbs_mdl_summary["TestTargetMethod"] = "%s()" % test_target 214 sbs_mdl_summary["RunType"] = str(json_data.get("run_type", "")) 215 sbs_mdl_summary["Mode"] = \ 216 str(json_data.get("aggregate_name", "normal")) 217 sbs_mdl_summary["Benchmark"] = benchmark_name 218 sbs_mdl_summary["Repetitions"] = json_data.get("repetitions", 0) 219 sbs_mdl_summary["RepetitionIndex"] = \ 220 json_data.get("repetition_index", 0) 221 sbs_mdl_summary["Threads"] = json_data.get("threads", 0) 222 sbs_mdl_summary["Iterations"] = json_data.get("iterations", 0) 223 224 score_unit = json_data.get("time_unit", "") 225 sbs_mdl_summary["ScoreUnit"] = score_unit 226 sbs_mdl_summary["CpuTime"] = "%.2e %s " % ( 227 json_data.get("cpu_time", 0), 228 score_unit 229 ) 230 sbs_mdl_summary["Score"] = "%.2e %s " % ( 231 json_data.get("real_time", 0), 232 score_unit 233 ) 234 sbs_mdl_summary["detail"] = "Link" 235 sbs_mdl_summary_list.append(sbs_mdl_summary) 236 return sbs_mdl_summary_list 237 238 def _generate_benchmark_summary_report(self, dest_dir_path): 239 tmpl_file_path = os.path.abspath(os.path.join( 240 os.path.dirname(__file__), 241 "..", "template", "benchmark_summary.html")) 242 if not os.path.exists(os.path.dirname(tmpl_file_path)): 243 print(SETTING_RED_STYLE % 244 ("Warning: %s not exists" % tmpl_file_path)) 245 return 246 247 out_report_file_path = os.path.join(dest_dir_path, "index.html") 248 if not os.path.exists(os.path.dirname(out_report_file_path)): 249 os.makedirs(os.path.dirname(out_report_file_path)) 250 251 if os.path.exists(tmpl_file_path): 252 try: 253 with open(os.path.abspath(tmpl_file_path), "r+") as file_read: 254 report_content = file_read.read() 255 file_read.close() 256 content_new = report_content 257 258 pos = content_new.find(BenchmarkReport.SUBSYSTEM_SUMMARY) 259 if pos >= 0: 260 content_new = \ 261 content_new[0:pos] + \ 262 str(self.sbs_mdl_summary_list) + \ 263 content_new[pos + 264 len(BenchmarkReport.SUBSYSTEM_SUMMARY): 265 len(content_new)] 266 267 try: 268 with open(os.path.abspath(out_report_file_path), "w") \ 269 as output_fd: 270 content_new = str(content_new) 271 output_fd.write(content_new) 272 except IOError as err_msg: 273 print("Error5 for open %s failed, with msg %s" % 274 (out_report_file_path, err_msg)) 275 except IOError as err_msg: 276 print("Error6 for open %s failed, with msg %s" % 277 (tmpl_file_path, err_msg)) 278 279 def _generate_all_benchmark_detail(self, dest_dir_parh): 280 for benchmark_info in self.benchmark_list: 281 self._generate_benchmark_detail(benchmark_info, 282 os.path.abspath(dest_dir_parh)) 283 284 def _is_filtered_id(self, item_key): 285 if item_key in self.filtered: 286 return True 287 return False 288 289 def _get_index_id(self, item_key): 290 pos = self.default_item.index(item_key) 291 if pos != -1: 292 return pos + 1 293 else: 294 self.max_index -= 1 295 return self.max_index 296 297 def _generate_benchmark_detail(self, benchmark_info, dest_dir_path): 298 report_tmpl_file_path = os.path.abspath( 299 os.path.join(os.path.dirname(__file__), 300 "..", "template", "benchmark_detail.html")) 301 if not os.path.exists(os.path.dirname(report_tmpl_file_path)): 302 print(SETTING_RED_STYLE % 303 ("Warning: %s not exists" % report_tmpl_file_path)) 304 return 305 306 out_report_file_path = os.path.join(os.path.abspath(dest_dir_path), 307 str(benchmark_info["Subsystem"]), 308 str(benchmark_info["Module"]), 309 str(benchmark_info["Testsuit"]), 310 str(benchmark_info["Benchmark"]) 311 + "_" 312 + str(benchmark_info["Mode"]) 313 + "_detail.html") 314 if not os.path.exists(os.path.dirname(out_report_file_path)): 315 os.makedirs(os.path.dirname(out_report_file_path)) 316 317 detail_info = self._get_detail_info(benchmark_info) 318 319 if os.path.exists(report_tmpl_file_path): 320 try: 321 with open(os.path.abspath(report_tmpl_file_path), "r+") \ 322 as file_read: 323 report_content = file_read.read() 324 file_read.close() 325 content_new = report_content 326 content_new = \ 327 self._update_report_summary(content_new, detail_info) 328 329 try: 330 with open(os.path.abspath(out_report_file_path), "w") \ 331 as output_fd: 332 output_fd.write(content_new) 333 except IOError as err_msg: 334 print("Error5 for open %s failed, with msg %s" % 335 (out_report_file_path, err_msg)) 336 except IOError as err_msg: 337 print("Error6 for open %s failed, with msg %s" % 338 (report_tmpl_file_path, err_msg)) 339 340 def _get_detail_info(self, benchmark_info): 341 detail_info = [] 342 self.max_index = 1000 343 for item_key, item_value in benchmark_info.items(): 344 if self._is_filtered_id(item_key): 345 continue 346 347 item_info = {"item": item_key, 348 "id": self._get_index_id(item_key), 349 "content": item_value.decode("UTF-8") 350 if isinstance(item_value, bytes) else item_value} 351 detail_info.append(item_info) 352 detail_info = sorted(detail_info, key=lambda s: s["id"]) 353 dest_detail_info = [] 354 index = 1 355 for item in detail_info: 356 item["id"] = index 357 dest_detail_info.append(item) 358 index += 1 359 return dest_detail_info 360 361 def _update_report_summary(self, content_new, detail_info): 362 pos = content_new.find(BenchmarkReport.REPORT_SUMMARY) 363 if pos >= 0: 364 content_new = \ 365 content_new[0:pos] + \ 366 str(detail_info) + \ 367 content_new[pos + 368 len(BenchmarkReport.REPORT_SUMMARY): 369 len(content_new)] 370 return content_new 371 372if __name__ == '__main__': 373 print("****************** Benchmark Report Starting ******************") 374 BenchmarkReport().generate_benchmark(sys.argv) 375 print("****************** Benchmark Report Finished ******************")