1#!/usr/bin/env python3 2# coding=utf-8 3 4# 5# Copyright (c) 2020 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19import json 20import os 21import shutil 22import sys 23import stat 24 25FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL 26MODES = stat.S_IWUSR | stat.S_IRUSR 27 28SETTING_RED_STYLE = """\033[33;31m%s\033[0m""" 29 30def load_json_data(json_file_path): 31 json_data = {} 32 if os.path.isfile(json_file_path): 33 try: 34 with open(json_file_path, 'r') as file_read: 35 json_data = json.load(file_read) 36 if not json_data: 37 print("Loading file \"%s\" error" % json_file_path) 38 return {} 39 except(IOError, ValueError) as err_msg: 40 print("Error for load_json_data: ", json_file_path) 41 else: 42 print("Info: \"%s\" not exist." % json_file_path) 43 return json_data 44 45 46def get_file_list(find_path, postfix): 47 file_names = os.listdir(find_path) 48 file_list = [] 49 if len(file_names) > 0: 50 for name in file_names: 51 if name.find(postfix) != -1 and name[-len(postfix):] == postfix: 52 file_list.append(name) 53 return file_list 54 55 56def get_file_list_by_postfix(path, postfix, filter_jar=""): 57 file_list = [] 58 for dirs in os.walk(path): 59 files = get_file_list(find_path=dirs[0], postfix=postfix) 60 for file_path in files: 61 if "" != file_path and -1 == file_path.find(__file__): 62 pos = file_path.rfind(os.sep) 63 file_name = file_path[pos + 1:] 64 file_path = os.path.join(dirs[0], file_path) 65 if filter_jar != "" and file_name == filter_jar: 66 print(SETTING_RED_STYLE % ("Skipped %s" % file_path)) 67 continue 68 file_list.append(file_path) 69 return file_list 70 71class BenchmarkReport(object): 72 SUBSYSTEM_SUMMARY = "OHOS_SUBSYSTEM_SUMMARY" 73 ENABLE_LINK = "OHOS_ENABLE_PASSCASE_LINK" 74 REPORT_SUMMARY = "OHOS_REPORT_SUMMARY" 75 LEGEND_DATA = "OHOS_LEGEND_DATA" 76 XAXIS_DATA = "OHOS_XAXIS_DATA" 77 SERIES_DATA = "OHOS_TITLE_DATA" 78 TITLE_TEXT = "OHOS_TITLE_TEST" 79 YAXIS_FORMATTER = "OHOS_YAXIS_FORMATTER" 80 81 82 def __init__(self): 83 self.index = 0 84 self.filtered = ["detail", "id", "pm", "owner", 85 "Count", "ScoreUnit", "Variance"] 86 self.default_item = [] 87 self.max_index = 1000 88 self.sbs_mdl_summary_list = [] 89 self.benchmark_list = [] 90 self._init_default_item() 91 92 def _init_default_item(self): 93 self.default_item.append("Subsystem") 94 self.default_item.append("Module") 95 self.default_item.append("Testsuit") 96 self.default_item.append("Benchmark") 97 self.default_item.append("Mode") 98 self.default_item.append("RunType") 99 self.default_item.append("TestTargetName") 100 self.default_item.append("TestTargetMethod") 101 self.default_item.append("Repetitions") 102 self.default_item.append("RepetitionIndex") 103 self.default_item.append("Threads") 104 self.default_item.append("Iterations") 105 self.default_item.append("Score") 106 self.default_item.append("CpuTime") 107 self.max_index = len(self.default_item) + 1000 108 109 def generate_benchmark(self, args): 110 if args is None or len(args) <= 2: 111 print(SETTING_RED_STYLE % 112 "Error: source_dir and report_dir can't be empty") 113 return 114 115 src_path = sys.argv[1] 116 dest_path = os.path.abspath(sys.argv[2]) 117 118 print("source_dir: %s" % src_path) 119 print("report_dir: %s" % dest_path) 120 121 if not os.path.exists(src_path): 122 print("%s not exists" % src_path) 123 return 124 125 if os.path.exists(dest_path): 126 shutil.rmtree(dest_path) 127 128 self._get_benchmark_result_data(src_path) 129 self._generate_benchmark_summary_report(os.path.abspath(dest_path)) 130 self._generate_all_benchmark_detail(os.path.abspath(dest_path)) 131 132 def _remove_iterations(self, mdl_summary_list): 133 final_mdl_summary = [] 134 for item_info in mdl_summary_list: 135 copy_item = item_info.copy() 136 copy_item.pop("Iterations") 137 final_mdl_summary.append(copy_item) 138 return final_mdl_summary 139 140 def _get_benchmark_result_data(self, src_path): 141 self.benchmark_list = [] 142 self.sbs_mdl_summary_list = [] 143 system_summary_dic = {} 144 json_files = get_file_list_by_postfix(src_path, ".json") 145 print("json_files %s" % json_files) 146 for json_file in json_files: 147 pos = json_file.find(src_path) 148 subsystem_root = json_file[pos + len(src_path):] 149 dir_list = subsystem_root.split(os.sep) 150 sbs_name = dir_list[1] 151 module_name = dir_list[2] 152 testsuit_name = dir_list[len(dir_list) - 2] 153 154 print(SETTING_RED_STYLE % ( 155 "subsystem_root: %s \n\n" 156 "subsystem_name: %s \n\n" 157 "module_name: %s \n\n" 158 "testsuit_name: %s \n\n" % 159 (subsystem_root, str(sbs_name), 160 str(module_name), str(testsuit_name)))) 161 162 mdl_summary_list = self._get_subsystem_cxx_benchmark(sbs_name, 163 module_name, testsuit_name, json_file) 164 self.benchmark_list.extend(mdl_summary_list) 165 166 if sbs_name in system_summary_dic.keys() \ 167 and testsuit_name in system_summary_dic[sbs_name].keys(): 168 subsystem_summary_dic = \ 169 system_summary_dic[sbs_name][testsuit_name] 170 subsystem_summary_dic["children"] += \ 171 self._remove_iterations(mdl_summary_list) 172 else: 173 self.index += 1 174 subsystem_summary_dic = dict() 175 subsystem_summary_dic["id"] = self.index 176 subsystem_summary_dic["Subsystem"] = sbs_name 177 subsystem_summary_dic["Testsuit"] = testsuit_name 178 subsystem_summary_dic["Module"] = "---" 179 subsystem_summary_dic["Detail"] = "" 180 subsystem_summary_dic["TestTargetName"] = "---" 181 subsystem_summary_dic["TestTargetMethod"] = "---" 182 subsystem_summary_dic["RunType"] = "---" 183 subsystem_summary_dic["Benchmark"] = "---" 184 subsystem_summary_dic["Mode"] = "---" 185 subsystem_summary_dic["Count"] = "---" 186 subsystem_summary_dic["Score"] = "---" 187 subsystem_summary_dic["ScoreUnit"] = "---" 188 subsystem_summary_dic["children"] = [] 189 subsystem_summary_dic["children"] += \ 190 self._remove_iterations(mdl_summary_list) 191 self.sbs_mdl_summary_list.append(subsystem_summary_dic) 192 system_summary_dic[sbs_name] = {} 193 system_summary_dic[sbs_name][testsuit_name] = \ 194 subsystem_summary_dic 195 subsystem_summary_dic["pm"] = "unknown" 196 subsystem_summary_dic["owner"] = "unknown" 197 198 def _get_subsystem_cxx_benchmark(self, sbs_name, module_name, 199 testsuit_name, json_file): 200 sbs_mdl_summary_list = list() 201 json_data_dic = load_json_data(json_file) 202 for json_data in json_data_dic.get("benchmarks", []): 203 self.index += 1 204 sbs_mdl_summary = dict() 205 sbs_mdl_summary["id"] = self.index 206 sbs_mdl_summary["Subsystem"] = sbs_name 207 sbs_mdl_summary["Module"] = module_name 208 sbs_mdl_summary["Testsuit"] = testsuit_name 209 sbs_mdl_summary["pm"] = "unknown" 210 sbs_mdl_summary["owner"] = "unknown" 211 212 benchmark_name = json_data.get("name", "").replace("/", "_"). \ 213 replace(":", "_") 214 test_target = benchmark_name.split("_")[0] 215 sbs_mdl_summary["TestTargetName"] = test_target 216 sbs_mdl_summary["TestTargetMethod"] = "%s()" % test_target 217 sbs_mdl_summary["RunType"] = str(json_data.get("run_type", "")) 218 sbs_mdl_summary["Mode"] = \ 219 str(json_data.get("aggregate_name", "normal")) 220 sbs_mdl_summary["Benchmark"] = benchmark_name 221 sbs_mdl_summary["Repetitions"] = json_data.get("repetitions", 0) 222 sbs_mdl_summary["RepetitionIndex"] = \ 223 json_data.get("repetition_index", 0) 224 sbs_mdl_summary["Threads"] = json_data.get("threads", 0) 225 sbs_mdl_summary["Iterations"] = json_data.get("iterations", 0) 226 227 score_unit = json_data.get("time_unit", "") 228 sbs_mdl_summary["ScoreUnit"] = score_unit 229 sbs_mdl_summary["CpuTime"] = "%.2e %s " % ( 230 json_data.get("cpu_time", 0), 231 score_unit 232 ) 233 sbs_mdl_summary["Score"] = "%.2e %s " % ( 234 json_data.get("real_time", 0), 235 score_unit 236 ) 237 sbs_mdl_summary["detail"] = "Link" 238 sbs_mdl_summary_list.append(sbs_mdl_summary) 239 return sbs_mdl_summary_list 240 241 def _generate_benchmark_summary_report(self, dest_dir_path): 242 tmpl_file_path = os.path.abspath(os.path.join( 243 os.path.dirname(__file__), 244 "..", "template", "benchmark_summary.html")) 245 if not os.path.exists(os.path.dirname(tmpl_file_path)): 246 print(SETTING_RED_STYLE % 247 ("Warning: %s not exists" % tmpl_file_path)) 248 return 249 250 out_report_file_path = os.path.join(dest_dir_path, "index.html") 251 if not os.path.exists(os.path.dirname(out_report_file_path)): 252 os.makedirs(os.path.dirname(out_report_file_path)) 253 254 if os.path.exists(tmpl_file_path): 255 try: 256 with open(os.path.abspath(tmpl_file_path), "r+") as file_read: 257 report_content = file_read.read() 258 file_read.close() 259 content_new = report_content 260 261 pos = content_new.find(BenchmarkReport.SUBSYSTEM_SUMMARY) 262 if pos >= 0: 263 content_new = \ 264 content_new[0:pos] + \ 265 str(self.sbs_mdl_summary_list) + \ 266 content_new[pos + 267 len(BenchmarkReport.SUBSYSTEM_SUMMARY): 268 len(content_new)] 269 270 try: 271 if os.path.exists(os.path.abspath(out_report_file_path)): 272 os.remove(os.path.abspath(out_report_file_path)) 273 with os.fdopen(os.open(os.path.abspath(out_report_file_path), 274 FLAGS, MODES), 'w') as output_fd: 275 content_new = str(content_new) 276 output_fd.write(content_new) 277 except IOError as err_msg: 278 print("Error5 for open %s failed:" % out_report_file_path) 279 except IOError as err_msg: 280 print("Error6 for open %s failed:" % tmpl_file_path) 281 282 def _generate_all_benchmark_detail(self, dest_dir_parh): 283 for benchmark_info in self.benchmark_list: 284 self._generate_benchmark_detail(benchmark_info, 285 os.path.abspath(dest_dir_parh)) 286 287 def _is_filtered_id(self, item_key): 288 if item_key in self.filtered: 289 return True 290 return False 291 292 def _get_index_id(self, item_key): 293 pos = self.default_item.index(item_key) 294 if pos != -1: 295 return pos + 1 296 else: 297 self.max_index -= 1 298 return self.max_index 299 300 def _generate_benchmark_detail(self, benchmark_info, dest_dir_path): 301 report_tmpl_file_path = os.path.abspath( 302 os.path.join(os.path.dirname(__file__), 303 "..", "template", "benchmark_detail.html")) 304 if not os.path.exists(os.path.dirname(report_tmpl_file_path)): 305 print(SETTING_RED_STYLE % 306 ("Warning: %s not exists" % report_tmpl_file_path)) 307 return 308 309 out_report_file_path = os.path.join(os.path.abspath(dest_dir_path), 310 str(benchmark_info["Subsystem"]), 311 str(benchmark_info["Module"]), 312 str(benchmark_info["Testsuit"]), 313 str(benchmark_info["Benchmark"]) 314 + "_" 315 + str(benchmark_info["Mode"]) 316 + "_detail.html") 317 if not os.path.exists(os.path.dirname(out_report_file_path)): 318 os.makedirs(os.path.dirname(out_report_file_path)) 319 320 detail_info = self._get_detail_info(benchmark_info) 321 322 if os.path.exists(report_tmpl_file_path): 323 try: 324 with open(os.path.abspath(report_tmpl_file_path), "r+") \ 325 as file_read: 326 report_content = file_read.read() 327 file_read.close() 328 content_new = report_content 329 content_new = \ 330 self._update_report_summary(content_new, detail_info) 331 332 try: 333 if os.path.exists(os.path.abspath(out_report_file_path)): 334 os.remove(os.path.abspath(out_report_file_path)) 335 with os.fdopen(os.open(os.path.abspath(out_report_file_path), 336 FLAGS, MODES), 'w') as output_fd: 337 output_fd.write(content_new) 338 except IOError as err_msg: 339 print("Error5 for open %s failed:" % out_report_file_path) 340 except IOError as err_msg: 341 print("Error6 for open %s failed" % report_tmpl_file_path) 342 343 def _get_detail_info(self, benchmark_info): 344 detail_info = [] 345 self.max_index = 1000 346 for item_key, item_value in benchmark_info.items(): 347 if self._is_filtered_id(item_key): 348 continue 349 350 item_info = {"item": item_key, 351 "id": self._get_index_id(item_key), 352 "content": item_value.decode("UTF-8") 353 if isinstance(item_value, bytes) else item_value} 354 detail_info.append(item_info) 355 detail_info = sorted(detail_info, key=lambda s: s["id"]) 356 dest_detail_info = [] 357 index = 1 358 for item in detail_info: 359 item["id"] = index 360 dest_detail_info.append(item) 361 index += 1 362 return dest_detail_info 363 364 def _update_report_summary(self, content_new, detail_info): 365 pos = content_new.find(BenchmarkReport.REPORT_SUMMARY) 366 if pos >= 0: 367 content_new = \ 368 content_new[0:pos] + \ 369 str(detail_info) + \ 370 content_new[pos + 371 len(BenchmarkReport.REPORT_SUMMARY): 372 len(content_new)] 373 return content_new 374 375if __name__ == '__main__': 376 print("****************** Benchmark Report Starting ******************") 377 BenchmarkReport().generate_benchmark(sys.argv) 378 print("****************** Benchmark Report Finished ******************")