1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright (c) 2022 Huawei Device Co., Ltd. 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16# This file contains a RomAnalyzer for rom analyzation of standard device. 17 18import argparse 19import json 20import os 21import sys 22import typing 23from copy import deepcopy 24from typing import * 25import re 26import subprocess 27from pkgs.rom_ram_baseline_collector import RomRamBaselineCollector 28from pkgs.basic_tool import BasicTool, unit_adaptive 29from pkgs.gn_common_tool import GnCommonTool, GnVariableParser 30from pkgs.simple_excel_writer import SimpleExcelWriter 31 32debug = bool(sys.gettrace()) 33 34NOTFOUND = "NOTFOUND" 35 36 37class PreCollector: 38 """ 39 collect some info that system_module_info.json dosn't contains 40 """ 41 42 def __init__(self, project_path: str) -> None: 43 self.info_dict: Dict[str, Any] = dict() 44 self.project_path = BasicTool.get_abs_path(project_path) 45 self.result_dict = dict() 46 47 def _process_single_sa(self, item: str, start_pattern: str): 48 gn, _, _ = item.split(':') 49 with open(gn, 'r', encoding='utf-8') as f: 50 content = f.read() 51 p_itr: Iterator[re.Match] = BasicTool.match_paragraph( 52 content=content, start_pattern=start_pattern) 53 for p in p_itr: 54 p_content = p.group() 55 files: List[str] = GnVariableParser.list_parser( 56 "sources", p_content) 57 component_name, subsystem_name = GnCommonTool.find_part_subsystem( 58 gn, self.project_path) 59 for f in files: 60 f = f.split('/')[-1] 61 self.result_dict[f] = { 62 "subsystem_name": subsystem_name, 63 "component_name": component_name, 64 "gn_path": gn 65 } 66 67 def collect_sa_profile(self): 68 grep_kw = r"ohos_sa_profile" 69 grep_cmd = f"grep -rn '{grep_kw}' --include=BUILD.gn {self.project_path}" 70 content = BasicTool.execute( 71 grep_cmd, post_processor=lambda x: x.split('\n')) 72 for item in content: 73 if not item: 74 continue 75 self._process_single_sa(item, start_pattern=grep_kw) 76 77 78class RomAnalyzer: 79 80 @classmethod 81 def __collect_product_info(cls, system_module_info_json: Text, 82 project_path: Text, extra_info: Dict[str, Dict]) -> Dict[Text, Dict[Text, Text]]: 83 """ 84 根据system_module_info.json生成target字典 85 format: 86 { 87 "{file_name}":{ 88 "{subsytem_name}": abc, 89 "{component_name}": def, 90 "{gn_path}": ghi 91 } 92 } 93 if the unit of system_module_info.json has not field "label" and the "type" is "sa_profile", 94 find the subsystem_name and component_name in the BUILD.gn 95 """ 96 with open(system_module_info_json, 'r', encoding='utf-8') as f: 97 product_list = json.loads(f.read()) 98 project_path = BasicTool.get_abs_path(project_path) 99 product_info_dict: Dict[Text, Dict[Text, Text]] = dict() 100 for unit in product_list: 101 cs_flag = False 102 dest: List = unit.get("dest") 103 if not dest: 104 print("warning: keyword 'dest' not found in {}".format( 105 system_module_info_json)) 106 continue 107 label: Text = unit.get("label") 108 gn_path = component_name = subsystem_name = None 109 if label: 110 cs_flag = True 111 gn_path = os.path.join(project_path, label.split(':')[ 112 0].lstrip('/'), "BUILD.gn") 113 component_name = unit.get("part_name") 114 subsystem_name = unit.get("subsystem_name") 115 if not component_name: 116 cn, sn = GnCommonTool.find_part_subsystem( 117 gn_path, project_path) 118 component_name = cn 119 if not subsystem_name: 120 cn, sn = GnCommonTool.find_part_subsystem( 121 gn_path, project_path) 122 subsystem_name = sn 123 else: 124 print("warning: keyword 'label' not found in {}".format(unit)) 125 for target in dest: 126 if cs_flag: 127 product_info_dict[target] = { 128 "component_name": component_name, 129 "subsystem_name": subsystem_name, 130 "gn_path": gn_path, 131 } 132 continue 133 tmp = target.split('/')[-1] 134 pre_info = extra_info.get(tmp) 135 if not pre_info: 136 continue 137 else: 138 product_info_dict[target] = pre_info 139 return product_info_dict 140 141 @classmethod 142 def __inside_save_result_as_excel(cls, add_baseline, subsystem_name, component_name, 143 baseline, file_name, size): 144 if add_baseline: 145 return [subsystem_name, component_name, 146 baseline, file_name, size] 147 else: 148 return [subsystem_name, component_name, file_name, size] 149 150 @classmethod 151 def __save_result_as_excel(cls, result_dict: dict, output_name: str, add_baseline: bool): 152 header = ["subsystem_name", "component_name", 153 "output_file", "size(Byte)"] 154 if add_baseline: 155 header = ["subsystem_name", "component_name", "baseline", 156 "output_file", "size(Byte)"] 157 tmp_dict = deepcopy(result_dict) 158 excel_writer = SimpleExcelWriter("rom") 159 excel_writer.set_sheet_header(headers=header) 160 subsystem_start_row = 1 161 subsystem_end_row = 0 162 subsystem_col = 0 163 component_start_row = 1 164 component_end_row = 0 165 component_col = 1 166 if add_baseline: 167 baseline_col = 2 168 for subsystem_name in tmp_dict.keys(): 169 subsystem_dict = tmp_dict.get(subsystem_name) 170 subsystem_size = subsystem_dict.get("size") 171 subsystem_file_count = subsystem_dict.get("file_count") 172 del subsystem_dict["file_count"] 173 del subsystem_dict["size"] 174 subsystem_end_row += subsystem_file_count 175 176 for component_name in subsystem_dict.keys(): 177 component_dict: Dict[str, int] = subsystem_dict.get( 178 component_name) 179 component_size = component_dict.get("size") 180 component_file_count = component_dict.get("file_count") 181 baseline = component_dict.get("baseline") 182 del component_dict["file_count"] 183 del component_dict["size"] 184 if add_baseline: 185 del component_dict["baseline"] 186 component_end_row += component_file_count 187 188 for file_name, size in component_dict.items(): 189 line = cls.__inside_save_result_as_excel(add_baseline, subsystem_name, component_name, 190 baseline, file_name, size) 191 excel_writer.append_line(line) 192 excel_writer.write_merge(component_start_row, component_col, component_end_row, component_col, 193 component_name) 194 if add_baseline: 195 excel_writer.write_merge(component_start_row, baseline_col, component_end_row, baseline_col, 196 baseline) 197 component_start_row = component_end_row + 1 198 excel_writer.write_merge(subsystem_start_row, subsystem_col, subsystem_end_row, subsystem_col, 199 subsystem_name) 200 subsystem_start_row = subsystem_end_row + 1 201 excel_writer.save(output_name + ".xls") 202 203 @classmethod 204 def __put(cls, unit: typing.Dict[Text, Any], result_dict: typing.Dict[Text, Dict], baseline_dict: Dict[str, Any], 205 baseline: bool): 206 207 component_name = NOTFOUND if unit.get( 208 "component_name") is None else unit.get("component_name") 209 subsystem_name = NOTFOUND if unit.get( 210 "subsystem_name") is None else unit.get("subsystem_name") 211 212 def get_rom_baseline(): 213 if (not baseline_dict.get(subsystem_name)) or (not baseline_dict.get(subsystem_name).get(component_name)): 214 return str() 215 return baseline_dict.get(subsystem_name).get(component_name).get("rom") 216 217 size = unit.get("size") 218 relative_filepath = unit.get("relative_filepath") 219 if result_dict.get(subsystem_name) is None: # 子系统 220 result_dict[subsystem_name] = dict() 221 result_dict[subsystem_name]["size"] = 0 222 result_dict[subsystem_name]["file_count"] = 0 223 224 if result_dict.get(subsystem_name).get(component_name) is None: # 部件 225 result_dict[subsystem_name][component_name] = dict() 226 result_dict[subsystem_name][component_name]["size"] = 0 227 result_dict[subsystem_name][component_name]["file_count"] = 0 228 if baseline: 229 result_dict[subsystem_name][component_name]["baseline"] = get_rom_baseline( 230 ) 231 232 result_dict[subsystem_name]["size"] += size 233 result_dict[subsystem_name]["file_count"] += 1 234 result_dict[subsystem_name][component_name]["size"] += size 235 result_dict[subsystem_name][component_name]["file_count"] += 1 236 result_dict[subsystem_name][component_name][relative_filepath] = size 237 238 @classmethod 239 def result_unit_adaptive(self, result_dict: Dict[str, Dict]) -> None: 240 for subsystem_name, subsystem_info in result_dict.items(): 241 size = unit_adaptive(subsystem_info["size"]) 242 count = subsystem_info["file_count"] 243 if "size" in subsystem_info.keys(): 244 del subsystem_info["size"] 245 if "file_count" in subsystem_info.keys(): 246 del subsystem_info["file_count"] 247 for component_name, component_info in subsystem_info.items(): 248 component_info["size"] = unit_adaptive(component_info["size"]) 249 subsystem_info["size"] = size 250 subsystem_info["file_count"] = count 251 252 @classmethod 253 def analysis(cls, system_module_info_json: Text, product_dirs: List[str], 254 project_path: Text, product_name: Text, output_file: Text, output_execel: bool, add_baseline: bool, 255 unit_adapt: bool): 256 """ 257 system_module_info_json: json文件 258 product_dirs：要处理的产物的路径列表如["vendor", "system/"] 259 project_path: 项目根路径 260 product_name: eg，rk3568 261 output_file: basename of output file 262 """ 263 project_path = BasicTool.get_abs_path(project_path) 264 rom_baseline_dict: Dict[str, Any] = RomRamBaselineCollector.collect( 265 project_path) 266 with open("rom_ram_baseline.json", 'w', encoding='utf-8') as f: 267 json.dump(rom_baseline_dict, f, indent=4) 268 phone_dir = os.path.join( 269 project_path, "out", product_name, "packages", "phone") 270 product_dirs = [os.path.join(phone_dir, d) for d in product_dirs] 271 pre_collector = PreCollector(project_path) 272 pre_collector.collect_sa_profile() 273 extra_product_info_dict: Dict[str, Dict] = pre_collector.result_dict 274 product_info_dict = cls.__collect_product_info( 275 system_module_info_json, project_path, 276 extra_info=extra_product_info_dict) # collect product info from json file 277 result_dict: Dict[Text:Dict] = dict() 278 for d in product_dirs: 279 file_list: List[Text] = BasicTool.find_all_files(d) 280 for f in file_list: 281 size = os.path.getsize(f) 282 relative_filepath = f.replace(phone_dir, "").lstrip(os.sep) 283 unit: Dict[Text, Any] = product_info_dict.get( 284 relative_filepath) 285 if not unit: 286 bf = f.split('/')[-1] 287 unit: Dict[Text, Any] = product_info_dict.get(bf) 288 if not unit: 289 unit = dict() 290 unit["size"] = size 291 unit["relative_filepath"] = relative_filepath 292 cls.__put(unit, result_dict, rom_baseline_dict, add_baseline) 293 output_dir, _ = os.path.split(output_file) 294 if len(output_dir) != 0: 295 os.makedirs(output_dir, exist_ok=True) 296 if unit_adapt: 297 cls.result_unit_adaptive(result_dict) 298 with open(output_file + ".json", 'w', encoding='utf-8') as f: 299 f.write(json.dumps(result_dict, indent=4)) 300 if output_execel: 301 cls.__save_result_as_excel(result_dict, output_file, add_baseline) 302 303 304def get_args(): 305 VERSION = 2.0 306 parser = argparse.ArgumentParser( 307 description=f"analyze rom size of component.\n") 308 parser.add_argument("-v", "-version", action="version", 309 version=f"version {VERSION}") 310 parser.add_argument("-p", "--project_path", type=str, required=True, 311 help="root path of openharmony. eg: -p ~/openharmony") 312 parser.add_argument("-j", "--module_info_json", required=True, type=str, 313 help="path of out/{product_name}/packages/phone/system_module_info.json") 314 parser.add_argument("-n", "--product_name", required=True, 315 type=str, help="product name. eg: -n rk3568") 316 parser.add_argument("-d", "--product_dir", required=True, action="append", 317 help="subdirectories of out/{product_name}/packages/phone to be counted." 318 "eg: -d system -d vendor") 319 parser.add_argument("-b", "--baseline", action="store_true", 320 help="add baseline of component to the result(-b) or not.") 321 parser.add_argument("-o", "--output_file", type=str, default="rom_analysis_result", 322 help="basename of output file, default: rom_analysis_result. eg: demo/rom_analysis_result") 323 parser.add_argument("-u", "--unit_adaptive", 324 action="store_true", help="unit adaptive") 325 parser.add_argument("-e", "--excel", type=bool, default=False, 326 help="if output result as excel, default: False. eg: -e True") 327 args = parser.parse_args() 328 return args 329 330 331if __name__ == '__main__': 332 args = get_args() 333 module_info_json = args.module_info_json 334 project_origin_path = args.project_path 335 product = args.product_name 336 product_dirs = args.product_dir 337 output_file_name = args.output_file 338 output_excel = args.excel 339 baseline_path = args.baseline 340 unit_adaptiv = args.unit_adaptive 341 RomAnalyzer.analysis(module_info_json, product_dirs, 342 project_origin_path, product, output_file_name, output_excel, baseline_path, unit_adaptiv) 343