1#!/usr/bin/env python 2# -*- coding: utf-8 -*- 3 4# 5# Copyright (c) 2020 Huawei Device Co., Ltd. 6# Licensed under the Apache License, Version 2.0 (the "License"); 7# you may not use this file except in compliance with the License. 8# You may obtain a copy of the License at 9# 10# http://www.apache.org/licenses/LICENSE-2.0 11# 12# Unless required by applicable law or agreed to in writing, software 13# distributed under the License is distributed on an "AS IS" BASIS, 14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15# See the License for the specific language governing permissions and 16# limitations under the License. 17# 18 19import os 20import json 21import shutil 22from subprocess import check_output 23from subprocess import CalledProcessError 24try: 25 from queue import Queue 26except ImportError: 27 from Queue import Queue 28from collections import defaultdict 29 30from hb_internal import CONFIG_JSON 31from hb_internal.common.utils import read_json_file 32from hb_internal.cts.common import check_path 33from hb_internal.common.utils import get_project_path 34from hb_internal.common.utils import hb_info 35from hb_internal.common.utils import hb_warning 36from hb_internal.common.utils import OHOSException 37 38 39class CTS(): 40 def __init__(self): 41 self._board = None 42 self._kernel = None 43 self.kernel_of_board = defaultdict(list) 44 self._set_path() 45 self.fields = defaultdict(list) 46 self.comp_fields = defaultdict(list) 47 48 def __iter__(self): 49 for subsystem_cls in self.subsystems: 50 yield subsystem_cls 51 52 def _set_path(self): 53 self._code_path = get_project_path(CONFIG_JSON) 54 if self._code_path is None: 55 raise OHOSException('Please run command "hb set" to ' 56 'init OHOS development environment') 57 58 self._components_path = os.path.join(self._code_path, 59 'build', 60 'lite', 61 'components') 62 63 self._platform_path = os.path.join(self._code_path, 64 'build', 65 'lite', 66 'platform') 67 68 self._kernel_path = os.path.join(self._components_path, 69 'kernel.json') 70 71 self._product_path = os.path.join(self._code_path, 72 'build', 73 'lite', 74 'product') 75 76 @property 77 def board(self): 78 return self._board 79 80 @board.setter 81 def board(self, board): 82 self._board = board 83 self.update_special_deps() 84 85 @property 86 def kernel(self): 87 return self._kernel 88 89 @kernel.setter 90 def kernel(self, kernel): 91 self._kernel = kernel 92 self.update_special_deps() 93 94 @property 95 def code_path(self): 96 return self._code_path 97 98 @property 99 def components_path(self): 100 return self._components_path 101 102 @property 103 def platform_path(self): 104 return self._platform_path 105 106 @property 107 def kernel_path(self): 108 return self._kernel_path 109 110 @property 111 def product_path(self): 112 return self._product_path 113 114 def init(self, board=None): 115 if board is None: 116 if self.board is None: 117 raise OHOSException('no board selected') 118 else: 119 self.board = board 120 121 self.init_from_json() 122 123 def init_from_json(self): 124 self.subsystems = [] 125 subsystem_list = os.listdir(self._components_path) 126 subsystem_list.sort() 127 128 for subsystem in subsystem_list: 129 sname = subsystem.replace('.json', '') 130 spath = os.path.join(self._components_path, subsystem) 131 scontent = read_json_file(spath) 132 subsystem_cls = Subsystem(sname, scontent, spath) 133 self.subsystems.append(subsystem_cls) 134 for cname, component in subsystem_cls: 135 for fpath in component.dirs: 136 self.fields[fpath] = cname 137 self.comp_fields[cname].append(fpath) 138 139 def update_special_deps(self): 140 if self._board is None: 141 return 142 if self._kernel is None: 143 return 144 145 for subsystem_cls in self: 146 for cname, component_cls in subsystem_cls: 147 deps_board = component_cls.deps_board.get(self._board, []) 148 deps_kernel = component_cls.deps_kernel.get(self._kernel, []) 149 component_cls.deps_comp += list(set(deps_board + deps_kernel)) 150 151 def update_subsystems_product(self, nodes=None, board=None, kernel=None): 152 if board is None: 153 board = self.board 154 if kernel is None: 155 kernel = self.kernel 156 157 subsystems_list = [] 158 comp_cls_dict = {} 159 for subsystem_cls in self: 160 subsystem_dict = { 161 'subsystem': subsystem_cls.name, 162 'components': [] 163 } 164 for cname, component_cls in subsystem_cls: 165 if not component_cls.is_board_in_comp(board): 166 continue 167 if not component_cls.is_kernel_in_comp(kernel): 168 continue 169 if nodes is not None and cname not in nodes: 170 continue 171 172 comp_cls_dict[cname] = component_cls 173 component_dict = {'component': component_cls.name, 174 'features': component_cls.features} 175 subsystem_dict['components'] += [component_dict] 176 if len(subsystem_dict['components']): 177 subsystems_list.append(subsystem_dict) 178 self.update_comp_deps(comp_cls_dict) 179 180 return subsystems_list 181 182 def update_subsystems_platform(self, subsystem_list=None): 183 for subsystem_cls in self: 184 if len(subsystem_list) and \ 185 subsystem_cls.name not in subsystem_list: 186 continue 187 subsystem_cls.update_json() 188 189 @classmethod 190 def update_comp_deps(cls, comp_cls_dict): 191 for cname, comp_cls in comp_cls_dict.items(): 192 cls_deps_list = [] 193 for dep_comp_name in comp_cls.deps_comp: 194 dep_comp_cls = comp_cls_dict.get(dep_comp_name, None) 195 if dep_comp_cls is None: 196 pass 197 cls_deps_list.append(dep_comp_cls) 198 comp_cls.deps_comp = cls_deps_list 199 200 201class Subsystem(): 202 def __init__(self, subsystem_name, subsystem_json, subsystem_json_path): 203 self.name = subsystem_name 204 self.comps = {} 205 self.json_path = subsystem_json_path 206 self.json_content = subsystem_json 207 self._init_comps(subsystem_json) 208 209 def __iter__(self): 210 for cname, component_cls in self.comps.items(): 211 yield cname, component_cls 212 213 def _init_comps(self, subsystem_json): 214 component_list = subsystem_json.get('components') 215 for component_json in component_list: 216 cname = component_json.get('component') 217 self.comps[cname] = (Component(cname, component_json)) 218 219 def update_json(self): 220 component_list = self.json_content.get('components', []) 221 for index, component in enumerate(component_list): 222 cname = component.get('component', '') 223 component_cls = self.comps.get(cname, None) 224 component_list[index]['deps'] = component_cls.get_real_deps() 225 226 self.json_content['components'] = component_list 227 with open(self.json_path, 'wt', encoding='utf-8') as file: 228 json.dump(self.json_content, file, 229 ensure_ascii=False, indent=2) 230 231 232class Component(): 233 def __init__(self, component_name, component_json): 234 self.name = component_name 235 self.deps_comp = [] 236 self.deps_thirdparty = [] 237 self.copy_dirs = [] 238 self._init_comp(component_json) 239 self.deps_dict = {} 240 self.thirdparty_set = set() 241 self.deps_real_dict = {} 242 243 def _init_comp(self, component_json): 244 self.dirs = component_json.get('dirs', []) 245 self.targets = component_json.get('targets', []) 246 self.adapted_board = component_json.get('adapted_board', []) 247 self.adapted_kernel = component_json.get('adapted_kernel', []) 248 self.features = component_json.get('features', None) 249 250 deps = component_json.get('deps', {}) 251 self.deps_comp += deps.get('components', []) 252 self.deps_thirdparty += deps.get('third_party', []) 253 self.deps_board = deps.get('board_special', {}) 254 self.deps_kernel = deps.get('kernel_special', {}) 255 256 def is_dir_in_comp(self, path): 257 if path in self.dirs: 258 return True 259 return False 260 261 def is_board_in_comp(self, board): 262 if not len(self.adapted_board): 263 return True 264 if board in self.adapted_board: 265 return True 266 267 return False 268 269 def is_kernel_in_comp(self, kernel): 270 if not len(self.adapted_kernel): 271 return True 272 if kernel in self.adapted_kernel: 273 return True 274 275 return False 276 277 def remove_copy_dirs(self, work_path): 278 os.chdir(work_path) 279 for copy_dir in self.copy_dirs: 280 if os.path.exists(copy_dir): 281 shutil.rmtree(copy_dir) 282 283 def get_deps_ready(self, work_path, root_path): 284 queue = Queue() 285 visited = set() 286 deps_cls_list = [self] 287 tree = [{'name': self.name, 'children': []}] 288 now_tree = tree[0]["children"] 289 queue.put((self, now_tree)) 290 291 while not queue.empty(): 292 cur_comp, now_tree = queue.get() 293 for i, dep_comp_cls in enumerate(cur_comp.deps_comp): 294 if dep_comp_cls in visited: 295 continue 296 if dep_comp_cls is None: 297 hb_warning('{} has NoneType dep'.format(cur_comp.name)) 298 continue 299 deps_cls_list.append(dep_comp_cls) 300 now_tree.append({'name': dep_comp_cls.name, 'children': []}) 301 queue.put((dep_comp_cls, now_tree[-1]['children'])) 302 visited.add(dep_comp_cls) 303 304 if not hasattr(self, 'copy_dirs'): 305 self.copy_dirs = [] 306 for comp_cls in deps_cls_list: 307 for path in comp_cls.dirs: 308 src_path = os.path.join(root_path, path) 309 des_path = os.path.join(work_path, path.split("../")[-1]) 310 try: 311 shutil.copytree(src_path, 312 des_path, 313 symlinks=False, 314 ignore=shutil.ignore_patterns('.git', 315 '.repo')) 316 except OSError: 317 pass 318 else: 319 self.copy_dirs.append(des_path) 320 321 return tree 322 323 @classmethod 324 def gn_desc(cls, out_file, target, *args): 325 cmd = ['./prebuilts/build-tools/linux-x86/bin/gn', 326 '--root=.', 327 '--dotfile=build/lite/.gn', 328 'desc', 329 out_file, 330 target, *args] 331 332 cwd = os.path.abspath(os.path.join(out_file, os.pardir, 333 os.pardir, os.pardir)) 334 ret = check_output(cmd, cwd=cwd).decode('utf-8') 335 return ret 336 337 def get_component_deps(self, out_file, now_target, comp_fields): 338 include_list = [] 339 include_dirs_list = [now_target] 340 deps_list = [] 341 342 ret = self.gn_desc(out_file, now_target, 'deps', '--tree') 343 hb_info(f'target: {now_target} cmd: deps --tree') 344 hb_info(ret) 345 346 ret_content = [line for line in ret.split('\n') 347 if len(line) and '\n' not in line] 348 349 index = 0 350 while index < len(ret_content): # get deps of the now_target 351 cur_line = ret_content[index] 352 if '...' in cur_line: # cur_line already exists. 353 index += 1 354 continue 355 space_now = cur_line.count(' ') 356 for path in comp_fields[self.name]: 357 # cur_line is part of current comp, 358 # needs to be further processed. 359 if path in cur_line: 360 index += 1 361 include_dirs_list.append(cur_line.replace(' ', '')) 362 break 363 else: 364 deps_list.append(cur_line.replace(' ', '').replace('//', '')) 365 index += 1 366 while index < len(ret_content) and \ 367 ret_content[index].count(' ') > space_now: 368 index += 1 369 for target in include_dirs_list: 370 try: 371 ret = self.gn_desc(out_file, target, 'include_dirs') 372 hb_info(f'target: {now_target} cmd: include_dirs') 373 hb_info(ret) 374 include_list += [include.replace('//', '') 375 for include in ret.split('\n') 376 if '\n' not in include and len(include)] 377 except CalledProcessError: 378 pass 379 return deps_list, include_list 380 381 def get_deps(self, board, kernel, out_file, comp_fields, fields): 382 deps_list = [] 383 include_list = [] 384 for now_target in self.targets: 385 now_deps, now_include = self.get_component_deps(out_file, 386 now_target, 387 comp_fields) 388 deps_list += now_deps 389 include_list += now_include 390 391 deps_ori = list(set(deps_list) | set(include_list)) 392 393 comp_deps_set = set() 394 for dep in deps_ori: 395 if 'prebuilts' in dep or 'build/lite' in dep: 396 continue 397 for path in comp_fields[self.name]: 398 if dep in path or path in dep: 399 break 400 else: 401 for path, cname in fields.items(): 402 if check_path(dep, path): 403 if 'hdf' in cname: # special processing for hdf 404 cname = 'hdf_{}_{}'.format(board, kernel) 405 if cname != self.name: 406 comp_deps_set.add(cname) 407 break 408 else: 409 # special processing for third_party 410 if dep.startswith('third_party'): 411 strlist = dep.split('/')[1].split(':')[0].lower() 412 self.thirdparty_set.add(strlist.replace('-', '_')) 413 else: 414 comp_deps_set.add(dep) 415 self.deps_dict[(board, kernel)] = comp_deps_set 416 417 def get_real_deps(self): 418 components_set = set() 419 board_common = None 420 kernel_common = None 421 board_special = {} 422 kernel_special = {} 423 424 if len(self.deps_dict) == 1: 425 _, components_set = self.deps_dict.popitem() 426 elif len(self.deps_dict) > 1: 427 for _, now_deps in self.deps_dict.items(): 428 board_common = now_deps if board_common is None\ 429 else board_common & now_deps 430 for _, now_deps in self.deps_dict.items(): 431 kernel_common = now_deps if kernel_common is None\ 432 else kernel_common & now_deps 433 434 for (board, kernel), now_deps in self.deps_dict.items(): 435 board_special[board] = now_deps - board_common 436 kernel_special[kernel] = now_deps - kernel_common 437 438 special_common = None 439 for _, now_deps in board_special.items(): 440 special_common = now_deps if special_common is None\ 441 else special_common & now_deps 442 for board, now_deps in board_special.items(): 443 board_special[board] -= special_common 444 445 components_set = components_set.union(board_common, 446 kernel_common, 447 special_common) 448 449 special_common = None 450 for _, now_deps in kernel_special.items(): 451 special_common = now_deps if special_common is None\ 452 else special_common & now_deps 453 for kernel, now_deps in kernel_special.items(): 454 kernel_special[kernel] -= special_common 455 456 components_set = components_set.union(special_common) 457 458 return self.get_deps_sort(components_set, 459 kernel_special, 460 board_special) 461 462 def get_deps_sort(self, components_set, kernel_special, board_special): 463 thirdparty_list = list(self.thirdparty_set) 464 thirdparty_list.sort() 465 components_list = list(components_set) 466 components_list.sort() 467 468 kernel_list = [kernel for kernel in kernel_special.keys()] 469 kernel_list.sort() 470 for kernel in kernel_list: 471 kernel_special[kernel] = list(kernel_special[kernel]) 472 kernel_special[kernel].sort() 473 474 board_list = [board for board in board_special.keys()] 475 board_list.sort() 476 for board in board_list: 477 board_special[board] = list(board_special[board]) 478 board_special[board].sort() 479 480 self.deps_real_dict = { 481 'third_party': thirdparty_list, 482 'kernel_special': {kernel: kernel_special[kernel] 483 for kernel in kernel_list 484 if len(kernel_special[kernel])}, 485 'board_special': {board: board_special[board] 486 for board in board_list 487 if len(board_special[board])}, 488 'components': components_list 489 } 490 491 return self.deps_real_dict 492