1#!/usr/bin/env python3 2 3from __future__ import print_function 4 5import argparse 6import codecs 7import collections 8import copy 9import csv 10import io 11import itertools 12import json 13import os 14import posixpath 15import re 16import shutil 17import stat 18import struct 19import subprocess 20import sys 21import zipfile 22 23 24#------------------------------------------------------------------------------ 25# Python 2 and 3 Compatibility Layer 26#------------------------------------------------------------------------------ 27 28if sys.version_info >= (3, 0): 29 from os import makedirs 30 from mmap import ACCESS_READ, mmap 31 32 def get_py3_bytes(buf): 33 return buf 34 35 create_chr = chr 36 enumerate_bytes = enumerate 37else: 38 from mmap import ACCESS_READ, mmap 39 40 def makedirs(path, exist_ok): 41 if exist_ok and os.path.isdir(path): 42 return 43 os.makedirs(path) 44 45 class mmap(mmap): 46 def __enter__(self): 47 return self 48 49 def __exit__(self, exc, value, tb): 50 self.close() 51 52 def __getitem__(self, key): 53 res = super(mmap, self).__getitem__(key) 54 if isinstance(key, int): 55 return ord(res) 56 return res 57 58 class Py3Bytes(bytes): 59 def __getitem__(self, key): 60 res = super(Py3Bytes, self).__getitem__(key) 61 if isinstance(key, int): 62 return ord(res) 63 return Py3Bytes(res) 64 65 def get_py3_bytes(buf): 66 return Py3Bytes(buf) 67 68 create_chr = unichr 69 70 def enumerate_bytes(iterable): 71 for i, byte in enumerate(iterable): 72 yield (i, ord(byte)) 73 74 FileNotFoundError = EnvironmentError 75 76try: 77 from sys import intern 78except ImportError: 79 pass 80 81try: 82 from tempfile import TemporaryDirectory 83except ImportError: 84 import shutil 85 import tempfile 86 87 class TemporaryDirectory(object): 88 def __init__(self, suffix='', prefix='tmp', dir=None): 89 # pylint: disable=redefined-builtin 90 self.name = tempfile.mkdtemp(suffix, prefix, dir) 91 92 def __del__(self): 93 self.cleanup() 94 95 def __enter__(self): 96 return self.name 97 98 def __exit__(self, exc, value, tb): 99 self.cleanup() 100 101 def cleanup(self): 102 if self.name: 103 shutil.rmtree(self.name) 104 self.name = None 105 106try: 107 from os import scandir 108except ImportError: 109 import stat 110 import os 111 112 class DirEntry(object): 113 def __init__(self, name, path): 114 self.name = name 115 self.path = path 116 self._stat = None 117 self._lstat = None 118 119 @staticmethod 120 def _stat_impl(path, follow_symlinks): 121 return os.stat(path) if follow_symlinks else os.lstat(path) 122 123 def stat(self, follow_symlinks=True): 124 attr = '_stat' if follow_symlinks else '_lstat' 125 stat_res = getattr(self, attr) 126 if stat_res is None: 127 stat_res = self._stat_impl(self.path, follow_symlinks) 128 setattr(self, attr, stat_res) 129 return stat_res 130 131 def is_dir(self, follow_symlinks=True): 132 try: 133 return stat.S_ISDIR(self.stat(follow_symlinks).st_mode) 134 except EnvironmentError: 135 return False 136 137 def is_file(self, follow_symlinks=True): 138 try: 139 return stat.S_ISREG(self.stat(follow_symlinks).st_mode) 140 except EnvironmentError: 141 return False 142 143 def is_symlink(self): 144 return stat.S_ISLNK(self.stat(follow_symlinks=False).st_mode) 145 146 def scandir(path): 147 for name in os.listdir(path): 148 yield DirEntry(name, os.path.join(path, name)) 149 150 151#------------------------------------------------------------------------------ 152# Print Function 153#------------------------------------------------------------------------------ 154 155def print_sb(*args, **kwargs): 156 """A print function that supports both str and bytes.""" 157 sep = kwargs.get('sep', ' ') 158 end = kwargs.get('end', '\n') 159 out_file = kwargs.get('file', sys.stdout) 160 for i, arg in enumerate(args): 161 if i > 0: 162 out_file.write(sep) 163 if isinstance(arg, str): 164 out_file.write(arg) 165 elif isinstance(arg, bytes): 166 out_file.flush() 167 out_file.buffer.write(arg) 168 out_file.flush() 169 else: 170 out_file.write(str(arg)) 171 out_file.write(end) 172 173 174#------------------------------------------------------------------------------ 175# Modified UTF-8 Encoder and Decoder 176#------------------------------------------------------------------------------ 177 178class UnicodeSurrogateDecodeError(UnicodeDecodeError): 179 pass 180 181 182def encode_mutf8(input, errors='strict'): 183 i = 0 184 res = io.BytesIO() 185 186 for i, char in enumerate(input): 187 code = ord(char) 188 if code == 0x00: 189 res.write(b'\xc0\x80') 190 elif code < 0x80: 191 res.write(bytearray((code,))) 192 elif code < 0x800: 193 res.write(bytearray((0xc0 | (code >> 6), 0x80 | (code & 0x3f)))) 194 elif code < 0x10000: 195 res.write(bytearray((0xe0 | (code >> 12), 196 0x80 | ((code >> 6) & 0x3f), 197 0x80 | (code & 0x3f)))) 198 elif code < 0x110000: 199 code -= 0x10000 200 code_hi = 0xd800 + (code >> 10) 201 code_lo = 0xdc00 + (code & 0x3ff) 202 res.write(bytearray((0xe0 | (code_hi >> 12), 203 0x80 | ((code_hi >> 6) & 0x3f), 204 0x80 | (code_hi & 0x3f), 205 0xe0 | (code_lo >> 12), 206 0x80 | ((code_lo >> 6) & 0x3f), 207 0x80 | (code_lo & 0x3f)))) 208 else: 209 raise UnicodeEncodeError('mutf-8', input, i, i + 1, 210 'illegal code point') 211 212 return (res.getvalue(), i) 213 214 215def decode_mutf8(input, errors='strict'): 216 res = io.StringIO() 217 218 num_next = 0 219 220 i = 0 221 code = 0 222 start = 0 223 224 code_surrogate = None 225 start_surrogate = None 226 227 def raise_error(start, reason): 228 raise UnicodeDecodeError('mutf-8', input, start, i + 1, reason) 229 230 def raise_surrogate_error(start, reason): 231 raise UnicodeSurrogateDecodeError( 232 'mutf-8', input, start, i + 1, reason) 233 234 for i, byte in enumerate_bytes(input): 235 if (byte & 0x80) == 0x00: 236 if num_next > 0: 237 raise_error(start, 'invalid continuation byte') 238 num_next = 0 239 code = byte 240 start = i 241 elif (byte & 0xc0) == 0x80: 242 if num_next < 1: 243 raise_error(start, 'invalid start byte') 244 num_next -= 1 245 code = (code << 6) | (byte & 0x3f) 246 elif (byte & 0xe0) == 0xc0: 247 if num_next > 0: 248 raise_error(start, 'invalid continuation byte') 249 num_next = 1 250 code = byte & 0x1f 251 start = i 252 elif (byte & 0xf0) == 0xe0: 253 if num_next > 0: 254 raise_error(start, 'invalid continuation byte') 255 num_next = 2 256 code = byte & 0x0f 257 start = i 258 else: 259 raise_error(i, 'invalid start byte') 260 261 if num_next == 0: 262 if 0xd800 <= code <= 0xdbff: # High surrogate 263 if code_surrogate is not None: 264 raise_surrogate_error( 265 start_surrogate, 'invalid high surrogate') 266 code_surrogate = code 267 start_surrogate = start 268 continue 269 270 if 0xdc00 <= code <= 0xdfff: # Low surrogate 271 if code_surrogate is None: 272 raise_surrogate_error(start, 'invalid low surrogate') 273 code = (((code_surrogate & 0x3f) << 10) | 274 (code & 0x3f) + 0x10000) 275 code_surrogate = None 276 start_surrogate = None 277 elif code_surrogate is not None: 278 if errors == 'ignore': 279 res.write(create_chr(code_surrogate)) 280 code_surrogate = None 281 start_surrogate = None 282 else: 283 raise_surrogate_error(start_surrogate, 'illegal surrogate') 284 285 res.write(create_chr(code)) 286 287 # Check the unexpected end of input 288 if num_next > 0: 289 raise_error(start, 'unexpected end') 290 if code_surrogate is not None: 291 raise_surrogate_error(start_surrogate, 'unexpected end') 292 293 return (res.getvalue(), i) 294 295 296def probe_mutf8(name): 297 if name == 'mutf-8': 298 return codecs.CodecInfo(encode_mutf8, decode_mutf8) 299 return None 300 301codecs.register(probe_mutf8) 302 303 304#------------------------------------------------------------------------------ 305# Collections 306#------------------------------------------------------------------------------ 307 308def defaultnamedtuple(typename, field_names, default): 309 """Create a namedtuple type with default values. 310 311 This function creates a namedtuple type which will fill in default value 312 when actual arguments to the constructor were omitted. 313 314 >>> Point = defaultnamedtuple('Point', ['x', 'y'], 0) 315 >>> Point() 316 Point(x=0, y=0) 317 >>> Point(1) 318 Point(x=1, y=0) 319 >>> Point(1, 2) 320 Point(x=1, y=2) 321 >>> Point(x=1, y=2) 322 Point(x=1, y=2) 323 >>> Point(y=2, x=1) 324 Point(x=1, y=2) 325 326 >>> PermSet = defaultnamedtuple('PermSet', 'allowed disallowed', set()) 327 >>> s = PermSet() 328 >>> s 329 PermSet(allowed=set(), disallowed=set()) 330 >>> s.allowed is not s.disallowed 331 True 332 >>> PermSet({1}) 333 PermSet(allowed={1}, disallowed=set()) 334 >>> PermSet({1}, {2}) 335 PermSet(allowed={1}, disallowed={2}) 336 """ 337 338 if isinstance(field_names, str): 339 field_names = field_names.replace(',', ' ').split() 340 field_names = list(map(str, field_names)) 341 num_fields = len(field_names) 342 343 base_cls = collections.namedtuple(typename, field_names) 344 def __new__(cls, *args, **kwargs): 345 args = list(args) 346 for i in range(len(args), num_fields): 347 arg = kwargs.get(field_names[i]) 348 if arg: 349 args.append(arg) 350 else: 351 args.append(copy.copy(default)) 352 return base_cls.__new__(cls, *args) 353 return type(typename, (base_cls,), {'__new__': __new__}) 354 355 356def create_struct(name, fields): 357 """Create a namedtuple with unpack_from() function. 358 >>> Point = create_struct('Point', [('x', 'I'), ('y', 'I')]) 359 >>> pt = Point.unpack_from(b'\\x00\\x00\\x00\\x00\\x01\\x00\\x00\\x00', 0) 360 >>> pt.x 361 0 362 >>> pt.y 363 1 364 """ 365 field_names = [name for name, ty in fields] 366 cls = collections.namedtuple(name, field_names) 367 cls.struct_fmt = ''.join(ty for name, ty in fields) 368 cls.struct_size = struct.calcsize(cls.struct_fmt) 369 def unpack_from(cls, buf, offset=0): 370 unpacked = struct.unpack_from(cls.struct_fmt, buf, offset) 371 return cls.__new__(cls, *unpacked) 372 cls.unpack_from = classmethod(unpack_from) 373 return cls 374 375 376#------------------------------------------------------------------------------ 377# ELF Parser 378#------------------------------------------------------------------------------ 379 380Elf_Hdr = collections.namedtuple( 381 'Elf_Hdr', 382 'ei_class ei_data ei_version ei_osabi e_type e_machine e_version ' 383 'e_entry e_phoff e_shoff e_flags e_ehsize e_phentsize e_phnum ' 384 'e_shentsize e_shnum e_shstridx') 385 386 387Elf_Shdr = collections.namedtuple( 388 'Elf_Shdr', 389 'sh_name sh_type sh_flags sh_addr sh_offset sh_size sh_link sh_info ' 390 'sh_addralign sh_entsize') 391 392 393Elf_Phdr = collections.namedtuple( 394 'Elf_Phdr', 395 'p_type p_offset p_vaddr p_paddr p_filesz p_memsz p_flags p_align') 396 397 398Elf_Dyn = collections.namedtuple('Elf_Dyn', 'd_tag d_val') 399 400 401class Elf_Sym(collections.namedtuple( 402 'ELF_Sym', 'st_name st_value st_size st_info st_other st_shndx')): 403 404 STB_LOCAL = 0 405 STB_GLOBAL = 1 406 STB_WEAK = 2 407 408 SHN_UNDEF = 0 409 410 411 @property 412 def st_bind(self): 413 return self.st_info >> 4 414 415 416 @property 417 def is_local(self): 418 return self.st_bind == Elf_Sym.STB_LOCAL 419 420 421 @property 422 def is_global(self): 423 return self.st_bind == Elf_Sym.STB_GLOBAL 424 425 426 @property 427 def is_weak(self): 428 return self.st_bind == Elf_Sym.STB_WEAK 429 430 431 @property 432 def is_undef(self): 433 return self.st_shndx == Elf_Sym.SHN_UNDEF 434 435 436class ELFError(ValueError): 437 pass 438 439 440class ELF(object): 441 # ELF file format constants. 442 ELF_MAGIC = b'\x7fELF' 443 444 EI_CLASS = 4 445 EI_DATA = 5 446 447 ELFCLASSNONE = 0 448 ELFCLASS32 = 1 449 ELFCLASS64 = 2 450 451 ELFDATANONE = 0 452 ELFDATA2LSB = 1 453 ELFDATA2MSB = 2 454 455 PT_LOAD = 1 456 457 PF_X = 1 458 PF_W = 2 459 PF_R = 4 460 461 DT_NEEDED = 1 462 DT_RPATH = 15 463 DT_RUNPATH = 29 464 465 _ELF_CLASS_NAMES = { 466 ELFCLASS32: '32', 467 ELFCLASS64: '64', 468 } 469 470 _ELF_DATA_NAMES = { 471 ELFDATA2LSB: 'Little-Endian', 472 ELFDATA2MSB: 'Big-Endian', 473 } 474 475 EM_NONE = 0 476 EM_386 = 3 477 EM_MIPS = 8 478 EM_ARM = 40 479 EM_X86_64 = 62 480 EM_AARCH64 = 183 481 482 483 def _create_elf_machines(d): 484 elf_machine_ids = {} 485 for key, value in d.items(): 486 if key.startswith('EM_'): 487 elf_machine_ids[value] = key 488 return elf_machine_ids 489 490 ELF_MACHINES = _create_elf_machines(locals()) 491 492 del _create_elf_machines 493 494 495 @staticmethod 496 def _dict_find_key_by_value(d, dst): 497 for key, value in d.items(): 498 if value == dst: 499 return key 500 raise KeyError(dst) 501 502 503 @staticmethod 504 def get_ei_class_from_name(name): 505 return ELF._dict_find_key_by_value(ELF._ELF_CLASS_NAMES, name) 506 507 508 @staticmethod 509 def get_ei_data_from_name(name): 510 return ELF._dict_find_key_by_value(ELF._ELF_DATA_NAMES, name) 511 512 513 @staticmethod 514 def get_e_machine_from_name(name): 515 return ELF._dict_find_key_by_value(ELF.ELF_MACHINES, name) 516 517 518 __slots__ = ('ei_class', 'ei_data', 'e_machine', 'dt_rpath', 'dt_runpath', 519 'dt_needed', 'exported_symbols', 'imported_symbols', 520 'file_size', 'ro_seg_file_size', 'ro_seg_mem_size', 521 'rw_seg_file_size', 'rw_seg_mem_size',) 522 523 524 def __init__(self, ei_class=ELFCLASSNONE, ei_data=ELFDATANONE, e_machine=0, 525 dt_rpath=None, dt_runpath=None, dt_needed=None, 526 exported_symbols=None, imported_symbols=None, 527 file_size=0, ro_seg_file_size=0, ro_seg_mem_size=0, 528 rw_seg_file_size=0, rw_seg_mem_size=0): 529 self.ei_class = ei_class 530 self.ei_data = ei_data 531 self.e_machine = e_machine 532 self.dt_rpath = dt_rpath if dt_rpath is not None else [] 533 self.dt_runpath = dt_runpath if dt_runpath is not None else [] 534 self.dt_needed = dt_needed if dt_needed is not None else [] 535 self.exported_symbols = \ 536 exported_symbols if exported_symbols is not None else set() 537 self.imported_symbols = \ 538 imported_symbols if imported_symbols is not None else set() 539 self.file_size = file_size 540 self.ro_seg_file_size = ro_seg_file_size 541 self.ro_seg_mem_size = ro_seg_mem_size 542 self.rw_seg_file_size = rw_seg_file_size 543 self.rw_seg_mem_size = rw_seg_mem_size 544 545 546 def __repr__(self): 547 args = (a + '=' + repr(getattr(self, a)) for a in self.__slots__) 548 return 'ELF(' + ', '.join(args) + ')' 549 550 551 def __eq__(self, rhs): 552 return all(getattr(self, a) == getattr(rhs, a) for a in self.__slots__) 553 554 555 @property 556 def elf_class_name(self): 557 return self._ELF_CLASS_NAMES.get(self.ei_class, 'None') 558 559 560 @property 561 def elf_data_name(self): 562 return self._ELF_DATA_NAMES.get(self.ei_data, 'None') 563 564 565 @property 566 def elf_machine_name(self): 567 return self.ELF_MACHINES.get(self.e_machine, str(self.e_machine)) 568 569 570 @property 571 def is_32bit(self): 572 return self.ei_class == ELF.ELFCLASS32 573 574 575 @property 576 def is_64bit(self): 577 return self.ei_class == ELF.ELFCLASS64 578 579 580 @property 581 def sorted_exported_symbols(self): 582 return sorted(list(self.exported_symbols)) 583 584 585 @property 586 def sorted_imported_symbols(self): 587 return sorted(list(self.imported_symbols)) 588 589 590 def dump(self, file=None): 591 """Print parsed ELF information to the file""" 592 file = file if file is not None else sys.stdout 593 594 print('EI_CLASS\t' + self.elf_class_name, file=file) 595 print('EI_DATA\t\t' + self.elf_data_name, file=file) 596 print('E_MACHINE\t' + self.elf_machine_name, file=file) 597 print('FILE_SIZE\t' + str(self.file_size), file=file) 598 print('RO_SEG_FILE_SIZE\t' + str(self.ro_seg_file_size), file=file) 599 print('RO_SEG_MEM_SIZE\t' + str(self.ro_seg_mem_size), file=file) 600 print('RW_SEG_FILE_SIZE\t' + str(self.rw_seg_file_size), file=file) 601 print('RW_SEG_MEM_SIZE\t' + str(self.rw_seg_mem_size), file=file) 602 for dt_rpath in self.dt_rpath: 603 print('DT_RPATH\t' + dt_rpath, file=file) 604 for dt_runpath in self.dt_runpath: 605 print('DT_RUNPATH\t' + dt_runpath, file=file) 606 for dt_needed in self.dt_needed: 607 print('DT_NEEDED\t' + dt_needed, file=file) 608 for symbol in self.sorted_exported_symbols: 609 print('EXP_SYMBOL\t' + symbol, file=file) 610 for symbol in self.sorted_imported_symbols: 611 print('IMP_SYMBOL\t' + symbol, file=file) 612 613 614 # Extract zero-terminated buffer slice. 615 def _extract_zero_terminated_buf_slice(self, buf, offset): 616 """Extract a zero-terminated buffer slice from the given offset""" 617 end = buf.find(b'\0', offset) 618 if end == -1: 619 return buf[offset:] 620 return buf[offset:end] 621 622 623 # Extract c-style interned string from the buffer. 624 if sys.version_info >= (3, 0): 625 def _extract_zero_terminated_str(self, buf, offset): 626 """Extract a c-style string from the given buffer and offset""" 627 buf_slice = self._extract_zero_terminated_buf_slice(buf, offset) 628 return intern(buf_slice.decode('utf-8')) 629 else: 630 def _extract_zero_terminated_str(self, buf, offset): 631 """Extract a c-style string from the given buffer and offset""" 632 return intern(self._extract_zero_terminated_buf_slice(buf, offset)) 633 634 635 def _parse_from_buf_internal(self, buf): 636 """Parse ELF image resides in the buffer""" 637 638 # Check ELF ident. 639 if len(buf) < 8: 640 raise ELFError('bad ident') 641 642 if buf[0:4] != ELF.ELF_MAGIC: 643 raise ELFError('bad magic') 644 645 self.ei_class = buf[ELF.EI_CLASS] 646 if self.ei_class not in (ELF.ELFCLASS32, ELF.ELFCLASS64): 647 raise ELFError('unknown word size') 648 649 self.ei_data = buf[ELF.EI_DATA] 650 if self.ei_data not in (ELF.ELFDATA2LSB, ELF.ELFDATA2MSB): 651 raise ELFError('unknown endianness') 652 653 self.file_size = len(buf) 654 655 # ELF structure definitions. 656 endian_fmt = '<' if self.ei_data == ELF.ELFDATA2LSB else '>' 657 658 if self.is_32bit: 659 elf_hdr_fmt = endian_fmt + '4x4B8xHHLLLLLHHHHHH' 660 elf_shdr_fmt = endian_fmt + 'LLLLLLLLLL' 661 elf_phdr_fmt = endian_fmt + 'LLLLLLLL' 662 elf_dyn_fmt = endian_fmt + 'lL' 663 elf_sym_fmt = endian_fmt + 'LLLBBH' 664 else: 665 elf_hdr_fmt = endian_fmt + '4x4B8xHHLQQQLHHHHHH' 666 elf_shdr_fmt = endian_fmt + 'LLQQQQLLQQ' 667 elf_phdr_fmt = endian_fmt + 'LLQQQQQQ' 668 elf_dyn_fmt = endian_fmt + 'QQ' 669 elf_sym_fmt = endian_fmt + 'LBBHQQ' 670 671 def parse_struct(cls, fmt, offset, error_msg): 672 try: 673 return cls._make(struct.unpack_from(fmt, buf, offset)) 674 except struct.error: 675 raise ELFError(error_msg) 676 677 def parse_elf_hdr(offset): 678 return parse_struct(Elf_Hdr, elf_hdr_fmt, offset, 'bad elf header') 679 680 def parse_elf_shdr(offset): 681 return parse_struct(Elf_Shdr, elf_shdr_fmt, offset, 682 'bad section header') 683 684 if self.is_32bit: 685 def parse_elf_phdr(offset): 686 return parse_struct(Elf_Phdr, elf_phdr_fmt, offset, 687 'bad program header') 688 else: 689 def parse_elf_phdr(offset): 690 try: 691 p = struct.unpack_from(elf_phdr_fmt, buf, offset) 692 return Elf_Phdr(p[0], p[2], p[3], p[4], p[5], p[6], p[1], 693 p[7]) 694 except struct.error: 695 raise ELFError('bad program header') 696 697 def parse_elf_dyn(offset): 698 return parse_struct(Elf_Dyn, elf_dyn_fmt, offset, 699 'bad .dynamic entry') 700 701 if self.is_32bit: 702 def parse_elf_sym(offset): 703 return parse_struct( 704 Elf_Sym, elf_sym_fmt, offset, 'bad elf sym') 705 else: 706 def parse_elf_sym(offset): 707 try: 708 p = struct.unpack_from(elf_sym_fmt, buf, offset) 709 return Elf_Sym(p[0], p[4], p[5], p[1], p[2], p[3]) 710 except struct.error: 711 raise ELFError('bad elf sym') 712 713 def extract_str(offset): 714 return self._extract_zero_terminated_str(buf, offset) 715 716 # Parse ELF header. 717 header = parse_elf_hdr(0) 718 self.e_machine = header.e_machine 719 720 # Parse ELF program header and calculate segment size. 721 if header.e_phentsize == 0: 722 raise ELFError('no program header') 723 724 ro_seg_file_size = 0 725 ro_seg_mem_size = 0 726 rw_seg_file_size = 0 727 rw_seg_mem_size = 0 728 729 assert struct.calcsize(elf_phdr_fmt) == header.e_phentsize 730 seg_end = header.e_phoff + header.e_phnum * header.e_phentsize 731 for phdr_off in range(header.e_phoff, seg_end, header.e_phentsize): 732 phdr = parse_elf_phdr(phdr_off) 733 if phdr.p_type != ELF.PT_LOAD: 734 continue 735 if phdr.p_flags & ELF.PF_W: 736 rw_seg_file_size += phdr.p_filesz 737 rw_seg_mem_size += phdr.p_memsz 738 else: 739 ro_seg_file_size += phdr.p_filesz 740 ro_seg_mem_size += phdr.p_memsz 741 742 self.ro_seg_file_size = ro_seg_file_size 743 self.ro_seg_mem_size = ro_seg_mem_size 744 self.rw_seg_file_size = rw_seg_file_size 745 self.rw_seg_mem_size = rw_seg_mem_size 746 747 # Check section header size. 748 if header.e_shentsize == 0: 749 raise ELFError('no section header') 750 751 # Find .shstrtab section. 752 shstrtab_shdr_off = \ 753 header.e_shoff + header.e_shstridx * header.e_shentsize 754 shstrtab_shdr = parse_elf_shdr(shstrtab_shdr_off) 755 shstrtab_off = shstrtab_shdr.sh_offset 756 757 # Parse ELF section header. 758 sections = dict() 759 header_end = header.e_shoff + header.e_shnum * header.e_shentsize 760 for shdr_off in range(header.e_shoff, header_end, header.e_shentsize): 761 shdr = parse_elf_shdr(shdr_off) 762 name = extract_str(shstrtab_off + shdr.sh_name) 763 sections[name] = shdr 764 765 # Find .dynamic and .dynstr section header. 766 dynamic_shdr = sections.get('.dynamic') 767 if not dynamic_shdr: 768 raise ELFError('no .dynamic section') 769 770 dynstr_shdr = sections.get('.dynstr') 771 if not dynstr_shdr: 772 raise ELFError('no .dynstr section') 773 774 dynamic_off = dynamic_shdr.sh_offset 775 dynstr_off = dynstr_shdr.sh_offset 776 777 # Parse entries in .dynamic section. 778 assert struct.calcsize(elf_dyn_fmt) == dynamic_shdr.sh_entsize 779 dynamic_end = dynamic_off + dynamic_shdr.sh_size 780 for ent_off in range( 781 dynamic_off, dynamic_end, dynamic_shdr.sh_entsize): 782 ent = parse_elf_dyn(ent_off) 783 if ent.d_tag == ELF.DT_NEEDED: 784 self.dt_needed.append(extract_str(dynstr_off + ent.d_val)) 785 elif ent.d_tag == ELF.DT_RPATH: 786 self.dt_rpath.extend( 787 extract_str(dynstr_off + ent.d_val).split(':')) 788 elif ent.d_tag == ELF.DT_RUNPATH: 789 self.dt_runpath.extend( 790 extract_str(dynstr_off + ent.d_val).split(':')) 791 792 # Parse exported symbols in .dynsym section. 793 dynsym_shdr = sections.get('.dynsym') 794 if dynsym_shdr: 795 exp_symbols = self.exported_symbols 796 imp_symbols = self.imported_symbols 797 798 dynsym_off = dynsym_shdr.sh_offset 799 dynsym_end = dynsym_off + dynsym_shdr.sh_size 800 dynsym_entsize = dynsym_shdr.sh_entsize 801 802 # Skip first symbol entry (null symbol). 803 dynsym_off += dynsym_entsize 804 805 for ent_off in range(dynsym_off, dynsym_end, dynsym_entsize): 806 ent = parse_elf_sym(ent_off) 807 symbol_name = extract_str(dynstr_off + ent.st_name) 808 if ent.is_undef: 809 imp_symbols.add(symbol_name) 810 elif not ent.is_local: 811 exp_symbols.add(symbol_name) 812 813 814 def _parse_from_buf(self, buf): 815 """Parse ELF image resides in the buffer""" 816 try: 817 self._parse_from_buf_internal(buf) 818 except IndexError: 819 raise ELFError('bad offset') 820 821 822 def _parse_from_file(self, path): 823 """Parse ELF image from the file path""" 824 with open(path, 'rb') as f: 825 st = os.fstat(f.fileno()) 826 if not st.st_size: 827 raise ELFError('empty file') 828 with mmap(f.fileno(), st.st_size, access=ACCESS_READ) as image: 829 self._parse_from_buf(image) 830 831 832 def _parse_from_dump_lines(self, path, lines): 833 patt = re.compile('^([A-Za-z_]+)\t+(.*)$') 834 for line_no, line in enumerate(lines): 835 match = patt.match(line) 836 if not match: 837 print('error: {}: {}: Failed to parse' 838 .format(path, line_no + 1), file=sys.stderr) 839 continue 840 key = match.group(1) 841 value = match.group(2) 842 843 if key == 'EI_CLASS': 844 self.ei_class = ELF.get_ei_class_from_name(value) 845 elif key == 'EI_DATA': 846 self.ei_data = ELF.get_ei_data_from_name(value) 847 elif key == 'E_MACHINE': 848 self.e_machine = ELF.get_e_machine_from_name(value) 849 elif key == 'FILE_SIZE': 850 self.file_size = int(value) 851 elif key == 'RO_SEG_FILE_SIZE': 852 self.ro_seg_file_size = int(value) 853 elif key == 'RO_SEG_MEM_SIZE': 854 self.ro_seg_mem_size = int(value) 855 elif key == 'RW_SEG_FILE_SIZE': 856 self.rw_seg_file_size = int(value) 857 elif key == 'RW_SEG_MEM_SIZE': 858 self.rw_seg_mem_size = int(value) 859 elif key == 'DT_RPATH': 860 self.dt_rpath.append(intern(value)) 861 elif key == 'DT_RUNPATH': 862 self.dt_runpath.append(intern(value)) 863 elif key == 'DT_NEEDED': 864 self.dt_needed.append(intern(value)) 865 elif key == 'EXP_SYMBOL': 866 self.exported_symbols.add(intern(value)) 867 elif key == 'IMP_SYMBOL': 868 self.imported_symbols.add(intern(value)) 869 else: 870 print('error: {}: {}: unknown tag name: {}' 871 .format(path, line_no + 1, key), file=sys.stderr) 872 873 874 def _parse_from_dump_file(self, path): 875 """Load information from ELF dump file.""" 876 with open(path, 'r') as f: 877 self._parse_from_dump_lines(path, f) 878 879 880 def _parse_from_dump_buf(self, buf): 881 """Load information from ELF dump buffer.""" 882 self._parse_from_dump_lines('<str:0x{:x}>'.format(id(buf)), 883 buf.splitlines()) 884 885 886 @staticmethod 887 def load(path): 888 """Create an ELF instance from the file path""" 889 elf = ELF() 890 elf._parse_from_file(path) 891 return elf 892 893 894 @staticmethod 895 def loads(buf): 896 """Create an ELF instance from the buffer""" 897 elf = ELF() 898 elf._parse_from_buf(buf) 899 return elf 900 901 902 @staticmethod 903 def load_dump(path): 904 """Create an ELF instance from a dump file path""" 905 elf = ELF() 906 elf._parse_from_dump_file(path) 907 return elf 908 909 910 @staticmethod 911 def load_dumps(buf): 912 """Create an ELF instance from a dump file buffer""" 913 elf = ELF() 914 elf._parse_from_dump_buf(buf) 915 return elf 916 917 918 def is_jni_lib(self): 919 """Test whether the ELF file looks like a JNI library.""" 920 for name in ['libnativehelper.so', 'libandroid_runtime.so']: 921 if name in self.dt_needed: 922 return True 923 for symbol in itertools.chain(self.imported_symbols, 924 self.exported_symbols): 925 if symbol.startswith('JNI_') or symbol.startswith('Java_') or \ 926 symbol == 'jniRegisterNativeMethods': 927 return True 928 return False 929 930 931#------------------------------------------------------------------------------ 932# APK / Dex File Reader 933#------------------------------------------------------------------------------ 934 935class DexFileReader(object): 936 @classmethod 937 def extract_dex_string(cls, buf, offset=0): 938 end = buf.find(b'\0', offset) 939 return buf[offset:] if end == -1 else buf[offset:end] 940 941 942 @classmethod 943 def extract_uleb128(cls, buf, offset=0): 944 num_bytes = 0 945 result = 0 946 shift = 0 947 while True: 948 byte = buf[offset + num_bytes] 949 result |= (byte & 0x7f) << shift 950 num_bytes += 1 951 if (byte & 0x80) == 0: 952 break 953 shift += 7 954 return (result, num_bytes) 955 956 957 Header = create_struct('Header', ( 958 ('magic', '4s'), 959 ('version', '4s'), 960 ('checksum', 'I'), 961 ('signature', '20s'), 962 ('file_size', 'I'), 963 ('header_size', 'I'), 964 ('endian_tag', 'I'), 965 ('link_size', 'I'), 966 ('link_off', 'I'), 967 ('map_off', 'I'), 968 ('string_ids_size', 'I'), 969 ('string_ids_off', 'I'), 970 ('type_ids_size', 'I'), 971 ('type_ids_off', 'I'), 972 ('proto_ids_size', 'I'), 973 ('proto_ids_off', 'I'), 974 ('field_ids_size', 'I'), 975 ('field_ids_off', 'I'), 976 ('method_ids_size', 'I'), 977 ('method_ids_off', 'I'), 978 ('class_defs_size', 'I'), 979 ('class_defs_off', 'I'), 980 ('data_size', 'I'), 981 ('data_off', 'I'), 982 )) 983 984 985 StringId = create_struct('StringId', ( 986 ('string_data_off', 'I'), 987 )) 988 989 990 @staticmethod 991 def generate_classes_dex_names(): 992 yield 'classes.dex' 993 for i in itertools.count(start=2): 994 yield 'classes{}.dex'.format(i) 995 996 997 @classmethod 998 def enumerate_dex_strings_buf(cls, buf, offset=0, data_offset=None): 999 buf = get_py3_bytes(buf) 1000 header = cls.Header.unpack_from(buf, offset=offset) 1001 1002 if data_offset is None: 1003 if header.magic == b'dex\n': 1004 # In the standard dex file, the data_offset is the offset of 1005 # the dex header. 1006 data_offset = offset 1007 else: 1008 # In the compact dex file, the data_offset is sum of the offset 1009 # of the dex header and header.data_off. 1010 data_offset = offset + header.data_off 1011 1012 StringId = cls.StringId 1013 struct_size = StringId.struct_size 1014 1015 offset_start = offset + header.string_ids_off 1016 offset_end = offset_start + header.string_ids_size * struct_size 1017 1018 for offset in range(offset_start, offset_end, struct_size): 1019 offset = StringId.unpack_from(buf, offset).string_data_off 1020 offset += data_offset 1021 1022 # Skip the ULEB128 integer for UTF-16 string length 1023 offset += cls.extract_uleb128(buf, offset)[1] 1024 1025 # Extract the string 1026 yield cls.extract_dex_string(buf, offset) 1027 1028 1029 @classmethod 1030 def enumerate_dex_strings_apk(cls, apk_file_path): 1031 with zipfile.ZipFile(apk_file_path, 'r') as zip_file: 1032 for name in cls.generate_classes_dex_names(): 1033 try: 1034 with zip_file.open(name) as dex_file: 1035 dex_file_content = dex_file.read() 1036 for s in cls.enumerate_dex_strings_buf(dex_file_content): 1037 yield s 1038 except KeyError: 1039 break 1040 1041 1042 @classmethod 1043 def is_vdex_file(cls, vdex_file_path): 1044 return vdex_file_path.endswith('.vdex') 1045 1046 1047 # VdexHeader 0 1048 VdexHeader0 = create_struct('VdexHeader0', ( 1049 ('magic', '4s'), 1050 ('vdex_version', '4s'), 1051 )) 1052 1053 1054 # VdexHeader 1 - 15 1055 VdexHeader1 = create_struct('VdexHeader1', ( 1056 ('magic', '4s'), 1057 ('vdex_version', '4s'), 1058 ('number_of_dex_files', 'I'), 1059 ('dex_size', 'I'), 1060 ('verifier_deps_size', 'I'), 1061 ('quickening_info_size', 'I'), 1062 # checksums 1063 )) 1064 1065 1066 # VdexHeader 16 - 18 1067 VdexHeader16 = create_struct('VdexHeader16', ( 1068 ('magic', '4s'), 1069 ('vdex_version', '4s'), 1070 ('number_of_dex_files', 'I'), 1071 ('dex_size', 'I'), 1072 ('dex_shared_data_size', 'I'), 1073 ('verifier_deps_size', 'I'), 1074 ('quickening_info_size', 'I'), 1075 # checksums 1076 )) 1077 1078 1079 # VdexHeader 19 1080 VdexHeader19 = create_struct('VdexHeader19', ( 1081 ('magic', '4s'), 1082 ('vdex_version', '4s'), 1083 ('dex_section_version', '4s'), 1084 ('number_of_dex_files', 'I'), 1085 ('verifier_deps_size', 'I'), 1086 # checksums 1087 )) 1088 1089 1090 DexSectionHeader = create_struct('DexSectionHeader', ( 1091 ('dex_size', 'I'), 1092 ('dex_shared_data_size', 'I'), 1093 ('quickening_info_size', 'I'), 1094 )) 1095 1096 1097 @classmethod 1098 def enumerate_dex_strings_vdex_buf(cls, buf): 1099 buf = get_py3_bytes(buf) 1100 1101 magic, version = struct.unpack_from('4s4s', buf) 1102 1103 # Check the vdex file magic word 1104 if magic != b'vdex': 1105 raise ValueError('bad vdex magic word') 1106 1107 # Parse vdex file header (w.r.t. version) 1108 if version == b'000\x00': 1109 VdexHeader = cls.VdexHeader0 1110 elif b'001\x00' <= version < b'016\x00': 1111 VdexHeader = cls.VdexHeader1 1112 elif b'016\x00' <= version < b'019\x00': 1113 VdexHeader = cls.VdexHeader16 1114 elif version == b'019\x00': 1115 VdexHeader = cls.VdexHeader19 1116 else: 1117 raise ValueError('unknown vdex version ' + repr(version)) 1118 1119 vdex_header = VdexHeader.unpack_from(buf, offset=0) 1120 1121 # Skip this vdex file if there is no dex file section 1122 if vdex_header.vdex_version < b'019\x00': 1123 if vdex_header.dex_size == 0: 1124 return 1125 else: 1126 if vdex_header.dex_section_version == b'000\x00': 1127 return 1128 1129 # Skip vdex file header struct 1130 offset = VdexHeader.struct_size 1131 1132 # Skip dex file checksums struct 1133 offset += 4 * vdex_header.number_of_dex_files 1134 1135 # Skip dex section header struct 1136 if vdex_header.vdex_version >= b'019\x00': 1137 offset += cls.DexSectionHeader.struct_size 1138 1139 # Calculate the quickening table offset 1140 if vdex_header.vdex_version >= b'012\x00': 1141 quickening_table_off_size = 4 1142 else: 1143 quickening_table_off_size = 0 1144 1145 for _ in range(vdex_header.number_of_dex_files): 1146 # Skip quickening_table_off size 1147 offset += quickening_table_off_size 1148 1149 # Check the dex file magic 1150 dex_magic = buf[offset:offset + 4] 1151 if dex_magic not in (b'dex\n', b'cdex'): 1152 raise ValueError('bad dex file offset {}'.format(offset)) 1153 1154 dex_header = cls.Header.unpack_from(buf, offset) 1155 dex_file_end = offset + dex_header.file_size 1156 for s in cls.enumerate_dex_strings_buf(buf, offset): 1157 yield s 1158 1159 # Align to the end of the dex file 1160 offset = (dex_file_end + 3) // 4 * 4 1161 1162 1163 @classmethod 1164 def enumerate_dex_strings_vdex(cls, vdex_file_path): 1165 with open(vdex_file_path, 'rb') as vdex_file: 1166 return cls.enumerate_dex_strings_vdex_buf(vdex_file.read()) 1167 1168 1169 @classmethod 1170 def enumerate_dex_strings(cls, path): 1171 if is_zipfile(path): 1172 return DexFileReader.enumerate_dex_strings_apk(path) 1173 if cls.is_vdex_file(path): 1174 return DexFileReader.enumerate_dex_strings_vdex(path) 1175 return None 1176 1177 1178#------------------------------------------------------------------------------ 1179# TaggedDict 1180#------------------------------------------------------------------------------ 1181 1182class TaggedDict(object): 1183 def _define_tag_constants(local_ns): 1184 tag_list = [ 1185 'll_ndk', 'll_ndk_indirect', 1186 'vndk_sp', 'vndk_sp_indirect', 'vndk_sp_indirect_private', 1187 'vndk', 1188 'fwk_only', 'fwk_only_rs', 1189 'sp_hal', 'sp_hal_dep', 1190 'vnd_only', 1191 'remove', 1192 ] 1193 assert len(tag_list) < 32 1194 1195 tags = {} 1196 for i, tag in enumerate(tag_list): 1197 local_ns[tag.upper()] = 1 << i 1198 tags[tag] = 1 << i 1199 1200 local_ns['TAGS'] = tags 1201 1202 _define_tag_constants(locals()) 1203 del _define_tag_constants 1204 1205 1206 _TAG_ALIASES = { 1207 'hl_ndk': 'fwk_only', # Treat HL-NDK as FWK-ONLY. 1208 'sp_ndk': 'll_ndk', 1209 'sp_ndk_indirect': 'll_ndk_indirect', 1210 'vndk_indirect': 'vndk', # Legacy 1211 'vndk_sp_hal': 'vndk_sp', # Legacy 1212 'vndk_sp_both': 'vndk_sp', # Legacy 1213 1214 # FIXME: LL-NDK-Private, VNDK-Private and VNDK-SP-Private are new tags. 1215 # They should not be treated as aliases. 1216 # TODO: Refine the code that compute and verify VNDK sets and reverse 1217 # the aliases. 1218 'll_ndk_private': 'll_ndk_indirect', 1219 'vndk_private': 'vndk', 1220 'vndk_sp_private': 'vndk_sp_indirect_private', 1221 } 1222 1223 1224 @classmethod 1225 def _normalize_tag(cls, tag): 1226 tag = tag.lower().replace('-', '_') 1227 tag = cls._TAG_ALIASES.get(tag, tag) 1228 if tag not in cls.TAGS: 1229 raise ValueError('unknown lib tag ' + tag) 1230 return tag 1231 1232 1233 _LL_NDK_VIS = {'ll_ndk', 'll_ndk_indirect'} 1234 _VNDK_SP_VIS = {'ll_ndk', 'vndk_sp', 'vndk_sp_indirect', 1235 'vndk_sp_indirect_private', 'fwk_only_rs'} 1236 _FWK_ONLY_VIS = {'ll_ndk', 'll_ndk_indirect', 1237 'vndk_sp', 'vndk_sp_indirect', 'vndk_sp_indirect_private', 1238 'vndk', 'fwk_only', 'fwk_only_rs', 'sp_hal'} 1239 _SP_HAL_VIS = {'ll_ndk', 'vndk_sp', 'sp_hal', 'sp_hal_dep'} 1240 1241 _TAG_VISIBILITY = { 1242 'll_ndk': _LL_NDK_VIS, 1243 'll_ndk_indirect': _LL_NDK_VIS, 1244 1245 'vndk_sp': _VNDK_SP_VIS, 1246 'vndk_sp_indirect': _VNDK_SP_VIS, 1247 'vndk_sp_indirect_private': _VNDK_SP_VIS, 1248 1249 'vndk': {'ll_ndk', 'vndk_sp', 'vndk_sp_indirect', 'vndk'}, 1250 1251 'fwk_only': _FWK_ONLY_VIS, 1252 'fwk_only_rs': _FWK_ONLY_VIS, 1253 1254 'sp_hal': _SP_HAL_VIS, 1255 'sp_hal_dep': _SP_HAL_VIS, 1256 1257 'vnd_only': {'ll_ndk', 'vndk_sp', 'vndk_sp_indirect', 1258 'vndk', 'sp_hal', 'sp_hal_dep', 'vnd_only'}, 1259 1260 'remove': set(), 1261 } 1262 1263 del _LL_NDK_VIS, _VNDK_SP_VIS, _FWK_ONLY_VIS, _SP_HAL_VIS 1264 1265 1266 @classmethod 1267 def is_tag_visible(cls, from_tag, to_tag): 1268 return to_tag in cls._TAG_VISIBILITY[from_tag] 1269 1270 1271 def __init__(self, vndk_lib_dirs=None): 1272 self._path_tag = dict() 1273 for tag in self.TAGS: 1274 setattr(self, tag, set()) 1275 self._regex_patterns = [] 1276 1277 if vndk_lib_dirs is None: 1278 self._vndk_suffixes = [''] 1279 else: 1280 self._vndk_suffixes = [VNDKLibDir.create_vndk_dir_suffix(version) 1281 for version in vndk_lib_dirs] 1282 1283 1284 def add(self, tag, lib): 1285 lib_set = getattr(self, tag) 1286 lib_set.add(lib) 1287 self._path_tag[lib] = tag 1288 1289 1290 def add_regex(self, tag, pattern): 1291 self._regex_patterns.append((re.compile(pattern), tag)) 1292 1293 1294 def get_path_tag(self, lib): 1295 try: 1296 return self._path_tag[lib] 1297 except KeyError: 1298 pass 1299 1300 for pattern, tag in self._regex_patterns: 1301 if pattern.match(lib): 1302 return tag 1303 1304 return self.get_path_tag_default(lib) 1305 1306 1307 def get_path_tag_default(self, lib): 1308 raise NotImplementedError() 1309 1310 1311 def get_path_tag_bit(self, lib): 1312 return self.TAGS[self.get_path_tag(lib)] 1313 1314 1315 def is_path_visible(self, from_lib, to_lib): 1316 return self.is_tag_visible(self.get_path_tag(from_lib), 1317 self.get_path_tag(to_lib)) 1318 1319 1320 @staticmethod 1321 def is_ll_ndk(tag_bit): 1322 return bool(tag_bit & TaggedDict.LL_NDK) 1323 1324 1325 @staticmethod 1326 def is_vndk_sp(tag_bit): 1327 return bool(tag_bit & TaggedDict.VNDK_SP) 1328 1329 1330 @staticmethod 1331 def is_vndk_sp_indirect(tag_bit): 1332 return bool(tag_bit & TaggedDict.VNDK_SP_INDIRECT) 1333 1334 1335 @staticmethod 1336 def is_vndk_sp_indirect_private(tag_bit): 1337 return bool(tag_bit & TaggedDict.VNDK_SP_INDIRECT_PRIVATE) 1338 1339 1340 @staticmethod 1341 def is_fwk_only_rs(tag_bit): 1342 return bool(tag_bit & TaggedDict.FWK_ONLY_RS) 1343 1344 1345 @staticmethod 1346 def is_sp_hal(tag_bit): 1347 return bool(tag_bit & TaggedDict.SP_HAL) 1348 1349 1350class TaggedPathDict(TaggedDict): 1351 def load_from_csv(self, fp): 1352 reader = csv.reader(fp) 1353 1354 # Read first row and check the existence of the header. 1355 try: 1356 row = next(reader) 1357 except StopIteration: 1358 return 1359 1360 try: 1361 path_col = row.index('Path') 1362 tag_col = row.index('Tag') 1363 except ValueError: 1364 path_col = 0 1365 tag_col = 1 1366 self.add(self._normalize_tag(row[tag_col]), row[path_col]) 1367 1368 # Read the rest of rows. 1369 for row in reader: 1370 self.add(self._normalize_tag(row[tag_col]), row[path_col]) 1371 1372 1373 @staticmethod 1374 def create_from_csv(fp, vndk_lib_dirs=None): 1375 d = TaggedPathDict(vndk_lib_dirs) 1376 d.load_from_csv(fp) 1377 return d 1378 1379 1380 @staticmethod 1381 def create_from_csv_path(path, vndk_lib_dirs=None): 1382 with open(path, 'r') as fp: 1383 return TaggedPathDict.create_from_csv(fp, vndk_lib_dirs) 1384 1385 1386 def _enumerate_paths_with_lib(self, pattern): 1387 if '${LIB}' in pattern: 1388 yield pattern.replace('${LIB}', 'lib') 1389 yield pattern.replace('${LIB}', 'lib64') 1390 else: 1391 yield pattern 1392 1393 1394 def _enumerate_paths(self, pattern): 1395 if '${VNDK_VER}' not in pattern: 1396 for path in self._enumerate_paths_with_lib(pattern): 1397 yield path 1398 return 1399 for suffix in self._vndk_suffixes: 1400 pattern_with_suffix = pattern.replace('${VNDK_VER}', suffix) 1401 for path in self._enumerate_paths_with_lib(pattern_with_suffix): 1402 yield path 1403 1404 1405 def add(self, tag, path): 1406 if path.startswith('[regex]'): 1407 super(TaggedPathDict, self).add_regex(tag, path[7:]) 1408 return 1409 for path in self._enumerate_paths(path): 1410 super(TaggedPathDict, self).add(tag, path) 1411 1412 1413 def get_path_tag_default(self, path): 1414 return 'vnd_only' if path.startswith('/vendor') else 'fwk_only' 1415 1416 1417class TaggedLibDict(object): 1418 def __init__(self): 1419 self._path_tag = dict() 1420 for tag in TaggedDict.TAGS: 1421 setattr(self, tag, set()) 1422 1423 1424 def add(self, tag, lib): 1425 lib_set = getattr(self, tag) 1426 lib_set.add(lib) 1427 self._path_tag[lib] = tag 1428 1429 1430 @staticmethod 1431 def create_from_graph(graph, tagged_paths, generic_refs=None): 1432 d = TaggedLibDict() 1433 1434 for lib in graph.lib_pt[PT_SYSTEM].values(): 1435 d.add(tagged_paths.get_path_tag(lib.path), lib) 1436 1437 sp_lib = graph.compute_sp_lib(generic_refs) 1438 for lib in graph.lib_pt[PT_VENDOR].values(): 1439 if lib in sp_lib.sp_hal: 1440 d.add('sp_hal', lib) 1441 elif lib in sp_lib.sp_hal_dep: 1442 d.add('sp_hal_dep', lib) 1443 else: 1444 d.add('vnd_only', lib) 1445 return d 1446 1447 1448 def get_path_tag(self, lib): 1449 try: 1450 return self._path_tag[lib] 1451 except KeyError: 1452 return self.get_path_tag_default(lib) 1453 1454 1455 def get_path_tag_default(self, lib): 1456 return 'vnd_only' if lib.path.startswith('/vendor') else 'fwk_only' 1457 1458 1459class LibProperties(object): 1460 Properties = collections.namedtuple( 1461 'Properties', 'vndk vndk_sp vendor_available rule') 1462 1463 1464 def __init__(self, csv_file=None): 1465 self.modules = {} 1466 1467 if csv_file: 1468 reader = csv.reader(csv_file) 1469 1470 header = next(reader) 1471 assert header == ['name', 'vndk', 'vndk_sp', 'vendor_available', 1472 'rule'], repr(header) 1473 1474 for name, vndk, vndk_sp, vendor_available, rule in reader: 1475 self.modules[name] = self.Properties( 1476 vndk == 'True', vndk_sp == 'True', 1477 vendor_available == 'True', rule) 1478 1479 1480 @classmethod 1481 def load_from_path_or_default(cls, path): 1482 if not path: 1483 return LibProperties() 1484 1485 try: 1486 with open(path, 'r') as csv_file: 1487 return LibProperties(csv_file) 1488 except FileNotFoundError: 1489 return LibProperties() 1490 1491 1492 def get(self, name): 1493 try: 1494 return self.modules[name] 1495 except KeyError: 1496 return self.Properties(False, False, False, None) 1497 1498 1499 @staticmethod 1500 def get_lib_properties_file_path(tag_file_path): 1501 root, ext = os.path.splitext(tag_file_path) 1502 return root + '-properties' + ext 1503 1504 1505#------------------------------------------------------------------------------ 1506# ELF Linker 1507#------------------------------------------------------------------------------ 1508 1509def is_zipfile(path): 1510 # zipfile.is_zipfile() tries to find the zip header in the file. But we 1511 # only want to scan the zip file that starts with the magic word. Thus, 1512 # we read the magic word by ourselves. 1513 try: 1514 with open(path, 'rb') as fp: 1515 if fp.read(2) != b'PK': 1516 return False 1517 except IOError: 1518 return False 1519 1520 # Check whether this is a valid zip file. 1521 return zipfile.is_zipfile(path) 1522 1523 1524def scan_zip_file(zip_file_path): 1525 """Scan all ELF files in a zip archive.""" 1526 with zipfile.ZipFile(zip_file_path, 'r') as zip_file: 1527 for name in zip_file.namelist(): 1528 yield (os.path.join(zip_file_path, name), 1529 zip_file.open(name, 'r').read()) 1530 1531 1532def dump_ext4_img(img_file_path, out_dir): 1533 if ' ' in out_dir: 1534 raise ValueError('out_dir must not have space character') 1535 1536 cmd = ['debugfs', img_file_path, '-R', 'rdump / ' + out_dir] 1537 1538 # Run the debugfs command. 1539 proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 1540 stdout, stderr = proc.communicate() 1541 if proc.returncode != 0: 1542 raise subprocess.CalledProcessError(proc.returncode, cmd, stderr) 1543 1544 # Print error messages if they are not the ones that should be ignored. 1545 for line in stderr.splitlines(): 1546 if line.startswith(b'debugfs '): 1547 continue 1548 if b'Operation not permitted while changing ownership of' in line: 1549 continue 1550 print_sb('error: debugfs:', line, file=sys.stderr) 1551 1552 1553def is_accessible(path): 1554 try: 1555 mode = os.stat(path).st_mode 1556 return (mode & (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)) != 0 1557 except FileNotFoundError: 1558 return False 1559 1560 1561def scan_ext4_image(img_file_path, mount_point, unzip_files): 1562 """Scan all ELF files in the ext4 image.""" 1563 with TemporaryDirectory() as tmp_dir: 1564 dump_ext4_img(img_file_path, tmp_dir) 1565 for path, elf in scan_elf_files(tmp_dir, mount_point, unzip_files): 1566 yield path, elf 1567 1568 1569def scan_apex_dir(apex_collection_root, apex_dir, unzip_files): 1570 # Read the manifest file. 1571 manifest_file_path = os.path.join(apex_dir, 'apex_manifest.json') 1572 try: 1573 with open(manifest_file_path, 'r') as manifest_file: 1574 manifest = json.load(manifest_file) 1575 except FileNotFoundError: 1576 print('error: Failed to find apex manifest: {}' 1577 .format(manifest_file_path), file=sys.stderr) 1578 return 1579 1580 # Read the module name. 1581 try: 1582 apex_name = manifest['name'] 1583 except KeyError: 1584 print('error: Failed to read apex name from manifest: {}' 1585 .format(manifest_file_path), file=sys.stderr) 1586 return 1587 1588 # Scan the payload (or the flatten payload). 1589 mount_point = os.path.join(apex_collection_root, apex_name) 1590 img_file_path = os.path.join(apex_dir, 'apex_payload.img') 1591 if os.path.exists(img_file_path): 1592 for path, elf in scan_ext4_image(img_file_path, mount_point, 1593 unzip_files): 1594 yield path, elf 1595 else: 1596 for path, elf in scan_elf_files(apex_dir, mount_point, unzip_files): 1597 yield path, elf 1598 1599 1600def scan_apex_file(apex_collection_root, apex_zip_file, unzip_files): 1601 with TemporaryDirectory() as tmp_dir: 1602 with zipfile.ZipFile(apex_zip_file) as zip_file: 1603 zip_file.extractall(tmp_dir) 1604 for path, elf in scan_apex_dir(apex_collection_root, tmp_dir, 1605 unzip_files): 1606 yield path, elf 1607 1608 1609def scan_apex_files(apex_collection_root, unzip_files): 1610 for ent in scandir(apex_collection_root): 1611 if ent.is_dir(): 1612 for path, elf in scan_apex_dir(apex_collection_root, ent.path, 1613 unzip_files): 1614 yield path, elf 1615 elif ent.is_file() and ent.name.endswith('.apex'): 1616 for path, elf in scan_apex_file(apex_collection_root, ent.path, 1617 unzip_files): 1618 yield path, elf 1619 1620 1621def scan_elf_files(root, mount_point=None, unzip_files=True): 1622 """Scan all ELF files under a directory.""" 1623 1624 if mount_point: 1625 root_prefix_len = len(root) + 1 1626 def norm_path(path): 1627 return os.path.join(mount_point, path[root_prefix_len:]) 1628 else: 1629 def norm_path(path): 1630 return path 1631 1632 for base, dirnames, filenames in os.walk(root): 1633 if base == root and 'apex' in dirnames: 1634 dirnames.remove('apex') 1635 for path, elf in scan_apex_files(os.path.join(root, 'apex'), 1636 unzip_files): 1637 yield (path, elf) 1638 1639 for filename in filenames: 1640 path = os.path.join(base, filename) 1641 if not is_accessible(path): 1642 continue 1643 1644 1645 # If this is a zip file and unzip_file is true, scan the ELF files 1646 # in the zip file. 1647 if unzip_files and is_zipfile(path): 1648 for path, content in scan_zip_file(path): 1649 try: 1650 yield (norm_path(path), ELF.loads(content)) 1651 except ELFError: 1652 pass 1653 continue 1654 1655 # Load ELF from the path. 1656 try: 1657 yield (norm_path(path), ELF.load(path)) 1658 except ELFError: 1659 pass 1660 1661 1662PT_SYSTEM = 0 1663PT_VENDOR = 1 1664NUM_PARTITIONS = 2 1665 1666 1667SPLibResult = collections.namedtuple( 1668 'SPLibResult', 1669 'sp_hal sp_hal_dep vndk_sp_hal ll_ndk ll_ndk_indirect vndk_sp_both') 1670 1671 1672VNDKLibTuple = defaultnamedtuple('VNDKLibTuple', 'vndk_sp vndk', []) 1673 1674 1675class VNDKLibDir(list): 1676 """VNDKLibDir is a dict which maps version to VNDK-SP and VNDK directory 1677 paths.""" 1678 1679 1680 @classmethod 1681 def create_vndk_dir_suffix(cls, version): 1682 """Create VNDK version suffix.""" 1683 return '' if version == 'current' else '-' + version 1684 1685 1686 @classmethod 1687 def create_vndk_sp_dir_name(cls, version): 1688 """Create VNDK-SP directory name from a given version.""" 1689 return 'vndk-sp' + cls.create_vndk_dir_suffix(version) 1690 1691 1692 @classmethod 1693 def create_vndk_dir_name(cls, version): 1694 """Create VNDK directory name from a given version.""" 1695 return 'vndk' + cls.create_vndk_dir_suffix(version) 1696 1697 1698 @classmethod 1699 def extract_version_from_name(cls, name): 1700 """Extract VNDK version from a name.""" 1701 if name in {'vndk', 'vndk-sp'}: 1702 return 'current' 1703 if name.startswith('vndk-sp-'): 1704 return name[len('vndk-sp-'):] 1705 if name.startswith('vndk-'): 1706 return name[len('vndk-'):] 1707 return None 1708 1709 1710 @classmethod 1711 def extract_path_component(cls, path, index): 1712 """Extract n-th path component from a posix path.""" 1713 start = 0 1714 for _ in range(index): 1715 pos = path.find('/', start) 1716 if pos == -1: 1717 return None 1718 start = pos + 1 1719 end = path.find('/', start) 1720 if end == -1: 1721 return None 1722 return path[start:end] 1723 1724 1725 @classmethod 1726 def extract_version_from_path(cls, path): 1727 """Extract VNDK version from the third path component.""" 1728 component = cls.extract_path_component(path, 3) 1729 if not component: 1730 return None 1731 return cls.extract_version_from_name(component) 1732 1733 1734 @classmethod 1735 def is_in_vndk_dir(cls, path): 1736 """Determine whether a path is under a VNDK directory.""" 1737 component = cls.extract_path_component(path, 3) 1738 if not component: 1739 return False 1740 return (component == 'vndk' or 1741 (component.startswith('vndk-') and 1742 not component == 'vndk-sp' and 1743 not component.startswith('vndk-sp-'))) 1744 1745 1746 @classmethod 1747 def is_in_vndk_sp_dir(cls, path): 1748 """Determine whether a path is under a VNDK-SP directory.""" 1749 component = cls.extract_path_component(path, 3) 1750 if not component: 1751 return False 1752 return component == 'vndk-sp' or component.startswith('vndk-sp-') 1753 1754 1755 @classmethod 1756 def create_vndk_search_paths(cls, lib_dir, version): 1757 """Create VNDK/VNDK-SP search paths from lib_dir and version.""" 1758 vndk_sp_name = cls.create_vndk_sp_dir_name(version) 1759 vndk_name = cls.create_vndk_dir_name(version) 1760 return VNDKLibTuple( 1761 [posixpath.join('/vendor', lib_dir, vndk_sp_name), 1762 posixpath.join('/system', lib_dir, vndk_sp_name)], 1763 [posixpath.join('/vendor', lib_dir, vndk_name), 1764 posixpath.join('/system', lib_dir, vndk_name)]) 1765 1766 1767 @classmethod 1768 def create_default(cls): 1769 """Create default VNDK-SP and VNDK paths without versions.""" 1770 vndk_lib_dirs = VNDKLibDir() 1771 vndk_lib_dirs.append('current') 1772 return vndk_lib_dirs 1773 1774 1775 @classmethod 1776 def create_from_version(cls, version): 1777 """Create default VNDK-SP and VNDK paths with the specified version.""" 1778 vndk_lib_dirs = VNDKLibDir() 1779 vndk_lib_dirs.append(version) 1780 return vndk_lib_dirs 1781 1782 1783 @classmethod 1784 def create_from_dirs(cls, system_dirs, vendor_dirs): 1785 """Scan system_dirs and vendor_dirs and collect all VNDK-SP and VNDK 1786 directory paths.""" 1787 1788 def collect_versions(base_dirs): 1789 versions = set() 1790 for base_dir in base_dirs: 1791 for lib_dir in ('lib', 'lib64'): 1792 lib_dir_path = os.path.join(base_dir, lib_dir) 1793 try: 1794 for name in os.listdir(lib_dir_path): 1795 version = cls.extract_version_from_name(name) 1796 if version: 1797 versions.add(version) 1798 except FileNotFoundError: 1799 pass 1800 return versions 1801 1802 versions = set() 1803 if system_dirs: 1804 versions.update(collect_versions(system_dirs)) 1805 if vendor_dirs: 1806 versions.update(collect_versions(vendor_dirs)) 1807 1808 # Sanity check: Versions must not be 'sp' or start with 'sp-'. 1809 bad_versions = [version for version in versions 1810 if version == 'sp' or version.startswith('sp-')] 1811 if bad_versions: 1812 raise ValueError('bad vndk version: ' + repr(bad_versions)) 1813 1814 return VNDKLibDir(cls.sorted_version(versions)) 1815 1816 1817 def classify_vndk_libs(self, libs): 1818 """Classify VNDK/VNDK-SP shared libraries.""" 1819 vndk_sp_libs = collections.defaultdict(set) 1820 vndk_libs = collections.defaultdict(set) 1821 other_libs = set() 1822 1823 for lib in libs: 1824 component = self.extract_path_component(lib.path, 3) 1825 if component is None: 1826 other_libs.add(lib) 1827 continue 1828 1829 version = self.extract_version_from_name(component) 1830 if version is None: 1831 other_libs.add(lib) 1832 continue 1833 1834 if component.startswith('vndk-sp'): 1835 vndk_sp_libs[version].add(lib) 1836 else: 1837 vndk_libs[version].add(lib) 1838 1839 return (vndk_sp_libs, vndk_libs, other_libs) 1840 1841 1842 @classmethod 1843 def _get_property(cls, property_file, name): 1844 """Read a property from a property file.""" 1845 for line in property_file: 1846 if line.startswith(name + '='): 1847 return line[len(name) + 1:].strip() 1848 return None 1849 1850 1851 @classmethod 1852 def get_ro_vndk_version(cls, vendor_dirs): 1853 """Read ro.vendor.version property from vendor partitions.""" 1854 for vendor_dir in vendor_dirs: 1855 path = os.path.join(vendor_dir, 'default.prop') 1856 try: 1857 with open(path, 'r') as property_file: 1858 result = cls._get_property( 1859 property_file, 'ro.vndk.version') 1860 if result is not None: 1861 return result 1862 except FileNotFoundError: 1863 pass 1864 return None 1865 1866 1867 @classmethod 1868 def sorted_version(cls, versions): 1869 """Sort versions in the following rule: 1870 1871 1. 'current' is the first. 1872 1873 2. The versions that cannot be converted to int are sorted 1874 lexicographically in descendant order. 1875 1876 3. The versions that can be converted to int are sorted as integers in 1877 descendant order. 1878 """ 1879 1880 current = [] 1881 alpha = [] 1882 numeric = [] 1883 1884 for version in versions: 1885 if version == 'current': 1886 current.append(version) 1887 continue 1888 try: 1889 numeric.append(int(version)) 1890 except ValueError: 1891 alpha.append(version) 1892 1893 alpha.sort(reverse=True) 1894 numeric.sort(reverse=True) 1895 1896 return current + alpha + [str(x) for x in numeric] 1897 1898 1899 def find_vendor_vndk_version(self, vendor_dirs): 1900 """Find the best-fitting VNDK version.""" 1901 1902 ro_vndk_version = self.get_ro_vndk_version(vendor_dirs) 1903 if ro_vndk_version is not None: 1904 return ro_vndk_version 1905 1906 if not self: 1907 return 'current' 1908 1909 return self.sorted_version(self)[0] 1910 1911 1912# File path patterns for Android apps 1913_APP_DIR_PATTERNS = re.compile('^(?:/[^/]+){1,2}/(?:priv-)?app/') 1914 1915 1916class ELFResolver(object): 1917 def __init__(self, lib_set, default_search_path): 1918 self.lib_set = lib_set 1919 self.default_search_path = default_search_path 1920 1921 1922 def get_candidates(self, requester, name, dt_rpath=None, dt_runpath=None): 1923 # Search app-specific search paths. 1924 if _APP_DIR_PATTERNS.match(requester): 1925 yield os.path.join(os.path.dirname(requester), name) 1926 1927 # Search default search paths. 1928 if dt_rpath: 1929 for d in dt_rpath: 1930 yield os.path.join(d, name) 1931 if dt_runpath: 1932 for d in dt_runpath: 1933 yield os.path.join(d, name) 1934 for d in self.default_search_path: 1935 yield os.path.join(d, name) 1936 1937 1938 def resolve(self, requester, name, dt_rpath=None, dt_runpath=None): 1939 for path in self.get_candidates(requester, name, dt_rpath, dt_runpath): 1940 try: 1941 return self.lib_set[path] 1942 except KeyError: 1943 continue 1944 return None 1945 1946 1947class ELFLinkData(object): 1948 def __init__(self, partition, path, elf, tag_bit): 1949 self.partition = partition 1950 self.path = path 1951 self.elf = elf 1952 self.deps_needed = set() 1953 self.deps_needed_hidden = set() 1954 self.deps_dlopen = set() 1955 self.deps_dlopen_hidden = set() 1956 self.users_needed = set() 1957 self.users_needed_hidden = set() 1958 self.users_dlopen = set() 1959 self.users_dlopen_hidden = set() 1960 self.imported_ext_symbols = collections.defaultdict(set) 1961 self._tag_bit = tag_bit 1962 self.unresolved_symbols = set() 1963 self.unresolved_dt_needed = [] 1964 self.linked_symbols = dict() 1965 1966 1967 @property 1968 def is_ll_ndk(self): 1969 return TaggedDict.is_ll_ndk(self._tag_bit) 1970 1971 1972 @property 1973 def is_vndk_sp(self): 1974 return TaggedDict.is_vndk_sp(self._tag_bit) 1975 1976 1977 @property 1978 def is_vndk_sp_indirect(self): 1979 return TaggedDict.is_vndk_sp_indirect(self._tag_bit) 1980 1981 1982 @property 1983 def is_vndk_sp_indirect_private(self): 1984 return TaggedDict.is_vndk_sp_indirect_private(self._tag_bit) 1985 1986 1987 @property 1988 def is_fwk_only_rs(self): 1989 return TaggedDict.is_fwk_only_rs(self._tag_bit) 1990 1991 1992 @property 1993 def is_sp_hal(self): 1994 return TaggedDict.is_sp_hal(self._tag_bit) 1995 1996 1997 def add_needed_dep(self, dst): 1998 assert dst not in self.deps_needed_hidden 1999 assert self not in dst.users_needed_hidden 2000 self.deps_needed.add(dst) 2001 dst.users_needed.add(self) 2002 2003 2004 def add_dlopen_dep(self, dst): 2005 assert dst not in self.deps_dlopen_hidden 2006 assert self not in dst.users_dlopen_hidden 2007 self.deps_dlopen.add(dst) 2008 dst.users_dlopen.add(self) 2009 2010 2011 def hide_needed_dep(self, dst): 2012 self.deps_needed.remove(dst) 2013 dst.users_needed.remove(self) 2014 self.deps_needed_hidden.add(dst) 2015 dst.users_needed_hidden.add(self) 2016 2017 2018 def hide_dlopen_dep(self, dst): 2019 self.deps_dlopen.remove(dst) 2020 dst.users_dlopen.remove(self) 2021 self.deps_dlopen_hidden.add(dst) 2022 dst.users_dlopen_hidden.add(self) 2023 2024 2025 @property 2026 def num_deps(self): 2027 """Get the number of dependencies. If a library is linked by both 2028 NEEDED and DLOPEN relationship, then it will be counted twice.""" 2029 return (len(self.deps_needed) + len(self.deps_needed_hidden) + 2030 len(self.deps_dlopen) + len(self.deps_dlopen_hidden)) 2031 2032 2033 @property 2034 def deps_all(self): 2035 return itertools.chain(self.deps_needed, self.deps_needed_hidden, 2036 self.deps_dlopen, self.deps_dlopen_hidden) 2037 2038 2039 @property 2040 def deps_good(self): 2041 return itertools.chain(self.deps_needed, self.deps_dlopen) 2042 2043 2044 @property 2045 def deps_needed_all(self): 2046 return itertools.chain(self.deps_needed, self.deps_needed_hidden) 2047 2048 2049 @property 2050 def deps_dlopen_all(self): 2051 return itertools.chain(self.deps_dlopen, self.deps_dlopen_hidden) 2052 2053 2054 @property 2055 def num_users(self): 2056 """Get the number of users. If a library is linked by both NEEDED and 2057 DLOPEN relationship, then it will be counted twice.""" 2058 return (len(self.users_needed) + len(self.users_needed_hidden) + 2059 len(self.users_dlopen) + len(self.users_dlopen_hidden)) 2060 2061 2062 @property 2063 def users_all(self): 2064 return itertools.chain(self.users_needed, self.users_needed_hidden, 2065 self.users_dlopen, self.users_dlopen_hidden) 2066 2067 2068 @property 2069 def users_good(self): 2070 return itertools.chain(self.users_needed, self.users_dlopen) 2071 2072 2073 @property 2074 def users_needed_all(self): 2075 return itertools.chain(self.users_needed, self.users_needed_hidden) 2076 2077 2078 @property 2079 def users_dlopen_all(self): 2080 return itertools.chain(self.users_dlopen, self.users_dlopen_hidden) 2081 2082 2083 def has_dep(self, dst): 2084 return (dst in self.deps_needed or dst in self.deps_needed_hidden or 2085 dst in self.deps_dlopen or dst in self.deps_dlopen_hidden) 2086 2087 2088 def has_user(self, dst): 2089 return (dst in self.users_needed or dst in self.users_needed_hidden or 2090 dst in self.users_dlopen or dst in self.users_dlopen_hidden) 2091 2092 2093 def is_system_lib(self): 2094 return self.partition == PT_SYSTEM 2095 2096 2097 def get_dep_linked_symbols(self, dep): 2098 symbols = set() 2099 for symbol, exp_lib in self.linked_symbols.items(): 2100 if exp_lib == dep: 2101 symbols.add(symbol) 2102 return sorted(symbols) 2103 2104 2105 def __lt__(self, rhs): 2106 return self.path < rhs.path 2107 2108 2109def sorted_lib_path_list(libs): 2110 libs = [lib.path for lib in libs] 2111 libs.sort() 2112 return libs 2113 2114 2115_VNDK_RESULT_FIELD_NAMES = ( 2116 'll_ndk', 'll_ndk_indirect', 2117 'vndk_sp', 'vndk_sp_unused', 'vndk_sp_indirect', 2118 'vndk_sp_indirect_unused', 'vndk_sp_indirect_private', 'vndk', 2119 'vndk_indirect', 'fwk_only', 'fwk_only_rs', 'sp_hal', 'sp_hal_dep', 2120 'vnd_only', 'vndk_ext', 'vndk_sp_ext', 'vndk_sp_indirect_ext', 2121 'extra_vendor_libs') 2122 2123 2124VNDKResult = defaultnamedtuple('VNDKResult', _VNDK_RESULT_FIELD_NAMES, set()) 2125 2126 2127_SIMPLE_VNDK_RESULT_FIELD_NAMES = ( 2128 'vndk_sp', 'vndk_sp_ext', 'extra_vendor_libs') 2129 2130 2131SimpleVNDKResult = defaultnamedtuple( 2132 'SimpleVNDKResult', _SIMPLE_VNDK_RESULT_FIELD_NAMES, set()) 2133 2134 2135class ELFLibDict(defaultnamedtuple('ELFLibDict', ('lib32', 'lib64'), {})): 2136 def get_lib_dict(self, elf_class): 2137 return self[elf_class - 1] 2138 2139 2140 def add(self, path, lib): 2141 self.get_lib_dict(lib.elf.ei_class)[path] = lib 2142 2143 2144 def remove(self, lib): 2145 del self.get_lib_dict(lib.elf.ei_class)[lib.path] 2146 2147 2148 def get(self, path, default=None): 2149 for lib_set in self: 2150 res = lib_set.get(path, None) 2151 if res: 2152 return res 2153 return default 2154 2155 2156 def keys(self): 2157 return itertools.chain(self.lib32.keys(), self.lib64.keys()) 2158 2159 2160 def values(self): 2161 return itertools.chain(self.lib32.values(), self.lib64.values()) 2162 2163 2164 def items(self): 2165 return itertools.chain(self.lib32.items(), self.lib64.items()) 2166 2167 2168class ELFLinker(object): 2169 def __init__(self, tagged_paths=None, vndk_lib_dirs=None, 2170 ro_vndk_version='current'): 2171 self.lib_pt = [ELFLibDict() for i in range(NUM_PARTITIONS)] 2172 2173 if vndk_lib_dirs is None: 2174 vndk_lib_dirs = VNDKLibDir.create_default() 2175 2176 self.vndk_lib_dirs = vndk_lib_dirs 2177 2178 if tagged_paths is None: 2179 script_dir = os.path.dirname(os.path.abspath(__file__)) 2180 dataset_path = os.path.join( 2181 script_dir, 'datasets', 'minimum_tag_file.csv') 2182 self.tagged_paths = TaggedPathDict.create_from_csv_path( 2183 dataset_path, vndk_lib_dirs) 2184 else: 2185 self.tagged_paths = tagged_paths 2186 2187 self.ro_vndk_version = ro_vndk_version 2188 2189 self.apex_module_names = set() 2190 2191 2192 def _add_lib_to_lookup_dict(self, lib): 2193 self.lib_pt[lib.partition].add(lib.path, lib) 2194 2195 2196 def _remove_lib_from_lookup_dict(self, lib): 2197 self.lib_pt[lib.partition].remove(lib) 2198 2199 2200 def _rename_lib(self, lib, new_path): 2201 self._remove_lib_from_lookup_dict(lib) 2202 lib.path = new_path 2203 self._add_lib_to_lookup_dict(lib) 2204 2205 2206 def add_lib(self, partition, path, elf): 2207 lib = ELFLinkData(partition, path, elf, 2208 self.tagged_paths.get_path_tag_bit(path)) 2209 self._add_lib_to_lookup_dict(lib) 2210 return lib 2211 2212 2213 def add_dlopen_dep(self, src_path, dst_path): 2214 num_matches = 0 2215 for elf_class in (ELF.ELFCLASS32, ELF.ELFCLASS64): 2216 srcs = self._get_libs_in_elf_class(elf_class, src_path) 2217 dsts = self._get_libs_in_elf_class(elf_class, dst_path) 2218 for src, dst in itertools.product(srcs, dsts): 2219 src.add_dlopen_dep(dst) 2220 num_matches += 1 2221 if num_matches == 0: 2222 raise ValueError('Failed to add dlopen dependency from {} to {}' 2223 .format(src_path, dst_path)) 2224 2225 2226 def _get_libs_in_elf_class(self, elf_class, path): 2227 result = set() 2228 if '${LIB}' in path: 2229 lib_dir = 'lib' if elf_class == ELF.ELFCLASS32 else 'lib64' 2230 path = path.replace('${LIB}', lib_dir) 2231 if path.startswith('[regex]'): 2232 patt = re.compile(path[7:]) 2233 for partition in range(NUM_PARTITIONS): 2234 lib_set = self.lib_pt[partition].get_lib_dict(elf_class) 2235 for path, lib in lib_set.items(): 2236 if patt.match(path): 2237 result.add(lib) 2238 else: 2239 for partition in range(NUM_PARTITIONS): 2240 lib_set = self.lib_pt[partition].get_lib_dict(elf_class) 2241 lib = lib_set.get(path) 2242 if lib: 2243 result.add(lib) 2244 return result 2245 2246 2247 def get_lib(self, path): 2248 for lib_set in self.lib_pt: 2249 lib = lib_set.get(path) 2250 if lib: 2251 return lib 2252 return None 2253 2254 2255 def get_libs(self, paths, report_error=None): 2256 result = set() 2257 for path in paths: 2258 lib = self.get_lib(path) 2259 if not lib: 2260 if report_error is None: 2261 raise ValueError('path not found ' + path) 2262 report_error(path) 2263 continue 2264 result.add(lib) 2265 return result 2266 2267 2268 def all_libs(self): 2269 for lib_set in self.lib_pt: 2270 for lib in lib_set.values(): 2271 yield lib 2272 2273 2274 def _compute_lib_dict(self, elf_class): 2275 res = dict() 2276 for lib_pt in self.lib_pt: 2277 res.update(lib_pt.get_lib_dict(elf_class)) 2278 return res 2279 2280 2281 @staticmethod 2282 def _compile_path_matcher(root, subdirs): 2283 dirs = [os.path.normpath(os.path.join(root, i)) for i in subdirs] 2284 patts = ['(?:' + re.escape(i) + os.sep + ')' for i in dirs] 2285 return re.compile('|'.join(patts)) 2286 2287 2288 def add_executables_in_dir(self, partition_name, partition, root, 2289 alter_partition, alter_subdirs, ignored_subdirs, 2290 scan_elf_files, unzip_files): 2291 root = os.path.abspath(root) 2292 prefix_len = len(root) + 1 2293 2294 if alter_subdirs: 2295 alter_patt = ELFLinker._compile_path_matcher(root, alter_subdirs) 2296 if ignored_subdirs: 2297 ignored_patt = ELFLinker._compile_path_matcher( 2298 root, ignored_subdirs) 2299 2300 for path, elf in scan_elf_files(root, unzip_files=unzip_files): 2301 # Ignore ELF files with unknown machine ID (eg. DSP). 2302 if elf.e_machine not in ELF.ELF_MACHINES: 2303 continue 2304 2305 # Ignore ELF files with matched path. 2306 if ignored_subdirs and ignored_patt.match(path): 2307 continue 2308 2309 short_path = os.path.join('/', partition_name, path[prefix_len:]) 2310 2311 if alter_subdirs and alter_patt.match(path): 2312 self.add_lib(alter_partition, short_path, elf) 2313 else: 2314 self.add_lib(partition, short_path, elf) 2315 2316 2317 def add_dlopen_deps(self, path): 2318 patt = re.compile('([^:]*):\\s*(.*)') 2319 with open(path, 'r') as dlopen_dep_file: 2320 for line_no, line in enumerate(dlopen_dep_file, start=1): 2321 match = patt.match(line) 2322 if not match: 2323 continue 2324 try: 2325 self.add_dlopen_dep(match.group(1), match.group(2)) 2326 except ValueError as e: 2327 print('error:{}:{}: {}.'.format(path, line_no, e), 2328 file=sys.stderr) 2329 2330 2331 def _find_exported_symbol(self, symbol, libs): 2332 """Find the shared library with the exported symbol.""" 2333 for lib in libs: 2334 if symbol in lib.elf.exported_symbols: 2335 return lib 2336 return None 2337 2338 2339 def _resolve_lib_imported_symbols(self, lib, imported_libs, generic_refs): 2340 """Resolve the imported symbols in a library.""" 2341 for symbol in lib.elf.imported_symbols: 2342 imported_lib = self._find_exported_symbol(symbol, imported_libs) 2343 if not imported_lib: 2344 lib.unresolved_symbols.add(symbol) 2345 else: 2346 lib.linked_symbols[symbol] = imported_lib 2347 if generic_refs: 2348 ref_lib = generic_refs.refs.get(imported_lib.path) 2349 if not ref_lib or not symbol in ref_lib.exported_symbols: 2350 lib.imported_ext_symbols[imported_lib].add(symbol) 2351 2352 2353 def _resolve_lib_dt_needed(self, lib, resolver): 2354 imported_libs = [] 2355 for dt_needed in lib.elf.dt_needed: 2356 dep = resolver.resolve(lib.path, dt_needed, lib.elf.dt_rpath, 2357 lib.elf.dt_runpath) 2358 if not dep: 2359 candidates = list(resolver.get_candidates( 2360 lib.path, dt_needed, lib.elf.dt_rpath, lib.elf.dt_runpath)) 2361 print('warning: {}: Missing needed library: {} Tried: {}' 2362 .format(lib.path, dt_needed, candidates), 2363 file=sys.stderr) 2364 lib.unresolved_dt_needed.append(dt_needed) 2365 continue 2366 lib.add_needed_dep(dep) 2367 imported_libs.append(dep) 2368 return imported_libs 2369 2370 2371 def _resolve_lib_deps(self, lib, resolver, generic_refs): 2372 # Resolve DT_NEEDED entries. 2373 imported_libs = self._resolve_lib_dt_needed(lib, resolver) 2374 2375 if generic_refs: 2376 for imported_lib in imported_libs: 2377 if imported_lib.path not in generic_refs.refs: 2378 # Add imported_lib to imported_ext_symbols to make sure 2379 # non-AOSP libraries are in the imported_ext_symbols key 2380 # set. 2381 lib.imported_ext_symbols[imported_lib].update() 2382 2383 # Resolve imported symbols. 2384 self._resolve_lib_imported_symbols(lib, imported_libs, generic_refs) 2385 2386 2387 def _resolve_lib_set_deps(self, lib_set, resolver, generic_refs): 2388 for lib in lib_set: 2389 self._resolve_lib_deps(lib, resolver, generic_refs) 2390 2391 2392 def _get_apex_bionic_search_paths(self, lib_dir): 2393 return ['/apex/com.android.runtime/' + lib_dir + '/bionic'] 2394 2395 2396 def _get_apex_search_paths(self, lib_dir): 2397 return ['/apex/' + name + '/' + lib_dir 2398 for name in sorted(self.apex_module_names)] 2399 2400 2401 def _get_system_search_paths(self, lib_dir): 2402 apex_lib_dirs = (self._get_apex_search_paths(lib_dir) + 2403 self._get_apex_bionic_search_paths(lib_dir)) 2404 system_lib_dirs = ['/system/' + lib_dir, '/system/product/' + lib_dir] 2405 vendor_lib_dirs = ['/vendor/' + lib_dir] 2406 return apex_lib_dirs + system_lib_dirs + vendor_lib_dirs 2407 2408 2409 def _get_vendor_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs): 2410 vendor_lib_dirs = [ 2411 '/vendor/' + lib_dir + '/hw', 2412 '/vendor/' + lib_dir + '/egl', 2413 '/vendor/' + lib_dir, 2414 ] 2415 # For degenerated VNDK libs. 2416 apex_lib_dirs = (self._get_apex_search_paths(lib_dir) + 2417 self._get_apex_bionic_search_paths(lib_dir)) 2418 system_lib_dirs = ['/system/' + lib_dir] 2419 return (vendor_lib_dirs + vndk_sp_dirs + vndk_dirs + apex_lib_dirs + 2420 system_lib_dirs) 2421 2422 2423 def _get_vndk_sp_search_paths(self, lib_dir, vndk_sp_dirs): 2424 # To find missing dependencies or LL-NDK. 2425 fallback_lib_dirs = [ 2426 '/vendor/' + lib_dir, 2427 '/system/' + lib_dir, 2428 ] 2429 fallback_lib_dirs += self._get_apex_search_paths(lib_dir) 2430 fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir) 2431 return vndk_sp_dirs + fallback_lib_dirs 2432 2433 2434 def _get_vndk_search_paths(self, lib_dir, vndk_sp_dirs, vndk_dirs): 2435 # To find missing dependencies or LL-NDK. 2436 fallback_lib_dirs = ['/system/' + lib_dir] 2437 fallback_lib_dirs += self._get_apex_search_paths(lib_dir) 2438 fallback_lib_dirs += self._get_apex_bionic_search_paths(lib_dir) 2439 return vndk_sp_dirs + vndk_dirs + fallback_lib_dirs 2440 2441 2442 def _resolve_elf_class_deps(self, lib_dir, elf_class, generic_refs): 2443 # Classify libs. 2444 vndk_lib_dirs = self.vndk_lib_dirs 2445 lib_dict = self._compute_lib_dict(elf_class) 2446 2447 system_lib_dict = self.lib_pt[PT_SYSTEM].get_lib_dict(elf_class) 2448 system_vndk_sp_libs, system_vndk_libs, system_libs = \ 2449 vndk_lib_dirs.classify_vndk_libs(system_lib_dict.values()) 2450 2451 vendor_lib_dict = self.lib_pt[PT_VENDOR].get_lib_dict(elf_class) 2452 vendor_vndk_sp_libs, vendor_vndk_libs, vendor_libs = \ 2453 vndk_lib_dirs.classify_vndk_libs(vendor_lib_dict.values()) 2454 2455 # Resolve system libs. 2456 search_paths = self._get_system_search_paths(lib_dir) 2457 resolver = ELFResolver(lib_dict, search_paths) 2458 self._resolve_lib_set_deps(system_libs, resolver, generic_refs) 2459 2460 # Resolve vndk-sp libs 2461 for version in vndk_lib_dirs: 2462 vndk_sp_dirs, vndk_dirs = \ 2463 vndk_lib_dirs.create_vndk_search_paths(lib_dir, version) 2464 vndk_sp_libs = \ 2465 system_vndk_sp_libs[version] | vendor_vndk_sp_libs[version] 2466 search_paths = self._get_vndk_sp_search_paths( 2467 lib_dir, vndk_sp_dirs) 2468 resolver = ELFResolver(lib_dict, search_paths) 2469 self._resolve_lib_set_deps(vndk_sp_libs, resolver, generic_refs) 2470 2471 # Resolve vndk libs 2472 for version in vndk_lib_dirs: 2473 vndk_sp_dirs, vndk_dirs = \ 2474 vndk_lib_dirs.create_vndk_search_paths(lib_dir, version) 2475 vndk_libs = system_vndk_libs[version] | vendor_vndk_libs[version] 2476 search_paths = self._get_vndk_search_paths( 2477 lib_dir, vndk_sp_dirs, vndk_dirs) 2478 resolver = ELFResolver(lib_dict, search_paths) 2479 self._resolve_lib_set_deps(vndk_libs, resolver, generic_refs) 2480 2481 # Resolve vendor libs. 2482 vndk_sp_dirs, vndk_dirs = vndk_lib_dirs.create_vndk_search_paths( 2483 lib_dir, self.ro_vndk_version) 2484 search_paths = self._get_vendor_search_paths( 2485 lib_dir, vndk_sp_dirs, vndk_dirs) 2486 resolver = ELFResolver(lib_dict, search_paths) 2487 self._resolve_lib_set_deps(vendor_libs, resolver, generic_refs) 2488 2489 2490 def resolve_deps(self, generic_refs=None): 2491 self._resolve_elf_class_deps('lib', ELF.ELFCLASS32, generic_refs) 2492 self._resolve_elf_class_deps('lib64', ELF.ELFCLASS64, generic_refs) 2493 2494 2495 def compute_predefined_sp_hal(self): 2496 """Find all same-process HALs.""" 2497 return set(lib for lib in self.all_libs() if lib.is_sp_hal) 2498 2499 2500 def compute_sp_lib(self, generic_refs, ignore_hidden_deps=False): 2501 def is_ll_ndk_or_sp_hal(lib): 2502 return lib.is_ll_ndk or lib.is_sp_hal 2503 2504 ll_ndk = set(lib for lib in self.all_libs() if lib.is_ll_ndk) 2505 ll_ndk_closure = self.compute_deps_closure( 2506 ll_ndk, is_ll_ndk_or_sp_hal, ignore_hidden_deps) 2507 ll_ndk_indirect = ll_ndk_closure - ll_ndk 2508 2509 def is_ll_ndk(lib): 2510 return lib.is_ll_ndk 2511 2512 sp_hal = self.compute_predefined_sp_hal() 2513 sp_hal_closure = self.compute_deps_closure( 2514 sp_hal, is_ll_ndk, ignore_hidden_deps) 2515 2516 def is_aosp_lib(lib): 2517 return (not generic_refs or 2518 generic_refs.classify_lib(lib) != GenericRefs.NEW_LIB) 2519 2520 vndk_sp_hal = set() 2521 sp_hal_dep = set() 2522 for lib in sp_hal_closure - sp_hal: 2523 if is_aosp_lib(lib): 2524 vndk_sp_hal.add(lib) 2525 else: 2526 sp_hal_dep.add(lib) 2527 2528 vndk_sp_both = ll_ndk_indirect & vndk_sp_hal 2529 ll_ndk_indirect -= vndk_sp_both 2530 vndk_sp_hal -= vndk_sp_both 2531 2532 return SPLibResult(sp_hal, sp_hal_dep, vndk_sp_hal, ll_ndk, 2533 ll_ndk_indirect, vndk_sp_both) 2534 2535 2536 def normalize_partition_tags(self, sp_hals, generic_refs): 2537 def is_system_lib_or_sp_hal(lib): 2538 return lib.is_system_lib() or lib in sp_hals 2539 2540 for lib in self.lib_pt[PT_SYSTEM].values(): 2541 if all(is_system_lib_or_sp_hal(dep) for dep in lib.deps_all): 2542 continue 2543 # If a system module is depending on a vendor shared library and 2544 # such shared library is not a SP-HAL library, then emit an error 2545 # and hide the dependency. 2546 for dep in list(lib.deps_needed_all): 2547 if not is_system_lib_or_sp_hal(dep): 2548 print('error: {}: system exe/lib must not depend on ' 2549 'vendor lib {}. Assume such dependency does ' 2550 'not exist.'.format(lib.path, dep.path), 2551 file=sys.stderr) 2552 lib.hide_needed_dep(dep) 2553 for dep in list(lib.deps_dlopen_all): 2554 if not is_system_lib_or_sp_hal(dep): 2555 print('error: {}: system exe/lib must not dlopen() ' 2556 'vendor lib {}. Assume such dependency does ' 2557 'not exist.'.format(lib.path, dep.path), 2558 file=sys.stderr) 2559 lib.hide_dlopen_dep(dep) 2560 2561 2562 def rewrite_apex_modules(self): 2563 """Rename ELF files under `/system/apex/${name}.apex` to 2564 `/apex/${name}/...` and collect apex module names.""" 2565 APEX_PREFIX = '/system/apex/' 2566 APEX_PREFIX_LEN = len(APEX_PREFIX) 2567 for lib in list(self.all_libs()): 2568 if not lib.path.startswith(APEX_PREFIX): 2569 continue 2570 2571 apex_name_end = lib.path.find('/', APEX_PREFIX_LEN) 2572 apex_name = lib.path[APEX_PREFIX_LEN:apex_name_end] 2573 self.apex_module_names.add(apex_name) 2574 2575 self._rename_lib(lib, '/apex/' + lib.path[APEX_PREFIX_LEN:]) 2576 2577 2578 @staticmethod 2579 def _parse_action_on_ineligible_lib(arg): 2580 follow = False 2581 warn = False 2582 for flag in arg.split(','): 2583 if flag == 'follow': 2584 follow = True 2585 elif flag == 'warn': 2586 warn = True 2587 elif flag == 'ignore': 2588 continue 2589 else: 2590 raise ValueError('unknown action \"{}\"'.format(flag)) 2591 return (follow, warn) 2592 2593 2594 def compute_degenerated_vndk(self, generic_refs, tagged_paths=None, 2595 action_ineligible_vndk_sp='warn', 2596 action_ineligible_vndk='warn'): 2597 # Find LL-NDK libs. 2598 ll_ndk = set(lib for lib in self.all_libs() if lib.is_ll_ndk) 2599 2600 # Find pre-defined libs. 2601 fwk_only_rs = set(lib for lib in self.all_libs() if lib.is_fwk_only_rs) 2602 predefined_vndk_sp = set( 2603 lib for lib in self.all_libs() if lib.is_vndk_sp) 2604 predefined_vndk_sp_indirect = set( 2605 lib for lib in self.all_libs() if lib.is_vndk_sp_indirect) 2606 predefined_vndk_sp_indirect_private = set( 2607 lib for lib in self.all_libs() if lib.is_vndk_sp_indirect_private) 2608 2609 # FIXME: Don't squash VNDK-SP-Indirect-Private into VNDK-SP-Indirect. 2610 predefined_vndk_sp_indirect |= predefined_vndk_sp_indirect_private 2611 2612 # Find SP-HAL libs. 2613 sp_hal = self.compute_predefined_sp_hal() 2614 2615 # Normalize partition tags. We expect many violations from the 2616 # pre-Treble world. Guess a resolution for the incorrect partition 2617 # tag. 2618 self.normalize_partition_tags(sp_hal, generic_refs) 2619 2620 # Find SP-HAL-Dep libs. 2621 def is_aosp_lib(lib): 2622 if not generic_refs: 2623 # If generic reference is not available, then assume all system 2624 # libs are AOSP libs. 2625 return lib.partition == PT_SYSTEM 2626 return generic_refs.has_same_name_lib(lib) 2627 2628 def is_not_sp_hal_dep(lib): 2629 if lib.is_ll_ndk or lib in sp_hal: 2630 return True 2631 return is_aosp_lib(lib) 2632 2633 sp_hal_dep = self.compute_deps_closure(sp_hal, is_not_sp_hal_dep, True) 2634 sp_hal_dep -= sp_hal 2635 2636 # Find VNDK-SP libs. 2637 def is_not_vndk_sp(lib): 2638 return lib.is_ll_ndk or lib in sp_hal or lib in sp_hal_dep 2639 2640 follow_ineligible_vndk_sp, warn_ineligible_vndk_sp = \ 2641 self._parse_action_on_ineligible_lib(action_ineligible_vndk_sp) 2642 vndk_sp = set() 2643 for lib in itertools.chain(sp_hal, sp_hal_dep): 2644 for dep in lib.deps_all: 2645 if is_not_vndk_sp(dep): 2646 continue 2647 if dep in predefined_vndk_sp: 2648 vndk_sp.add(dep) 2649 continue 2650 if warn_ineligible_vndk_sp: 2651 print('error: SP-HAL {} depends on non vndk-sp ' 2652 'library {}.'.format(lib.path, dep.path), 2653 file=sys.stderr) 2654 if follow_ineligible_vndk_sp: 2655 vndk_sp.add(dep) 2656 2657 # Find VNDK-SP-Indirect libs. 2658 def is_not_vndk_sp_indirect(lib): 2659 return lib.is_ll_ndk or lib in vndk_sp or lib in fwk_only_rs 2660 2661 vndk_sp_indirect = self.compute_deps_closure( 2662 vndk_sp, is_not_vndk_sp_indirect, True) 2663 vndk_sp_indirect -= vndk_sp 2664 2665 # Find unused predefined VNDK-SP libs. 2666 vndk_sp_unused = set(lib for lib in predefined_vndk_sp 2667 if VNDKLibDir.is_in_vndk_sp_dir(lib.path)) 2668 vndk_sp_unused -= vndk_sp 2669 vndk_sp_unused -= vndk_sp_indirect 2670 2671 # Find dependencies of unused predefined VNDK-SP libs. 2672 def is_not_vndk_sp_indirect_unused(lib): 2673 return is_not_vndk_sp_indirect(lib) or lib in vndk_sp_indirect 2674 vndk_sp_unused_deps = self.compute_deps_closure( 2675 vndk_sp_unused, is_not_vndk_sp_indirect_unused, True) 2676 vndk_sp_unused_deps -= vndk_sp_unused 2677 2678 vndk_sp_indirect_unused = set( 2679 lib for lib in predefined_vndk_sp_indirect 2680 if VNDKLibDir.is_in_vndk_sp_dir(lib.path)) 2681 vndk_sp_indirect_unused -= vndk_sp_indirect 2682 vndk_sp_indirect_unused -= vndk_sp_unused 2683 vndk_sp_indirect_unused |= vndk_sp_unused_deps 2684 2685 # TODO: Compute VNDK-SP-Indirect-Private. 2686 vndk_sp_indirect_private = set() 2687 2688 assert not vndk_sp & vndk_sp_indirect 2689 assert not vndk_sp_unused & vndk_sp_indirect_unused 2690 2691 # Define helper functions for vndk_sp sets. 2692 def is_vndk_sp_public(lib): 2693 return lib in vndk_sp or lib in vndk_sp_unused or \ 2694 lib in vndk_sp_indirect or \ 2695 lib in vndk_sp_indirect_unused 2696 2697 def is_vndk_sp(lib): 2698 return is_vndk_sp_public(lib) or lib in vndk_sp_indirect_private 2699 2700 def is_vndk_sp_unused(lib): 2701 return lib in vndk_sp_unused or lib in vndk_sp_indirect_unused 2702 2703 def relabel_vndk_sp_as_used(lib): 2704 assert is_vndk_sp_unused(lib) 2705 2706 if lib in vndk_sp_unused: 2707 vndk_sp_unused.remove(lib) 2708 vndk_sp.add(lib) 2709 else: 2710 vndk_sp_indirect_unused.remove(lib) 2711 vndk_sp_indirect.add(lib) 2712 2713 # Add the dependencies to vndk_sp_indirect if they are not vndk_sp. 2714 closure = self.compute_deps_closure( 2715 {lib}, lambda lib: lib not in vndk_sp_indirect_unused, True) 2716 closure.remove(lib) 2717 vndk_sp_indirect_unused.difference_update(closure) 2718 vndk_sp_indirect.update(closure) 2719 2720 # Find VNDK-SP-Ext libs. 2721 vndk_sp_ext = set() 2722 def collect_vndk_ext(libs): 2723 result = set() 2724 for lib in libs: 2725 for dep in lib.imported_ext_symbols: 2726 if dep in vndk_sp and dep not in vndk_sp_ext: 2727 result.add(dep) 2728 return result 2729 2730 candidates = collect_vndk_ext(self.lib_pt[PT_VENDOR].values()) 2731 while candidates: 2732 vndk_sp_ext |= candidates 2733 candidates = collect_vndk_ext(candidates) 2734 2735 # Find VNDK-SP-Indirect-Ext libs. 2736 vndk_sp_indirect_ext = set() 2737 def collect_vndk_sp_indirect_ext(libs): 2738 result = set() 2739 for lib in libs: 2740 exts = set(lib.imported_ext_symbols.keys()) 2741 for dep in lib.deps_all: 2742 if not is_vndk_sp_public(dep): 2743 continue 2744 if dep in vndk_sp_ext or dep in vndk_sp_indirect_ext: 2745 continue 2746 # If lib is using extended definition from deps, then we 2747 # have to make a copy of dep. 2748 if dep in exts: 2749 result.add(dep) 2750 continue 2751 # If lib is using non-predefined VNDK-SP-Indirect, then we 2752 # have to make a copy of dep. 2753 if dep not in predefined_vndk_sp and \ 2754 dep not in predefined_vndk_sp_indirect: 2755 result.add(dep) 2756 continue 2757 return result 2758 2759 def is_not_vndk_sp_indirect(lib): 2760 return lib.is_ll_ndk or lib in vndk_sp or lib in fwk_only_rs 2761 2762 candidates = collect_vndk_sp_indirect_ext(vndk_sp_ext) 2763 while candidates: 2764 vndk_sp_indirect_ext |= candidates 2765 candidates = collect_vndk_sp_indirect_ext(candidates) 2766 2767 # Find VNDK libs (a.k.a. system shared libs directly used by vendor 2768 # partition.) 2769 def is_not_vndk(lib): 2770 if lib.is_ll_ndk or is_vndk_sp_public(lib) or lib in fwk_only_rs: 2771 return True 2772 return lib.partition != PT_SYSTEM 2773 2774 def is_eligible_lib_access(lib, dep): 2775 return not tagged_paths or \ 2776 tagged_paths.is_path_visible(lib.path, dep.path) 2777 2778 follow_ineligible_vndk, warn_ineligible_vndk = \ 2779 self._parse_action_on_ineligible_lib(action_ineligible_vndk) 2780 vndk = set() 2781 extra_vendor_libs = set() 2782 def collect_vndk(vendor_libs): 2783 next_vendor_libs = set() 2784 for lib in vendor_libs: 2785 for dep in lib.deps_all: 2786 if is_vndk_sp_unused(dep): 2787 relabel_vndk_sp_as_used(dep) 2788 continue 2789 if is_not_vndk(dep): 2790 continue 2791 if not is_aosp_lib(dep): 2792 # The dependency should be copied into vendor partition 2793 # as an extra vendor lib. 2794 if dep not in extra_vendor_libs: 2795 next_vendor_libs.add(dep) 2796 extra_vendor_libs.add(dep) 2797 continue 2798 if is_eligible_lib_access(lib, dep): 2799 vndk.add(dep) 2800 continue 2801 if warn_ineligible_vndk: 2802 print('warning: vendor lib/exe {} depends on ' 2803 'ineligible framework shared lib {}.' 2804 .format(lib.path, dep.path), file=sys.stderr) 2805 if follow_ineligible_vndk: 2806 vndk.add(dep) 2807 return next_vendor_libs 2808 2809 candidates = collect_vndk(self.lib_pt[PT_VENDOR].values()) 2810 while candidates: 2811 candidates = collect_vndk(candidates) 2812 2813 vndk_indirect = self.compute_deps_closure(vndk, is_not_vndk, True) 2814 vndk_indirect -= vndk 2815 2816 def is_vndk(lib): 2817 return lib in vndk or lib in vndk_indirect 2818 2819 # Find VNDK-EXT libs (VNDK libs with extended definitions and the 2820 # extended definitions are used by the vendor modules (including 2821 # extra_vendor_libs). 2822 2823 # FIXME: DAUX libraries won't be found by the following algorithm. 2824 vndk_ext = set() 2825 2826 def collect_vndk_ext(libs): 2827 result = set() 2828 for lib in libs: 2829 for dep in lib.imported_ext_symbols: 2830 if dep in vndk and dep not in vndk_ext: 2831 result.add(dep) 2832 return result 2833 2834 candidates = collect_vndk_ext(self.lib_pt[PT_VENDOR].values()) 2835 candidates |= collect_vndk_ext(extra_vendor_libs) 2836 2837 while candidates: 2838 vndk_ext |= candidates 2839 candidates = collect_vndk_ext(candidates) 2840 2841 # Compute LL-NDK-Indirect. 2842 def is_not_ll_ndk_indirect(lib): 2843 return lib.is_ll_ndk or lib.is_sp_hal or is_vndk_sp(lib) or \ 2844 is_vndk_sp(lib) or is_vndk(lib) 2845 2846 ll_ndk_indirect = self.compute_deps_closure( 2847 ll_ndk, is_not_ll_ndk_indirect, True) 2848 ll_ndk_indirect -= ll_ndk 2849 2850 # Return the VNDK classifications. 2851 return VNDKResult( 2852 ll_ndk=ll_ndk, 2853 ll_ndk_indirect=ll_ndk_indirect, 2854 vndk_sp=vndk_sp, 2855 vndk_sp_indirect=vndk_sp_indirect, 2856 # vndk_sp_indirect_private=vndk_sp_indirect_private, 2857 vndk_sp_unused=vndk_sp_unused, 2858 vndk_sp_indirect_unused=vndk_sp_indirect_unused, 2859 vndk=vndk, 2860 vndk_indirect=vndk_indirect, 2861 # fwk_only=fwk_only, 2862 fwk_only_rs=fwk_only_rs, 2863 sp_hal=sp_hal, 2864 sp_hal_dep=sp_hal_dep, 2865 # vnd_only=vnd_only, 2866 vndk_ext=vndk_ext, 2867 vndk_sp_ext=vndk_sp_ext, 2868 vndk_sp_indirect_ext=vndk_sp_indirect_ext, 2869 extra_vendor_libs=extra_vendor_libs) 2870 2871 2872 @staticmethod 2873 def _compute_closure(root_set, is_excluded, get_successors): 2874 closure = set(root_set) 2875 stack = list(root_set) 2876 while stack: 2877 lib = stack.pop() 2878 for succ in get_successors(lib): 2879 if is_excluded(succ): 2880 continue 2881 if succ not in closure: 2882 closure.add(succ) 2883 stack.append(succ) 2884 return closure 2885 2886 2887 @classmethod 2888 def compute_deps_closure(cls, root_set, is_excluded, 2889 ignore_hidden_deps=False): 2890 get_successors = (lambda x: x.deps_good) if ignore_hidden_deps else \ 2891 (lambda x: x.deps_all) 2892 return cls._compute_closure(root_set, is_excluded, get_successors) 2893 2894 2895 @classmethod 2896 def compute_users_closure(cls, root_set, is_excluded, 2897 ignore_hidden_users=False): 2898 get_successors = (lambda x: x.users_good) if ignore_hidden_users else \ 2899 (lambda x: x.users_all) 2900 return cls._compute_closure(root_set, is_excluded, get_successors) 2901 2902 2903 @staticmethod 2904 def _create_internal(scan_elf_files, system_dirs, system_dirs_as_vendor, 2905 system_dirs_ignored, vendor_dirs, 2906 vendor_dirs_as_system, vendor_dirs_ignored, 2907 extra_deps, generic_refs, tagged_paths, 2908 vndk_lib_dirs, unzip_files): 2909 if vndk_lib_dirs is None: 2910 vndk_lib_dirs = VNDKLibDir.create_from_dirs( 2911 system_dirs, vendor_dirs) 2912 ro_vndk_version = vndk_lib_dirs.find_vendor_vndk_version(vendor_dirs) 2913 graph = ELFLinker(tagged_paths, vndk_lib_dirs, ro_vndk_version) 2914 2915 if system_dirs: 2916 for path in system_dirs: 2917 graph.add_executables_in_dir( 2918 'system', PT_SYSTEM, path, PT_VENDOR, 2919 system_dirs_as_vendor, system_dirs_ignored, 2920 scan_elf_files, unzip_files) 2921 2922 if vendor_dirs: 2923 for path in vendor_dirs: 2924 graph.add_executables_in_dir( 2925 'vendor', PT_VENDOR, path, PT_SYSTEM, 2926 vendor_dirs_as_system, vendor_dirs_ignored, 2927 scan_elf_files, unzip_files) 2928 2929 if extra_deps: 2930 for path in extra_deps: 2931 graph.add_dlopen_deps(path) 2932 2933 graph.rewrite_apex_modules() 2934 graph.resolve_deps(generic_refs) 2935 2936 return graph 2937 2938 2939 @staticmethod 2940 def create(system_dirs=None, system_dirs_as_vendor=None, 2941 system_dirs_ignored=None, vendor_dirs=None, 2942 vendor_dirs_as_system=None, vendor_dirs_ignored=None, 2943 extra_deps=None, generic_refs=None, tagged_paths=None, 2944 vndk_lib_dirs=None, unzip_files=True): 2945 return ELFLinker._create_internal( 2946 scan_elf_files, system_dirs, system_dirs_as_vendor, 2947 system_dirs_ignored, vendor_dirs, vendor_dirs_as_system, 2948 vendor_dirs_ignored, extra_deps, generic_refs, tagged_paths, 2949 vndk_lib_dirs, unzip_files) 2950 2951 2952#------------------------------------------------------------------------------ 2953# Generic Reference 2954#------------------------------------------------------------------------------ 2955 2956class GenericRefs(object): 2957 NEW_LIB = 0 2958 EXPORT_EQUAL = 1 2959 EXPORT_SUPER_SET = 2 2960 MODIFIED = 3 2961 2962 2963 def __init__(self): 2964 self.refs = dict() 2965 self._lib_names = set() 2966 2967 2968 def add(self, path, elf): 2969 self.refs[path] = elf 2970 self._lib_names.add(os.path.basename(path)) 2971 2972 2973 def _load_from_sym_dir(self, root): 2974 root = os.path.abspath(root) 2975 prefix_len = len(root) + 1 2976 for base, _, filenames in os.walk(root): 2977 for filename in filenames: 2978 if not filename.endswith('.sym'): 2979 continue 2980 path = os.path.join(base, filename) 2981 lib_path = '/' + path[prefix_len:-4] 2982 self.add(lib_path, ELF.load_dump(path)) 2983 2984 2985 @staticmethod 2986 def create_from_sym_dir(root): 2987 result = GenericRefs() 2988 result._load_from_sym_dir(root) 2989 return result 2990 2991 2992 def _load_from_image_dir(self, root, prefix): 2993 root = os.path.abspath(root) 2994 root_len = len(root) + 1 2995 for path, elf in scan_elf_files(root): 2996 self.add(os.path.join(prefix, path[root_len:]), elf) 2997 2998 2999 @staticmethod 3000 def create_from_image_dir(root, prefix): 3001 result = GenericRefs() 3002 result._load_from_image_dir(root, prefix) 3003 return result 3004 3005 3006 def classify_lib(self, lib): 3007 ref_lib = self.refs.get(lib.path) 3008 if not ref_lib: 3009 return GenericRefs.NEW_LIB 3010 exported_symbols = lib.elf.exported_symbols 3011 if exported_symbols == ref_lib.exported_symbols: 3012 return GenericRefs.EXPORT_EQUAL 3013 if exported_symbols > ref_lib.exported_symbols: 3014 return GenericRefs.EXPORT_SUPER_SET 3015 return GenericRefs.MODIFIED 3016 3017 3018 def is_equivalent_lib(self, lib): 3019 return self.classify_lib(lib) == GenericRefs.EXPORT_EQUAL 3020 3021 3022 def has_same_name_lib(self, lib): 3023 return os.path.basename(lib.path) in self._lib_names 3024 3025 3026#------------------------------------------------------------------------------ 3027# APK Dep 3028#------------------------------------------------------------------------------ 3029def _build_lib_names_dict(graph, min_name_len=6, lib_ext='.so'): 3030 names = collections.defaultdict(set) 3031 for lib in graph.all_libs(): 3032 name = os.path.basename(lib.path) 3033 root, ext = os.path.splitext(name) 3034 3035 if ext != lib_ext: 3036 continue 3037 3038 if not lib.elf.is_jni_lib(): 3039 continue 3040 3041 names[name].add(lib) 3042 names[root].add(lib) 3043 3044 if root.startswith('lib') and len(root) > min_name_len: 3045 # FIXME: libandroid.so is a JNI lib. However, many apps have 3046 # "android" as a constant string literal, thus "android" is 3047 # skipped here to reduce the false positives. 3048 # 3049 # Note: It is fine to exclude libandroid.so because it is only 3050 # a user of JNI and it does not define any JNI methods. 3051 if root != 'libandroid': 3052 names[root[3:]].add(lib) 3053 return names 3054 3055 3056def _enumerate_partition_paths(partition, root): 3057 prefix_len = len(root) + 1 3058 for base, _, files in os.walk(root): 3059 for filename in files: 3060 path = os.path.join(base, filename) 3061 if not is_accessible(path): 3062 continue 3063 android_path = posixpath.join('/', partition, path[prefix_len:]) 3064 yield (android_path, path) 3065 3066 3067def _enumerate_paths(system_dirs, vendor_dirs): 3068 for root in system_dirs: 3069 for ap, path in _enumerate_partition_paths('system', root): 3070 yield (ap, path) 3071 for root in vendor_dirs: 3072 for ap, path in _enumerate_partition_paths('vendor', root): 3073 yield (ap, path) 3074 3075 3076def scan_apk_dep(graph, system_dirs, vendor_dirs): 3077 libnames = _build_lib_names_dict(graph) 3078 results = [] 3079 3080 if str is bytes: 3081 def decode(string): # PY2 3082 return string.decode('mutf-8').encode('utf-8') 3083 else: 3084 def decode(string): # PY3 3085 return string.decode('mutf-8') 3086 3087 for ap, path in _enumerate_paths(system_dirs, vendor_dirs): 3088 # Read the dex file from various file formats 3089 try: 3090 dex_string_iter = DexFileReader.enumerate_dex_strings(path) 3091 if dex_string_iter is None: 3092 continue 3093 3094 strings = set() 3095 for string in dex_string_iter: 3096 try: 3097 strings.add(decode(string)) 3098 except UnicodeSurrogateDecodeError: 3099 pass 3100 except FileNotFoundError: 3101 continue 3102 except: 3103 print('error: Failed to parse', path, file=sys.stderr) 3104 raise 3105 3106 # Skip the file that does not call System.loadLibrary() 3107 if 'loadLibrary' not in strings: 3108 continue 3109 3110 # Collect libraries from string tables 3111 libs = set() 3112 for string in strings: 3113 try: 3114 for dep_file in libnames[string]: 3115 match = _APP_DIR_PATTERNS.match(dep_file.path) 3116 3117 # List the lib if it is not embedded in the app. 3118 if not match: 3119 libs.add(dep_file) 3120 continue 3121 3122 # Only list the embedded lib if it is in the same app. 3123 common = os.path.commonprefix([ap, dep_file.path]) 3124 if len(common) > len(match.group(0)): 3125 libs.add(dep_file) 3126 continue 3127 except KeyError: 3128 pass 3129 3130 if libs: 3131 results.append((ap, sorted_lib_path_list(libs))) 3132 3133 results.sort() 3134 return results 3135 3136 3137#------------------------------------------------------------------------------ 3138# Module Info 3139#------------------------------------------------------------------------------ 3140 3141class ModuleInfo(object): 3142 def __init__(self, json=None): 3143 if not json: 3144 self._mods = dict() 3145 return 3146 3147 mods = collections.defaultdict(set) 3148 installed_path_patt = re.compile( 3149 '.*[\\\\/]target[\\\\/]product[\\\\/][^\\\\/]+([\\\\/].*)$') 3150 for module in json.values(): 3151 for path in module['installed']: 3152 match = installed_path_patt.match(path) 3153 if match: 3154 for path in module['path']: 3155 mods[match.group(1)].add(path) 3156 self._mods = {installed_path: sorted(src_dirs) 3157 for installed_path, src_dirs in mods.items()} 3158 3159 3160 def get_module_path(self, installed_path): 3161 return self._mods.get(installed_path, []) 3162 3163 3164 @staticmethod 3165 def load(f): 3166 return ModuleInfo(json.load(f)) 3167 3168 3169 @staticmethod 3170 def load_from_path_or_default(path): 3171 if not path: 3172 return ModuleInfo() 3173 with open(path, 'r') as f: 3174 return ModuleInfo.load(f) 3175 3176 3177#------------------------------------------------------------------------------ 3178# Commands 3179#------------------------------------------------------------------------------ 3180 3181class Command(object): 3182 def __init__(self, name, help): 3183 self.name = name 3184 self.help = help 3185 3186 def add_argparser_options(self, parser): 3187 pass 3188 3189 def main(self, args): 3190 return 0 3191 3192 3193class ELFDumpCommand(Command): 3194 def __init__(self): 3195 super(ELFDumpCommand, self).__init__( 3196 'elfdump', help='Dump ELF .dynamic section') 3197 3198 3199 def add_argparser_options(self, parser): 3200 parser.add_argument('path', help='path to an ELF file') 3201 3202 3203 def main(self, args): 3204 try: 3205 ELF.load(args.path).dump() 3206 except ELFError as e: 3207 print('error: {}: Bad ELF file ({})'.format(args.path, e), 3208 file=sys.stderr) 3209 sys.exit(1) 3210 return 0 3211 3212 3213class CreateGenericRefCommand(Command): 3214 def __init__(self): 3215 super(CreateGenericRefCommand, self).__init__( 3216 'create-generic-ref', help='Create generic references') 3217 3218 3219 def add_argparser_options(self, parser): 3220 parser.add_argument('dir') 3221 3222 parser.add_argument('-o', '--output', required=True, 3223 help='output directory') 3224 3225 3226 def main(self, args): 3227 root = os.path.abspath(args.dir) 3228 print(root) 3229 prefix_len = len(root) + 1 3230 for path, elf in scan_elf_files(root): 3231 name = path[prefix_len:] 3232 print('Processing:', name, file=sys.stderr) 3233 out = os.path.join(args.output, name) + '.sym' 3234 makedirs(os.path.dirname(out), exist_ok=True) 3235 with open(out, 'w') as f: 3236 elf.dump(f) 3237 return 0 3238 3239 3240class ELFGraphCommand(Command): 3241 def add_argparser_options(self, parser, is_tag_file_required=None): 3242 parser.add_argument( 3243 '--load-extra-deps', action='append', 3244 help='load extra module dependencies') 3245 3246 parser.add_argument( 3247 '--system', action='append', 3248 help='path to system partition contents') 3249 3250 parser.add_argument( 3251 '--vendor', action='append', 3252 help='path to vendor partition contents') 3253 3254 parser.add_argument( 3255 '--system-dir-as-vendor', action='append', 3256 help='sub directory of system partition that has vendor files') 3257 3258 parser.add_argument( 3259 '--system-dir-ignored', action='append', 3260 help='sub directory of system partition that must be ignored') 3261 3262 parser.add_argument( 3263 '--vendor-dir-as-system', action='append', 3264 help='sub directory of vendor partition that has system files') 3265 3266 parser.add_argument( 3267 '--vendor-dir-ignored', action='append', 3268 help='sub directory of vendor partition that must be ignored') 3269 3270 parser.add_argument( 3271 '--load-generic-refs', 3272 help='compare with generic reference symbols') 3273 3274 parser.add_argument( 3275 '--aosp-system', 3276 help='compare with AOSP generic system image directory') 3277 3278 parser.add_argument( 3279 '--unzip-files', action='store_true', default=True, 3280 help='scan ELF files in zip files') 3281 3282 parser.add_argument( 3283 '--no-unzip-files', action='store_false', dest='unzip_files', 3284 help='do not scan ELF files in zip files') 3285 3286 parser.add_argument( 3287 '--tag-file', required=is_tag_file_required, 3288 help='lib tag file') 3289 3290 3291 def get_generic_refs_from_args(self, args): 3292 if args.load_generic_refs: 3293 return GenericRefs.create_from_sym_dir(args.load_generic_refs) 3294 if args.aosp_system: 3295 return GenericRefs.create_from_image_dir( 3296 args.aosp_system, '/system') 3297 return None 3298 3299 3300 def _check_arg_dir_exists(self, arg_name, dirs): 3301 for path in dirs: 3302 if not os.path.exists(path): 3303 print('error: Failed to find the directory "{}" specified in ' 3304 '"{}"'.format(path, arg_name), file=sys.stderr) 3305 sys.exit(1) 3306 if not os.path.isdir(path): 3307 print('error: Path "{}" specified in {} is not a directory' 3308 .format(path, arg_name), file=sys.stderr) 3309 sys.exit(1) 3310 3311 3312 def check_dirs_from_args(self, args): 3313 self._check_arg_dir_exists('--system', args.system) 3314 self._check_arg_dir_exists('--vendor', args.vendor) 3315 3316 3317 def create_from_args(self, args): 3318 self.check_dirs_from_args(args) 3319 3320 generic_refs = self.get_generic_refs_from_args(args) 3321 3322 vndk_lib_dirs = VNDKLibDir.create_from_dirs(args.system, args.vendor) 3323 3324 if args.tag_file: 3325 tagged_paths = TaggedPathDict.create_from_csv_path( 3326 args.tag_file, vndk_lib_dirs) 3327 else: 3328 tagged_paths = None 3329 3330 graph = ELFLinker.create(args.system, args.system_dir_as_vendor, 3331 args.system_dir_ignored, 3332 args.vendor, args.vendor_dir_as_system, 3333 args.vendor_dir_ignored, 3334 args.load_extra_deps, 3335 generic_refs=generic_refs, 3336 tagged_paths=tagged_paths, 3337 unzip_files=args.unzip_files) 3338 3339 return (generic_refs, graph, tagged_paths, vndk_lib_dirs) 3340 3341 3342class VNDKCommandBase(ELFGraphCommand): 3343 def add_argparser_options(self, parser): 3344 super(VNDKCommandBase, self).add_argparser_options(parser) 3345 3346 parser.add_argument( 3347 '--no-default-dlopen-deps', action='store_true', 3348 help='do not add default dlopen dependencies') 3349 3350 parser.add_argument( 3351 '--action-ineligible-vndk-sp', default='warn', 3352 help='action when a sp-hal uses non-vndk-sp libs ' 3353 '(option: follow,warn,ignore)') 3354 3355 parser.add_argument( 3356 '--action-ineligible-vndk', default='warn', 3357 help='action when a vendor lib/exe uses fwk-only libs ' 3358 '(option: follow,warn,ignore)') 3359 3360 3361 def create_from_args(self, args): 3362 """Create all essential data structures for VNDK computation.""" 3363 3364 generic_refs, graph, tagged_paths, vndk_lib_dirs = \ 3365 super(VNDKCommandBase, self).create_from_args(args) 3366 3367 if not args.no_default_dlopen_deps: 3368 script_dir = os.path.dirname(os.path.abspath(__file__)) 3369 minimum_dlopen_deps = os.path.join( 3370 script_dir, 'datasets', 'minimum_dlopen_deps.txt') 3371 graph.add_dlopen_deps(minimum_dlopen_deps) 3372 3373 return (generic_refs, graph, tagged_paths, vndk_lib_dirs) 3374 3375 3376class VNDKCommand(VNDKCommandBase): 3377 def __init__(self): 3378 super(VNDKCommand, self).__init__( 3379 'vndk', help='Compute VNDK libraries set') 3380 3381 3382 def add_argparser_options(self, parser): 3383 super(VNDKCommand, self).add_argparser_options(parser) 3384 3385 parser.add_argument( 3386 '--warn-incorrect-partition', action='store_true', 3387 help='warn about libraries only have cross partition linkages') 3388 3389 parser.add_argument( 3390 '--full', action='store_true', 3391 help='print all classification') 3392 3393 parser.add_argument( 3394 '--output-format', default='tag', 3395 help='output format for vndk classification') 3396 3397 parser.add_argument( 3398 '--file-size-output', 3399 help='output file for calculated file sizes') 3400 3401 3402 def _warn_incorrect_partition_lib_set(self, lib_set, partition, error_msg): 3403 for lib in lib_set.values(): 3404 if not lib.num_users: 3405 continue 3406 if all((user.partition != partition for user in lib.users_all)): 3407 print(error_msg.format(lib.path), file=sys.stderr) 3408 3409 3410 def _warn_incorrect_partition(self, graph): 3411 self._warn_incorrect_partition_lib_set( 3412 graph.lib_pt[PT_VENDOR], PT_VENDOR, 3413 'warning: {}: This is a vendor library with framework-only ' 3414 'usages.') 3415 3416 self._warn_incorrect_partition_lib_set( 3417 graph.lib_pt[PT_SYSTEM], PT_SYSTEM, 3418 'warning: {}: This is a framework library with vendor-only ' 3419 'usages.') 3420 3421 3422 @staticmethod 3423 def _extract_simple_vndk_result(vndk_result): 3424 field_name_tags = [ 3425 ('vndk_sp', 'vndk_sp'), 3426 ('vndk_sp_unused', 'vndk_sp'), 3427 ('vndk_sp_indirect', 'vndk_sp'), 3428 ('vndk_sp_indirect_unused', 'vndk_sp'), 3429 ('vndk_sp_indirect_private', 'vndk_sp'), 3430 3431 ('vndk_sp_ext', 'vndk_sp_ext'), 3432 ('vndk_sp_indirect_ext', 'vndk_sp_ext'), 3433 3434 ('vndk_ext', 'extra_vendor_libs'), 3435 ('extra_vendor_libs', 'extra_vendor_libs'), 3436 ] 3437 results = SimpleVNDKResult() 3438 for field_name, tag in field_name_tags: 3439 getattr(results, tag).update(getattr(vndk_result, field_name)) 3440 return results 3441 3442 3443 def _print_tags(self, vndk_lib, full, file=sys.stdout): 3444 if full: 3445 result_tags = _VNDK_RESULT_FIELD_NAMES 3446 results = vndk_lib 3447 else: 3448 # Simplified VNDK output with only three sets. 3449 result_tags = _SIMPLE_VNDK_RESULT_FIELD_NAMES 3450 results = self._extract_simple_vndk_result(vndk_lib) 3451 3452 for tag in result_tags: 3453 libs = getattr(results, tag) 3454 tag += ':' 3455 for lib in sorted_lib_path_list(libs): 3456 print(tag, lib, file=file) 3457 3458 3459 def _print_make(self, vndk_lib, file=sys.stdout): 3460 def get_module_name(path): 3461 return os.path.splitext(os.path.basename(path))[0] 3462 3463 def get_module_names(lib_set): 3464 return sorted({get_module_name(lib.path) for lib in lib_set}) 3465 3466 results = self._extract_simple_vndk_result(vndk_lib) 3467 vndk_sp = get_module_names(results.vndk_sp) 3468 vndk_sp_ext = get_module_names(results.vndk_sp_ext) 3469 extra_vendor_libs = get_module_names(results.extra_vendor_libs) 3470 3471 def format_module_names(module_names): 3472 return '\\\n ' + ' \\\n '.join(module_names) 3473 3474 script_dir = os.path.dirname(os.path.abspath(__file__)) 3475 template_path = os.path.join(script_dir, 'templates', 'vndk.txt') 3476 with open(template_path, 'r') as f: 3477 template = f.read() 3478 3479 template = template.replace('##_VNDK_SP_##', 3480 format_module_names(vndk_sp)) 3481 template = template.replace('##_VNDK_SP_EXT_##', 3482 format_module_names(vndk_sp_ext)) 3483 template = template.replace('##_EXTRA_VENDOR_LIBS_##', 3484 format_module_names(extra_vendor_libs)) 3485 3486 file.write(template) 3487 3488 3489 def _print_file_size_output(self, graph, vndk_lib, file=sys.stderr): 3490 def collect_tags(lib): 3491 tags = [] 3492 for field_name in _VNDK_RESULT_FIELD_NAMES: 3493 if lib in getattr(vndk_lib, field_name): 3494 tags.append(field_name) 3495 return ' '.join(tags) 3496 3497 writer = csv.writer(file, lineterminator='\n') 3498 writer.writerow(('Path', 'Tag', 'File size', 'RO segment file size', 3499 'RO segment mem size', 'RW segment file size', 3500 'RW segment mem size')) 3501 3502 # Print the file size of all ELF files. 3503 for lib in sorted(graph.all_libs()): 3504 writer.writerow(( 3505 lib.path, collect_tags(lib), lib.elf.file_size, 3506 lib.elf.ro_seg_file_size, lib.elf.ro_seg_mem_size, 3507 lib.elf.rw_seg_file_size, lib.elf.rw_seg_mem_size)) 3508 3509 # Calculate the summation of each sets. 3510 def calc_total_size(lib_set): 3511 total_file_size = 0 3512 total_ro_seg_file_size = 0 3513 total_ro_seg_mem_size = 0 3514 total_rw_seg_file_size = 0 3515 total_rw_seg_mem_size = 0 3516 3517 for lib in lib_set: 3518 total_file_size += lib.elf.file_size 3519 total_ro_seg_file_size += lib.elf.ro_seg_file_size 3520 total_ro_seg_mem_size += lib.elf.ro_seg_mem_size 3521 total_rw_seg_file_size += lib.elf.rw_seg_file_size 3522 total_rw_seg_mem_size += lib.elf.rw_seg_mem_size 3523 3524 return [total_file_size, total_ro_seg_file_size, 3525 total_ro_seg_mem_size, total_rw_seg_file_size, 3526 total_rw_seg_mem_size] 3527 3528 SEPARATOR = ('----------', None, None, None, None, None, None) 3529 3530 writer.writerow(SEPARATOR) 3531 for tag in _VNDK_RESULT_FIELD_NAMES: 3532 lib_set = getattr(vndk_lib, tag) 3533 lib_set = [lib for lib in lib_set if lib.elf.is_32bit] 3534 total = calc_total_size(lib_set) 3535 writer.writerow(['Subtotal ' + tag + ' (32-bit)', None] + total) 3536 3537 writer.writerow(SEPARATOR) 3538 for tag in _VNDK_RESULT_FIELD_NAMES: 3539 lib_set = getattr(vndk_lib, tag) 3540 lib_set = [lib for lib in lib_set if not lib.elf.is_32bit] 3541 total = calc_total_size(lib_set) 3542 writer.writerow(['Subtotal ' + tag + ' (64-bit)', None] + total) 3543 3544 writer.writerow(SEPARATOR) 3545 for tag in _VNDK_RESULT_FIELD_NAMES: 3546 total = calc_total_size(getattr(vndk_lib, tag)) 3547 writer.writerow(['Subtotal ' + tag + ' (both)', None] + total) 3548 3549 # Calculate the summation of all ELF files. 3550 writer.writerow(SEPARATOR) 3551 writer.writerow(['Total', None] + calc_total_size(graph.all_libs())) 3552 3553 3554 def main(self, args): 3555 generic_refs, graph, tagged_paths, _ = self.create_from_args(args) 3556 3557 if args.warn_incorrect_partition: 3558 self._warn_incorrect_partition(graph) 3559 3560 # Compute vndk heuristics. 3561 vndk_lib = graph.compute_degenerated_vndk( 3562 generic_refs, tagged_paths, args.action_ineligible_vndk_sp, 3563 args.action_ineligible_vndk) 3564 3565 # Print results. 3566 if args.output_format == 'make': 3567 self._print_make(vndk_lib) 3568 else: 3569 self._print_tags(vndk_lib, args.full) 3570 3571 # Calculate and print file sizes. 3572 if args.file_size_output: 3573 with open(args.file_size_output, 'w') as fp: 3574 self._print_file_size_output(graph, vndk_lib, file=fp) 3575 return 0 3576 3577 3578class DepsInsightCommand(VNDKCommandBase): 3579 def __init__(self): 3580 super(DepsInsightCommand, self).__init__( 3581 'deps-insight', help='Generate HTML to show dependencies') 3582 3583 3584 def add_argparser_options(self, parser): 3585 super(DepsInsightCommand, self).add_argparser_options(parser) 3586 3587 parser.add_argument('--module-info') 3588 3589 parser.add_argument('-o', '--output', required=True, 3590 help='output directory') 3591 3592 3593 @staticmethod 3594 def serialize_data(libs, vndk_lib, module_info): 3595 strs = [] 3596 strs_dict = dict() 3597 3598 libs.sort(key=lambda lib: lib.path) 3599 libs_dict = {lib: i for i, lib in enumerate(libs)} 3600 3601 def get_str_idx(s): 3602 try: 3603 return strs_dict[s] 3604 except KeyError: 3605 idx = len(strs) 3606 strs_dict[s] = idx 3607 strs.append(s) 3608 return idx 3609 3610 def collect_path_sorted_lib_idxs(libs): 3611 return [libs_dict[lib] for lib in sorted(libs)] 3612 3613 def collect_deps(lib): 3614 queue = list(lib.deps_all) 3615 visited = set(queue) 3616 visited.add(lib) 3617 deps = [] 3618 3619 # Traverse dependencies with breadth-first search. 3620 while queue: 3621 # Collect dependencies for next queue. 3622 next_queue = [] 3623 for lib in queue: 3624 for dep in lib.deps_all: 3625 if dep not in visited: 3626 next_queue.append(dep) 3627 visited.add(dep) 3628 3629 # Append current queue to result. 3630 deps.append(collect_path_sorted_lib_idxs(queue)) 3631 3632 queue = next_queue 3633 3634 return deps 3635 3636 def collect_source_dir_paths(lib): 3637 return [get_str_idx(path) 3638 for path in module_info.get_module_path(lib.path)] 3639 3640 def collect_tags(lib): 3641 tags = [] 3642 for field_name in _VNDK_RESULT_FIELD_NAMES: 3643 if lib in getattr(vndk_lib, field_name): 3644 tags.append(get_str_idx(field_name)) 3645 return tags 3646 3647 mods = [] 3648 for lib in libs: 3649 mods.append([get_str_idx(lib.path), 3650 32 if lib.elf.is_32bit else 64, 3651 collect_tags(lib), 3652 collect_deps(lib), 3653 collect_path_sorted_lib_idxs(lib.users_all), 3654 collect_source_dir_paths(lib)]) 3655 3656 return (strs, mods) 3657 3658 3659 def main(self, args): 3660 generic_refs, graph, tagged_paths, _ = self.create_from_args(args) 3661 3662 module_info = ModuleInfo.load_from_path_or_default(args.module_info) 3663 3664 # Compute vndk heuristics. 3665 vndk_lib = graph.compute_degenerated_vndk( 3666 generic_refs, tagged_paths, args.action_ineligible_vndk_sp, 3667 args.action_ineligible_vndk) 3668 3669 # Serialize data. 3670 strs, mods = self.serialize_data( 3671 list(graph.all_libs()), vndk_lib, module_info) 3672 3673 # Generate output files. 3674 makedirs(args.output, exist_ok=True) 3675 script_dir = os.path.dirname(os.path.abspath(__file__)) 3676 for name in ('index.html', 'insight.css', 'insight.js'): 3677 shutil.copyfile( 3678 os.path.join(script_dir, 'assets', 'insight', name), 3679 os.path.join(args.output, name)) 3680 3681 with open(os.path.join(args.output, 'insight-data.js'), 'w') as f: 3682 f.write('''(function () { 3683 var strs = ''' + json.dumps(strs) + '''; 3684 var mods = ''' + json.dumps(mods) + '''; 3685 insight.init(document, strs, mods); 3686})();''') 3687 3688 return 0 3689 3690 3691class DepsCommand(ELFGraphCommand): 3692 def __init__(self): 3693 super(DepsCommand, self).__init__( 3694 'deps', help='Print binary dependencies for debugging') 3695 3696 3697 def add_argparser_options(self, parser): 3698 super(DepsCommand, self).add_argparser_options(parser) 3699 3700 parser.add_argument( 3701 '--revert', action='store_true', 3702 help='print usage dependency') 3703 3704 parser.add_argument( 3705 '--leaf', action='store_true', 3706 help='print binaries without dependencies or usages') 3707 3708 parser.add_argument( 3709 '--symbols', action='store_true', 3710 help='print symbols') 3711 3712 parser.add_argument( 3713 '--path-filter', 3714 help='filter paths by a regular expression') 3715 3716 parser.add_argument('--module-info') 3717 3718 3719 def main(self, args): 3720 _, graph, _, _ = self.create_from_args(args) 3721 3722 module_info = ModuleInfo.load_from_path_or_default(args.module_info) 3723 3724 if args.path_filter: 3725 path_filter = re.compile(args.path_filter) 3726 else: 3727 path_filter = None 3728 3729 if args.symbols: 3730 def collect_symbols(user, definer): 3731 return user.get_dep_linked_symbols(definer) 3732 else: 3733 def collect_symbols(user, definer): 3734 return () 3735 3736 results = [] 3737 for partition in range(NUM_PARTITIONS): 3738 for name, lib in graph.lib_pt[partition].items(): 3739 if path_filter and not path_filter.match(name): 3740 continue 3741 3742 data = [] 3743 if args.revert: 3744 for assoc_lib in sorted(lib.users_all): 3745 data.append((assoc_lib.path, 3746 collect_symbols(assoc_lib, lib))) 3747 else: 3748 for assoc_lib in sorted(lib.deps_all): 3749 data.append((assoc_lib.path, 3750 collect_symbols(lib, assoc_lib))) 3751 results.append((name, data)) 3752 results.sort() 3753 3754 if args.leaf: 3755 for name, deps in results: 3756 if not deps: 3757 print(name) 3758 else: 3759 delimiter = '' 3760 for name, assoc_libs in results: 3761 print(delimiter, end='') 3762 delimiter = '\n' 3763 3764 print(name) 3765 for module_path in module_info.get_module_path(name): 3766 print('\tMODULE_PATH:', module_path) 3767 for assoc_lib, symbols in assoc_libs: 3768 print('\t' + assoc_lib) 3769 for module_path in module_info.get_module_path(assoc_lib): 3770 print('\t\tMODULE_PATH:', module_path) 3771 for symbol in symbols: 3772 print('\t\t' + symbol) 3773 return 0 3774 3775 3776class DepsClosureCommand(ELFGraphCommand): 3777 def __init__(self): 3778 super(DepsClosureCommand, self).__init__( 3779 'deps-closure', help='Find transitive closure of dependencies') 3780 3781 3782 def add_argparser_options(self, parser): 3783 super(DepsClosureCommand, self).add_argparser_options(parser) 3784 3785 parser.add_argument('lib', nargs='*', 3786 help='root set of the shared libraries') 3787 3788 parser.add_argument('--exclude-lib', action='append', default=[], 3789 help='libraries to be excluded') 3790 3791 parser.add_argument('--exclude-ndk', action='store_true', 3792 help='exclude ndk libraries') 3793 3794 parser.add_argument('--revert', action='store_true', 3795 help='print usage dependency') 3796 3797 parser.add_argument('--enumerate', action='store_true', 3798 help='print closure for each lib instead of union') 3799 3800 3801 def print_deps_closure(self, root_libs, graph, is_excluded_libs, 3802 is_reverted, indent): 3803 if is_reverted: 3804 closure = graph.compute_users_closure(root_libs, is_excluded_libs) 3805 else: 3806 closure = graph.compute_deps_closure(root_libs, is_excluded_libs) 3807 3808 for lib in sorted_lib_path_list(closure): 3809 print(indent + lib) 3810 3811 3812 def main(self, args): 3813 _, graph, _, _ = self.create_from_args(args) 3814 3815 # Find root/excluded libraries by their paths. 3816 def report_error(path): 3817 print('error: no such lib: {}'.format(path), file=sys.stderr) 3818 root_libs = graph.get_libs(args.lib, report_error) 3819 excluded_libs = graph.get_libs(args.exclude_lib, report_error) 3820 3821 # Define the exclusion filter. 3822 if args.exclude_ndk: 3823 def is_excluded_libs(lib): 3824 return lib.is_ll_ndk or lib in excluded_libs 3825 else: 3826 def is_excluded_libs(lib): 3827 return lib in excluded_libs 3828 3829 if not args.enumerate: 3830 self.print_deps_closure(root_libs, graph, is_excluded_libs, 3831 args.revert, '') 3832 else: 3833 if not root_libs: 3834 root_libs = list(graph.all_libs()) 3835 for lib in sorted(root_libs): 3836 print(lib.path) 3837 self.print_deps_closure({lib}, graph, is_excluded_libs, 3838 args.revert, '\t') 3839 return 0 3840 3841 3842class DepsUnresolvedCommand(ELFGraphCommand): 3843 def __init__(self): 3844 super(DepsUnresolvedCommand, self).__init__( 3845 'deps-unresolved', 3846 help='Show unresolved dt_needed entries or symbols') 3847 3848 3849 def add_argparser_options(self, parser): 3850 super(DepsUnresolvedCommand, self).add_argparser_options(parser) 3851 parser.add_argument('--module-info') 3852 parser.add_argument('--path-filter') 3853 3854 3855 def _dump_unresolved(self, lib, module_info, delimiter): 3856 if not lib.unresolved_symbols and not lib.unresolved_dt_needed: 3857 return 3858 3859 print(delimiter, end='') 3860 print(lib.path) 3861 for module_path in module_info.get_module_path(lib.path): 3862 print('\tMODULE_PATH:', module_path) 3863 for dt_needed in sorted(lib.unresolved_dt_needed): 3864 print('\tUNRESOLVED_DT_NEEDED:', dt_needed) 3865 for symbol in sorted(lib.unresolved_symbols): 3866 print('\tUNRESOLVED_SYMBOL:', symbol) 3867 3868 3869 def main(self, args): 3870 _, graph, _, _ = self.create_from_args(args) 3871 module_info = ModuleInfo.load_from_path_or_default(args.module_info) 3872 3873 libs = graph.all_libs() 3874 if args.path_filter: 3875 path_filter = re.compile(args.path_filter) 3876 libs = [lib for lib in libs if path_filter.match(lib.path)] 3877 3878 delimiter = '' 3879 for lib in sorted(libs): 3880 self._dump_unresolved(lib, module_info, delimiter) 3881 delimiter = '\n' 3882 3883 3884class ApkDepsCommand(ELFGraphCommand): 3885 def __init__(self): 3886 super(ApkDepsCommand, self).__init__( 3887 'apk-deps', help='Print APK dependencies for debugging') 3888 3889 3890 def main(self, args): 3891 _, graph, _, _ = self.create_from_args(args) 3892 3893 apk_deps = scan_apk_dep(graph, args.system, args.vendor) 3894 3895 for apk_path, dep_paths in apk_deps: 3896 print(apk_path) 3897 for dep_path in dep_paths: 3898 print('\t' + dep_path) 3899 3900 return 0 3901 3902 3903class CheckDepCommandBase(ELFGraphCommand): 3904 def __init__(self, *args, **kwargs): 3905 super(CheckDepCommandBase, self).__init__(*args, **kwargs) 3906 self.delimiter = '' 3907 3908 3909 def add_argparser_options(self, parser): 3910 super(CheckDepCommandBase, self).add_argparser_options(parser) 3911 parser.add_argument('--module-info') 3912 3913 3914 def _print_delimiter(self): 3915 print(self.delimiter, end='') 3916 self.delimiter = '\n' 3917 3918 3919 def _dump_dep(self, lib, bad_deps, module_info): 3920 self._print_delimiter() 3921 print(lib.path) 3922 for module_path in module_info.get_module_path(lib.path): 3923 print('\tMODULE_PATH:', module_path) 3924 for dep in sorted(bad_deps): 3925 print('\t' + dep.path) 3926 for module_path in module_info.get_module_path(dep.path): 3927 print('\t\tMODULE_PATH:', module_path) 3928 for symbol in lib.get_dep_linked_symbols(dep): 3929 print('\t\t' + symbol) 3930 3931 3932 def _dump_apk_dep(self, apk_path, bad_deps, module_info): 3933 self._print_delimiter() 3934 print(apk_path) 3935 for module_path in module_info.get_module_path(apk_path): 3936 print('\tMODULE_PATH:', module_path) 3937 for dep_path in sorted(bad_deps): 3938 print('\t' + dep_path) 3939 for module_path in module_info.get_module_path(dep_path): 3940 print('\t\tMODULE_PATH:', module_path) 3941 3942 3943class CheckDepCommand(CheckDepCommandBase): 3944 def __init__(self): 3945 super(CheckDepCommand, self).__init__( 3946 'check-dep', help='Check the eligible dependencies') 3947 3948 3949 def add_argparser_options(self, parser): 3950 super(CheckDepCommand, self).add_argparser_options(parser) 3951 3952 group = parser.add_mutually_exclusive_group() 3953 3954 group.add_argument('--check-apk', action='store_true', default=False, 3955 help='Check JNI dependencies in APK files') 3956 3957 group.add_argument('--no-check-apk', action='store_false', 3958 dest='check_apk', 3959 help='Do not check JNI dependencies in APK files') 3960 3961 group = parser.add_mutually_exclusive_group() 3962 3963 group.add_argument('--check-dt-needed-ordering', 3964 action='store_true', default=False, 3965 help='Check ordering of DT_NEEDED entries') 3966 3967 group.add_argument('--no-check-dt-needed-ordering', 3968 action='store_false', 3969 dest='check_dt_needed_ordering', 3970 help='Do not check ordering of DT_NEEDED entries') 3971 3972 3973 def _check_vendor_dep(self, graph, tagged_libs, lib_properties, 3974 module_info): 3975 """Check whether vendor libs are depending on non-eligible libs.""" 3976 num_errors = 0 3977 3978 vendor_libs = set(graph.lib_pt[PT_VENDOR].values()) 3979 3980 eligible_libs = (tagged_libs.ll_ndk | tagged_libs.vndk_sp | 3981 tagged_libs.vndk_sp_indirect | tagged_libs.vndk) 3982 3983 for lib in sorted(vendor_libs): 3984 bad_deps = set() 3985 3986 # Check whether vendor modules depend on extended NDK symbols. 3987 for dep, symbols in lib.imported_ext_symbols.items(): 3988 if dep.is_ll_ndk: 3989 num_errors += 1 3990 bad_deps.add(dep) 3991 for symbol in symbols: 3992 print('error: vendor lib "{}" depends on extended ' 3993 'NDK symbol "{}" from "{}".' 3994 .format(lib.path, symbol, dep.path), 3995 file=sys.stderr) 3996 3997 # Check whether vendor modules depend on ineligible libs. 3998 for dep in lib.deps_all: 3999 if dep not in vendor_libs and dep not in eligible_libs: 4000 num_errors += 1 4001 bad_deps.add(dep) 4002 4003 dep_name = os.path.splitext(os.path.basename(dep.path))[0] 4004 dep_properties = lib_properties.get(dep_name) 4005 if not dep_properties.vendor_available: 4006 print('error: vendor lib "{}" depends on non-eligible ' 4007 'lib "{}".'.format(lib.path, dep.path), 4008 file=sys.stderr) 4009 elif dep_properties.vndk_sp: 4010 print('error: vendor lib "{}" depends on vndk-sp "{}" ' 4011 'but it must be copied to ' 4012 '/system/lib[64]/vndk-sp.' 4013 .format(lib.path, dep.path), 4014 file=sys.stderr) 4015 elif dep_properties.vndk: 4016 print('error: vendor lib "{}" depends on vndk "{}" ' 4017 'but it must be copied to /system/lib[64]/vndk.' 4018 .format(lib.path, dep.path), 4019 file=sys.stderr) 4020 else: 4021 print('error: vendor lib "{}" depends on ' 4022 'vendor_available "{}" but it must be copied to ' 4023 '/vendor/lib[64].'.format(lib.path, dep.path), 4024 file=sys.stderr) 4025 4026 if bad_deps: 4027 self._dump_dep(lib, bad_deps, module_info) 4028 4029 return num_errors 4030 4031 4032 def _check_dt_needed_ordering(self, graph): 4033 """Check DT_NEEDED entries order of all libraries""" 4034 4035 num_errors = 0 4036 4037 def _is_libc_prior_to_libdl(lib): 4038 dt_needed = lib.elf.dt_needed 4039 try: 4040 return dt_needed.index('libc.so') < dt_needed.index('libdl.so') 4041 except ValueError: 4042 return True 4043 4044 for lib in sorted(graph.all_libs()): 4045 if _is_libc_prior_to_libdl(lib): 4046 continue 4047 4048 print('error: The ordering of DT_NEEDED entries in "{}" may be ' 4049 'problematic. libc.so must be prior to libdl.so. ' 4050 'But found: {}.' 4051 .format(lib.path, lib.elf.dt_needed), file=sys.stderr) 4052 4053 num_errors += 1 4054 4055 return num_errors 4056 4057 4058 def _check_apk_dep(self, graph, system_dirs, vendor_dirs, module_info): 4059 num_errors = 0 4060 4061 def is_in_system_partition(path): 4062 return path.startswith('/system/') or \ 4063 path.startswith('/product/') or \ 4064 path.startswith('/oem/') 4065 4066 apk_deps = scan_apk_dep(graph, system_dirs, vendor_dirs) 4067 4068 for apk_path, dep_paths in apk_deps: 4069 apk_in_system = is_in_system_partition(apk_path) 4070 bad_deps = [] 4071 for dep_path in dep_paths: 4072 dep_in_system = is_in_system_partition(dep_path) 4073 if apk_in_system != dep_in_system: 4074 bad_deps.append(dep_path) 4075 print('error: apk "{}" has cross-partition dependency ' 4076 'lib "{}".'.format(apk_path, dep_path), 4077 file=sys.stderr) 4078 num_errors += 1 4079 if bad_deps: 4080 self._dump_apk_dep(apk_path, sorted(bad_deps), module_info) 4081 return num_errors 4082 4083 4084 def main(self, args): 4085 generic_refs, graph, tagged_paths, vndk_lib_dirs = \ 4086 self.create_from_args(args) 4087 4088 tagged_paths = TaggedPathDict.create_from_csv_path( 4089 args.tag_file, vndk_lib_dirs) 4090 tagged_libs = TaggedLibDict.create_from_graph( 4091 graph, tagged_paths, generic_refs) 4092 4093 module_info = ModuleInfo.load_from_path_or_default(args.module_info) 4094 4095 lib_properties_path = \ 4096 LibProperties.get_lib_properties_file_path(args.tag_file) 4097 lib_properties = \ 4098 LibProperties.load_from_path_or_default(lib_properties_path) 4099 4100 num_errors = self._check_vendor_dep(graph, tagged_libs, lib_properties, 4101 module_info) 4102 4103 if args.check_dt_needed_ordering: 4104 num_errors += self._check_dt_needed_ordering(graph) 4105 4106 if args.check_apk: 4107 num_errors += self._check_apk_dep(graph, args.system, args.vendor, 4108 module_info) 4109 4110 return 0 if num_errors == 0 else 1 4111 4112 4113class DumpDexStringCommand(Command): 4114 def __init__(self): 4115 super(DumpDexStringCommand, self).__init__( 4116 'dump-dex-string', 4117 help='Dump string literals defined in a dex file') 4118 4119 4120 def add_argparser_options(self, parser): 4121 super(DumpDexStringCommand, self).add_argparser_options(parser) 4122 4123 parser.add_argument('dex_file', help='path to an input dex file') 4124 4125 4126 def main(self, args): 4127 for string in DexFileReader.enumerate_dex_strings(args.dex_file): 4128 try: 4129 print(string) 4130 except (UnicodeEncodeError, UnicodeDecodeError): 4131 print(repr(string)) 4132 4133 4134class DepGraphCommand(ELFGraphCommand): 4135 def __init__(self): 4136 super(DepGraphCommand, self).__init__( 4137 'dep-graph', help='Visualize violating dependencies with HTML') 4138 4139 4140 def add_argparser_options(self, parser): 4141 super(DepGraphCommand, self).add_argparser_options( 4142 parser, is_tag_file_required=True) 4143 4144 parser.add_argument('-o', '--output', required=True, 4145 help='output directory') 4146 4147 4148 @staticmethod 4149 def _create_tag_hierarchy(): 4150 hierarchy = dict() 4151 for tag in TaggedPathDict.TAGS: 4152 if tag in {'sp_hal', 'sp_hal_dep', 'vnd_only'}: 4153 hierarchy[tag] = 'vendor.private.{}'.format(tag) 4154 else: 4155 vendor_visible = TaggedPathDict.is_tag_visible('vnd_only', tag) 4156 pub = 'public' if vendor_visible else 'private' 4157 hierarchy[tag] = 'system.{}.{}'.format(pub, tag) 4158 return hierarchy 4159 4160 4161 @staticmethod 4162 def _get_lib_tag(hierarchy, tagged_paths, lib): 4163 return hierarchy[tagged_paths.get_path_tag(lib.path)] 4164 4165 4166 @staticmethod 4167 def _is_dep_allowed(user_tag, dep_tag): 4168 user_partition, _, _ = user_tag.split('.') 4169 dep_partition, dep_visibility, _ = dep_tag.split('.') 4170 if user_partition == 'system' and dep_partition == 'vendor': 4171 return False 4172 if user_partition == 'vendor' and dep_partition == 'system': 4173 if dep_visibility == 'private': 4174 return False 4175 return True 4176 4177 4178 def _get_dep_graph(self, graph, tagged_paths): 4179 hierarchy = self._create_tag_hierarchy() 4180 4181 # Build data and violate_libs. 4182 data = [] 4183 violate_libs = collections.defaultdict(list) 4184 4185 for lib in graph.all_libs(): 4186 lib_tag = self._get_lib_tag(hierarchy, tagged_paths, lib) 4187 lib_item = { 4188 'name': lib.path, 4189 'tag': lib_tag, 4190 'depends': [], 4191 'violates': [], 4192 } 4193 violate_count = 0 4194 for dep in lib.deps_all: 4195 dep_tag = self._get_lib_tag(hierarchy, tagged_paths, dep) 4196 if self._is_dep_allowed(lib_tag, dep_tag): 4197 lib_item['depends'].append(dep.path) 4198 else: 4199 lib_item['violates'].append([ 4200 dep.path, lib.get_dep_linked_symbols(dep)]) 4201 violate_count += 1 4202 lib_item['violate_count'] = violate_count 4203 if violate_count > 0: 4204 violate_libs[lib_tag].append((lib.path, violate_count)) 4205 data.append(lib_item) 4206 4207 # Sort data and violate_libs. 4208 data.sort( 4209 key=lambda lib_item: (lib_item['tag'], lib_item['violate_count'])) 4210 for libs in violate_libs.values(): 4211 libs.sort(key=lambda violate_item: violate_item[1], reverse=True) 4212 4213 return data, violate_libs 4214 4215 4216 def main(self, args): 4217 _, graph, tagged_paths, _ = self.create_from_args(args) 4218 4219 data, violate_libs = self._get_dep_graph(graph, tagged_paths) 4220 4221 makedirs(args.output, exist_ok=True) 4222 script_dir = os.path.dirname(os.path.abspath(__file__)) 4223 for name in ('index.html', 'dep-graph.js', 'dep-graph.css'): 4224 shutil.copyfile(os.path.join(script_dir, 'assets', 'visual', name), 4225 os.path.join(args.output, name)) 4226 with open(os.path.join(args.output, 'dep-data.js'), 'w') as f: 4227 f.write('var violatedLibs = ' + json.dumps(violate_libs) + ';\n') 4228 f.write('var depData = ' + json.dumps(data) + ';\n') 4229 4230 return 0 4231 4232 4233def main(): 4234 parser = argparse.ArgumentParser() 4235 subparsers = parser.add_subparsers(dest='subcmd') 4236 subcmds = dict() 4237 4238 def register_subcmd(cmd): 4239 subcmds[cmd.name] = cmd 4240 cmd.add_argparser_options( 4241 subparsers.add_parser(cmd.name, help=cmd.help)) 4242 4243 register_subcmd(ELFDumpCommand()) 4244 register_subcmd(CreateGenericRefCommand()) 4245 register_subcmd(VNDKCommand()) 4246 register_subcmd(DepsCommand()) 4247 register_subcmd(DepsClosureCommand()) 4248 register_subcmd(DepsInsightCommand()) 4249 register_subcmd(DepsUnresolvedCommand()) 4250 register_subcmd(ApkDepsCommand()) 4251 register_subcmd(CheckDepCommand()) 4252 register_subcmd(DepGraphCommand()) 4253 register_subcmd(DumpDexStringCommand()) 4254 4255 args = parser.parse_args() 4256 if not args.subcmd: 4257 parser.print_help() 4258 sys.exit(1) 4259 return subcmds[args.subcmd].main(args) 4260 4261if __name__ == '__main__': 4262 sys.exit(main()) 4263