1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2021 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18The tool for making updater package. 19 20positional arguments: 21 target_package Target package file path. 22 update_package Update package file path. 23 24optional arguments: 25 -h, --help show this help message and exit 26 -s SOURCE_PACKAGE, --source_package SOURCE_PACKAGE 27 Source package file path. 28 -nz, --no_zip No zip mode, 29 which means to output the update package without zip. 30 -pf PARTITION_FILE, --partition_file PARTITION_FILE 31 Variable partition mode, Partition list file path. 32 -sa {ECC,RSA}, --signing_algorithm {ECC,RSA} 33 The signing algorithms 34 supported by the tool include ['ECC', 'RSA']. 35 -ha {sha256,sha384}, --hash_algorithm {sha256,sha384} 36 The hash algorithms 37 supported by the tool include ['sha256', 'sha384']. 38 -pk PRIVATE_KEY, --private_key PRIVATE_KEY 39 Private key file path. 40 -nl2, --not_l2 Not L2 mode, Distinguish between L1 and L2. 41 -sl {256,384}, --signing_length {256,384} 42 The signing content length 43 supported by the tool include ['256', '384']. 44 -xp, --xml_path XML file path. 45 -sc, --sd_card SD Card mode, Create update package for SD Card. 46 -su, --stream_update Stream update mode, Create update package for stream update. 47 -ab, --ab_partition_update Ab partition update mode, Create update package for ab partition update. 48""" 49import filecmp 50import os 51import sys 52import argparse 53import subprocess 54import tempfile 55import hashlib 56import xmltodict 57import patch_package_process 58import math 59 60 61from gigraph_process import GigraphProcess 62from image_class import FullUpdateImage 63from image_class import IncUpdateImage 64from transfers_manager import TransfersManager 65from log_exception import UPDATE_LOGGER 66from script_generator import PreludeScript 67from script_generator import VerseScript 68from script_generator import RefrainScript 69from script_generator import EndingScript 70from update_package import build_update_package 71from unpack_updater_package import UnpackPackage 72from utils import OPTIONS_MANAGER 73from utils import UPDATER_CONFIG 74from utils import parse_partition_file_xml 75from utils import unzip_package 76from utils import clear_resource 77from utils import PRODUCT 78from utils import XML_FILE_PATH 79from utils import get_update_info 80from utils import SCRIPT_KEY_LIST 81from utils import PER_BLOCK_SIZE 82from utils import E2FSDROID_PATH 83from utils import MAXIMUM_RECURSION_DEPTH 84from utils import VERSE_SCRIPT_EVENT 85from utils import INC_IMAGE_EVENT 86from utils import DIFF_EXE_PATH 87from utils import PARTITION_CHANGE_EVENT 88from utils import DECOUPLED_EVENT 89from utils import get_update_config_softversion 90from vendor_script import create_vendor_script_class 91from create_chunk import CreateChunk 92from concurrent.futures import ThreadPoolExecutor 93 94sys.setrecursionlimit(MAXIMUM_RECURSION_DEPTH) 95 96 97def type_check(arg): 98 """ 99 Argument check, which is used to check whether the specified arg is a file. 100 :param arg: the arg to check 101 :return: Check result, which is False if the arg is invalid. 102 """ 103 if arg is not None and not os.path.exists(arg): 104 UPDATE_LOGGER.print_log( 105 "FileNotFoundError, path: %s" % arg, UPDATE_LOGGER.ERROR_LOG) 106 return False 107 return arg 108 109 110def private_key_check(arg): 111 """ 112 Argument check, which is used to check whether 113 the specified arg is a private_key. 114 :param arg: The arg to check. 115 :return: Check result, which is False if the arg is invalid. 116 """ 117 if arg != "ON_SERVER" and not os.path.isfile(arg): 118 UPDATE_LOGGER.print_log( 119 "FileNotFoundError, path: %s" % arg, UPDATE_LOGGER.ERROR_LOG) 120 return False 121 return arg 122 123 124def check_update_package(arg): 125 """ 126 Argument check, which is used to check whether 127 the update package path exists. 128 :param arg: The arg to check. 129 :return: Check result 130 """ 131 make_dir_path = None 132 if os.path.exists(arg): 133 if os.path.isfile(arg): 134 UPDATE_LOGGER.print_log( 135 "Update package must be a dir path, not a file path. " 136 "path: %s" % arg, UPDATE_LOGGER.ERROR_LOG) 137 return False 138 else: 139 try: 140 UPDATE_LOGGER.print_log( 141 "Update package path does not exist. The dir will be created!" 142 "path: %s" % arg, UPDATE_LOGGER.WARNING_LOG) 143 os.makedirs(arg) 144 make_dir_path = arg 145 except OSError: 146 UPDATE_LOGGER.print_log( 147 "Make update package path dir failed! " 148 "path: %s" % arg, UPDATE_LOGGER.ERROR_LOG) 149 return False 150 if make_dir_path is not None: 151 OPTIONS_MANAGER.make_dir_path = make_dir_path 152 OPTIONS_MANAGER.update_package = arg 153 return arg 154 155 156def unpack_check(arg): 157 """ 158 Argument check, which is used to check whether 159 the update package path exists. 160 :param arg: The arg to check. 161 :return: Check result 162 """ 163 unpack_package = os.path.join(OPTIONS_MANAGER.update_package, arg) 164 if not os.path.isfile(unpack_package): 165 UPDATE_LOGGER.print_log( 166 "FileNotFoundError, path: %s" % unpack_package, UPDATE_LOGGER.ERROR_LOG) 167 OPTIONS_MANAGER.unpack_package_path = None 168 return False 169 OPTIONS_MANAGER.unpack_package_path = unpack_package 170 return arg 171 172 173def create_entrance_args(): 174 """ 175 Arguments for the tool to create an update package 176 :return source_package : source version package 177 target_package : target version package 178 update_package : update package output path 179 no_zip : whether to enable the update package zip function. 180 partition_file : partition table XML file 181 signing_algorithm : signature algorithm (ECC and RSA (default)) 182 private_key : path of the private key file 183 """ 184 parser = OPTIONS_MANAGER.parser 185 parser.description = "Tool for creating update package." 186 parser.add_argument("-unpack", "--unpack_package", type=unpack_check, 187 default=None, help="Unpack updater package.") 188 parser.add_argument("-s", "--source_package", type=type_check, 189 default=None, help="Source package file path.") 190 parser.add_argument("target_package", type=type_check, 191 help="Target package file path.") 192 parser.add_argument("update_package", type=check_update_package, 193 help="Update package file path.") 194 parser.add_argument("-nz", "--no_zip", action='store_true', 195 help="No zip mode, Output update package without zip.") 196 parser.add_argument("-pf", "--partition_file", default=None, 197 help="Variable partition mode, " 198 "Partition list file path.") 199 parser.add_argument("-sa", "--signing_algorithm", default='RSA', 200 choices=['ECC', 'RSA'], 201 help="The signing algorithm " 202 "supported by the tool include ['ECC', 'RSA'].") 203 parser.add_argument("-ha", "--hash_algorithm", default='sha256', 204 choices=['sha256', 'sha384'], 205 help="The hash algorithm " 206 "supported by the tool include " 207 "['sha256', 'sha384'].") 208 parser.add_argument("-pk", "--private_key", type=private_key_check, 209 default=None, help="Private key file path.") 210 parser.add_argument("-nl2", "--not_l2", action='store_true', 211 help="Not L2 mode, Distinguish between L1 and L2.") 212 parser.add_argument("-sl", "--signing_length", default='256', 213 choices=['256', '384'], 214 help="The signing content length " 215 "supported by the tool include " 216 "['256', '384'].") 217 parser.add_argument("-xp", "--xml_path", type=private_key_check, 218 default=None, help="XML file path.") 219 parser.add_argument("-sc", "--sd_card", action='store_true', 220 help="SD Card mode, " 221 "Create update package for SD Card.") 222 parser.add_argument("-su", "--stream_update", action='store_true', 223 help="Stream update mode, " 224 "Create update package for stream update.") 225 parser.add_argument("-ab", "--ab_partition_update", action='store_true', 226 help="Ab partition update mode, " 227 "Create update package for ab partition update.") 228 229 230def parse_args(): 231 args = OPTIONS_MANAGER.parser.parse_args() 232 OPTIONS_MANAGER.source_package = args.source_package 233 OPTIONS_MANAGER.target_package = args.target_package 234 OPTIONS_MANAGER.update_package = args.update_package 235 OPTIONS_MANAGER.no_zip = args.no_zip 236 OPTIONS_MANAGER.partition_file = args.partition_file 237 OPTIONS_MANAGER.signing_algorithm = args.signing_algorithm 238 OPTIONS_MANAGER.hash_algorithm = args.hash_algorithm 239 OPTIONS_MANAGER.private_key = args.private_key 240 OPTIONS_MANAGER.not_l2 = args.not_l2 241 OPTIONS_MANAGER.signing_length = int(args.signing_length) 242 OPTIONS_MANAGER.xml_path = args.xml_path 243 OPTIONS_MANAGER.sd_card = args.sd_card 244 OPTIONS_MANAGER.stream_update = args.stream_update 245 OPTIONS_MANAGER.ab_partition_update = args.ab_partition_update 246 247 248def get_args(): 249 ret_args = \ 250 [OPTIONS_MANAGER.source_package, 251 OPTIONS_MANAGER.target_package, 252 OPTIONS_MANAGER.update_package, 253 OPTIONS_MANAGER.no_zip, 254 OPTIONS_MANAGER.not_l2, 255 OPTIONS_MANAGER.partition_file, 256 OPTIONS_MANAGER.signing_algorithm, 257 OPTIONS_MANAGER.hash_algorithm, 258 OPTIONS_MANAGER.private_key] 259 return ret_args 260 261 262def get_script_obj(): 263 """ 264 Obtain Opera script object 265 :return: 266 """ 267 script_obj_list = create_vendor_script_class() 268 if script_obj_list == [None] * len(SCRIPT_KEY_LIST): 269 prelude_script = PreludeScript() 270 verse_script = VerseScript() 271 refrain_script = RefrainScript() 272 ending_script = EndingScript() 273 274 generate_verse_script = \ 275 OPTIONS_MANAGER.init.invoke_event(VERSE_SCRIPT_EVENT) 276 if generate_verse_script: 277 verse_script = generate_verse_script() 278 else: 279 UPDATE_LOGGER.print_log( 280 "Get vendor extension object completed!" 281 "The vendor extension script will be generated.") 282 prelude_script = script_obj_list[0] 283 verse_script = script_obj_list[1] 284 refrain_script = script_obj_list[2] 285 ending_script = script_obj_list[3] 286 return prelude_script, verse_script, refrain_script, ending_script 287 288 289def get_source_package_path(source_package): 290 """ 291 get_source_package_path. 292 :param source_package: source package path 293 :return: 294 """ 295 if os.path.isdir(source_package): 296 OPTIONS_MANAGER.source_package_dir = source_package 297 elif source_package.endswith('.zip'): 298 # Decompress the source package. 299 tmp_dir_obj, unzip_dir = unzip_package(source_package) 300 if tmp_dir_obj is False or unzip_dir is False: 301 clear_resource(err_clear=True) 302 return False 303 OPTIONS_MANAGER.source_package_dir = unzip_dir 304 OPTIONS_MANAGER.source_package_temp_obj = tmp_dir_obj 305 else: 306 UPDATE_LOGGER.print_log("Input Update Package type exception!" 307 "path: %s" % source_package, UPDATE_LOGGER.ERROR_LOG) 308 clear_resource(err_clear=True) 309 return False 310 return True 311 312 313def check_incremental_args(no_zip, partition_file, source_package, 314 incremental_img_list): 315 """ 316 When the incremental list is not empty, incremental processing is required. 317 In this case, check related arguments. 318 :param no_zip: no zip mode 319 :param partition_file: 320 :param source_package: 321 :param incremental_img_list: 322 :return: 323 """ 324 if "boot" in incremental_img_list: 325 UPDATE_LOGGER.print_log("boot cannot be incrementally processed!", UPDATE_LOGGER.ERROR_LOG) 326 clear_resource(err_clear=True) 327 return False 328 if source_package is None: 329 UPDATE_LOGGER.print_log("The source package is missing, " 330 "cannot be incrementally processed!", UPDATE_LOGGER.ERROR_LOG) 331 clear_resource(err_clear=True) 332 return False 333 if no_zip: 334 UPDATE_LOGGER.print_log("No ZIP mode, cannot be incrementally processed!", UPDATE_LOGGER.ERROR_LOG) 335 clear_resource(err_clear=True) 336 return False 337 if partition_file is not None: 338 UPDATE_LOGGER.print_log("Partition file is not None, " 339 "cannot be incrementally processed!", UPDATE_LOGGER.ERROR_LOG) 340 clear_resource(err_clear=True) 341 return False 342 343 if not get_source_package_path(source_package): 344 return False 345 partition_change = OPTIONS_MANAGER.init.invoke_event(PARTITION_CHANGE_EVENT) 346 if callable(partition_change) and partition_change() is False: 347 return False 348 UPDATE_LOGGER.print_log("Partition interception check finish.", UPDATE_LOGGER.INFO_LOG) 349 xml_path = '' 350 if OPTIONS_MANAGER.source_package_dir is not False: 351 xml_path = os.path.join(OPTIONS_MANAGER.source_package_dir, UPDATER_CONFIG, XML_FILE_PATH) 352 if OPTIONS_MANAGER.source_package_dir is False: 353 OPTIONS_MANAGER.source_package_temp_obj = None 354 OPTIONS_MANAGER.source_package_dir = None 355 if os.path.exists(xml_path): 356 with open(xml_path, 'r') as xml_file: 357 xml_str = xml_file.read() 358 else: 359 UPDATE_LOGGER.print_log("XML file does not exist! xml path: %s" % 360 xml_path, UPDATE_LOGGER.ERROR_LOG) 361 return False 362 363 xml_content_dict = xmltodict.parse(xml_str, encoding='utf-8') 364 package_dict = xml_content_dict.get('package', {}) 365 get_update_config_softversion(OPTIONS_MANAGER.source_package_dir, package_dict.get('head', {})) 366 head_dict = package_dict.get('head', {}).get('info') 367 OPTIONS_MANAGER.source_package_version = head_dict.get("@softVersion") 368 no_need_version_check_res = OPTIONS_MANAGER.init.invoke_event(DECOUPLED_EVENT) 369 if no_need_version_check_res is False: 370 if check_package_version(OPTIONS_MANAGER.target_package_version, 371 OPTIONS_MANAGER.source_package_version) is False: 372 clear_resource(err_clear=True) 373 return False 374 return True 375 376 377def check_userdata_image(): 378 """ 379 Check the userdata image. Updating this image is prohibited. 380 :return: 381 """ 382 if 'userdata' in OPTIONS_MANAGER.full_img_list or \ 383 'userdata' in OPTIONS_MANAGER.incremental_img_list: 384 UPDATE_LOGGER.print_log( 385 "userdata image does not participate in update!" 386 "Please check xml config, path: %s!" % 387 os.path.join(OPTIONS_MANAGER.target_package_config_dir, 388 XML_FILE_PATH), 389 UPDATE_LOGGER.ERROR_LOG) 390 clear_resource(err_clear=True) 391 return False 392 return True 393 394 395def check_images_list(): 396 """ 397 Check full_img_list and incremental_img_list. 398 If their lengths are 0, an error will be logged. 399 :return: 400 """ 401 if len(OPTIONS_MANAGER.full_img_list) == 0 and \ 402 len(OPTIONS_MANAGER.incremental_img_list) == 0: 403 UPDATE_LOGGER.print_log( 404 "The image list is empty!" 405 "Please check xml config, path: %s!" % 406 os.path.join(OPTIONS_MANAGER.target_package_config_dir, 407 XML_FILE_PATH), 408 UPDATE_LOGGER.ERROR_LOG) 409 clear_resource(err_clear=True) 410 return False 411 return True 412 413 414def check_target_package_path(target_package): 415 """ 416 Check the target_package path. 417 :param target_package: target package path 418 :return: 419 """ 420 if os.path.isdir(target_package): 421 OPTIONS_MANAGER.target_package_dir = target_package 422 temp_dir_list = os.listdir(target_package) 423 if UPDATER_CONFIG in temp_dir_list: 424 OPTIONS_MANAGER.target_package_config_dir = \ 425 os.path.join(target_package, UPDATER_CONFIG) 426 else: 427 UPDATE_LOGGER.print_log( 428 "Exception's target package path! path: %s" % 429 target_package, UPDATE_LOGGER.ERROR_LOG) 430 return False 431 elif target_package.endswith('.zip'): 432 # Decompress the target package. 433 tmp_dir_obj, unzip_dir = unzip_package(target_package) 434 if tmp_dir_obj is False or unzip_dir is False: 435 clear_resource(err_clear=True) 436 return False 437 OPTIONS_MANAGER.target_package_dir = unzip_dir 438 OPTIONS_MANAGER.target_package_temp_obj = tmp_dir_obj 439 OPTIONS_MANAGER.target_package_config_dir = \ 440 os.path.join(unzip_dir, UPDATER_CONFIG) 441 else: 442 UPDATE_LOGGER.print_log( 443 "Input Update Package type exception! path: %s" % 444 target_package, UPDATE_LOGGER.ERROR_LOG) 445 clear_resource(err_clear=True) 446 return False 447 return True 448 449 450def check_miss_private_key(private_key): 451 """ 452 Check private key. 453 :param private_key: 454 :return: 455 """ 456 if private_key is None: 457 UPDATE_LOGGER.print_log( 458 "Private key is None, update package cannot be signed! " 459 "Please specify the signature private key by -pk.", 460 UPDATE_LOGGER.ERROR_LOG) 461 clear_resource(err_clear=True) 462 return False 463 return True 464 465 466def check_package_version(target_ver, source_ver): 467 """ 468 target_ver: target version 469 source_ver: source version 470 return: 471 """ 472 try: 473 target_num = ''.join(target_ver.split(' ')[-1].replace('.', '')) 474 source_num = ''.join(source_ver.split(' ')[-1].replace('.', '')) 475 if int(target_num) <= int(source_num): 476 UPDATE_LOGGER.print_log( 477 'Target package version %s <= Source package version!' 478 'Unable to make updater package!', 479 UPDATE_LOGGER.ERROR_LOG) 480 return False 481 except ValueError: 482 UPDATE_LOGGER.print_log('your package version number is not compliant.' 483 'Please check your package version number!', 484 UPDATE_LOGGER.ERROR_LOG) 485 return False 486 return True 487 488 489def generate_image_map_file(image_path, map_path, image_name): 490 """ 491 :param image_path: image path 492 :param map_path: image map file path 493 :param image_name: image name 494 :return: 495 """ 496 if not os.path.exists(image_path): 497 UPDATE_LOGGER.print_log("The source %s.img file is missing from the" 498 "source package, cannot be incrementally processed. " 499 % image_name, UPDATE_LOGGER.ERROR_LOG) 500 return False 501 502 cmd = [E2FSDROID_PATH, "-B", map_path, "-a", "/%s" % image_name, image_path, "-e"] 503 res = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 504 _, err = res.communicate(timeout=300) 505 if res.returncode != 0: 506 UPDATE_LOGGER.print_log("%s.map generate failed, reason:%s" % 507 (image_name, err.decode()), UPDATE_LOGGER.ERROR_LOG) 508 return False 509 UPDATE_LOGGER.print_log("%s.map generate success" % image_name, UPDATE_LOGGER.INFO_LOG) 510 return True 511 512 513def get_file_sha256(update_package): 514 sha256obj = hashlib.sha256() 515 maxbuf = 8192 516 with open(update_package, 'rb') as package_file: 517 while True: 518 buf = package_file.read(maxbuf) 519 if not buf: 520 break 521 sha256obj.update(buf) 522 hash_value = sha256obj.hexdigest() 523 return str(hash_value).upper() 524 525 526def write_image_patch_script(partition, src_image_path, tgt_image_path, 527 script_check_cmd_list, script_write_cmd_list, verse_script): 528 """ 529 Add command content to the script. 530 :param partition: image name 531 :param script_check_cmd_list: incremental check command list 532 :param script_write_cmd_list: incremental write command list 533 :param verse_script: verse script object 534 :return: 535 """ 536 src_sha = get_file_sha256(src_image_path) 537 src_size = os.path.getsize(src_image_path) 538 tgt_sha = get_file_sha256(tgt_image_path) 539 tgt_size = os.path.getsize(tgt_image_path) 540 541 sha_check_cmd = verse_script.image_sha_check(partition, 542 src_size, src_sha, tgt_size, tgt_sha) 543 544 first_block_check_cmd = verse_script.first_block_check(partition) 545 546 abort_cmd = verse_script.abort(partition) 547 548 cmd = 'if ({sha_check_cmd} != 0)' \ 549 '{{\n {abort_cmd}}}\n'.format( 550 sha_check_cmd=sha_check_cmd, 551 abort_cmd=abort_cmd) 552 553 script_check_cmd_list.append(cmd) 554 555 image_patch_cmd = verse_script.image_patch(partition, os.path.getsize(src_image_path), 556 get_file_sha256(src_image_path), os.path.getsize(tgt_image_path), 557 get_file_sha256(tgt_image_path)) 558 559 cmd = '%s_WRITE_FLAG%s' % (partition, image_patch_cmd) 560 script_write_cmd_list.append(cmd) 561 return True 562 563 564def increment_image_diff_processing( 565 partition, src_image_path, tgt_image_path, 566 script_check_cmd_list, script_write_cmd_list, verse_script): 567 """ 568 Incremental image processing 569 :param verse_script: verse script 570 :param incremental_img_list: incremental image list 571 :param source_package_dir: source package path 572 :param target_package_dir: target package path 573 :return: 574 """ 575 patch_file_obj = tempfile.NamedTemporaryFile( 576 prefix="%s_patch.dat-" % partition, mode='wb') 577 OPTIONS_MANAGER.incremental_image_file_obj_dict[partition] = patch_file_obj 578 cmd = [DIFF_EXE_PATH] 579 580 cmd.extend(['-s', src_image_path, '-d', tgt_image_path, 581 '-p', patch_file_obj.name, '-l', '4096']) 582 sub_p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 583 stderr=subprocess.STDOUT) 584 try: 585 output, _ = sub_p.communicate(timeout=1800) 586 except subprocess.TimeoutExpired: 587 sub_p.kill() 588 589 sub_p.wait() 590 if sub_p.returncode != 0: 591 raise ValueError(output) 592 593 return write_image_patch_script(partition, src_image_path, tgt_image_path, 594 script_check_cmd_list, script_write_cmd_list, verse_script) 595 596 597def copy_in_ab_process(patch_process, src_image_class, need_copy_blocks, 598 non_continuous_blocks, each_img): 599 """ 600 Handle the process of copying blocks in an AB partition during an update. 601 :param patch_process: The patch process object responsible for managing the update. 602 :param src_image_class: The source image class containing metadata about the image. 603 :param need_copy_blocks: A list to store blocks that need to be copied. 604 :param non_continuous_blocks: A list to store blocks that are non-continuous. 605 :param each_img: The current image being processed. 606 :return: 607 """ 608 if OPTIONS_MANAGER.ab_partition_update: 609 chunk_pkgdiff_list = patch_process.get_chunk_pkgdiff_list() 610 transfer_content = patch_process.get_transfer_content_in_chunk() 611 chunk_new_list = patch_process.get_chunk_new_list() 612 613 # Parase transfer_content, record all blocks involved in the content 614 no_copy_blocks_in_ab, transfer_content_no_startoffert = parse_transfer_content(transfer_content) 615 616 # Find the blocks to copy, total blocks includeing zero, excluding total_blocks 617 no_copy_blocks_set = set(no_copy_blocks_in_ab) 618 619 def process_block(src_block): 620 if src_block in no_copy_blocks_set: 621 return None, src_block # Not to copy 622 return src_block, None # To copy 623 with ThreadPoolExecutor() as executor: 624 results = executor.map(process_block, range(src_image_class.total_blocks)) 625 626 for to_copy, not_to_copy in results: 627 if to_copy is not None: 628 need_copy_blocks.append(to_copy) 629 if not_to_copy is not None: 630 non_continuous_blocks.append(not_to_copy) 631 # Sort blocks consecutively 632 if len(need_copy_blocks): 633 group_numbers_list = group_numbers(need_copy_blocks) 634 else: 635 group_numbers_list = [] 636 print(f'there is no copy blocks in image {each_img} !') 637 # Add the copy command for ab partition synchronization 638 transfer_content = patch_process.add_ab_copy_content(len(need_copy_blocks), 639 group_numbers_list, transfer_content_no_startoffert) 640 OPTIONS_MANAGER.image_transfer_dict_contents[each_img] = transfer_content 641 OPTIONS_MANAGER.image_patch_dic[each_img] = chunk_pkgdiff_list 642 OPTIONS_MANAGER.image_new_dic[each_img] = chunk_new_list 643 644 645def add_incremental_command(verse_script, script_check_cmd_list, script_write_cmd_list): 646 """ 647 add command for increment_image_progressing to verse_script 648 :param verse_script: verse script 649 :param script_check_cmd_list: verse_script check command list 650 :param script_write_cmd_list: verse_script write command list 651 :return: 652 """ 653 verse_script.add_command("\n# ---- start incremental check here ----\n") 654 for each_check_cmd in script_check_cmd_list: 655 verse_script.add_command(each_check_cmd) 656 verse_script.add_command("\n# ---- start incremental write here ----\n") 657 for each_write_cmd in script_write_cmd_list: 658 verse_script.add_command(each_write_cmd) 659 660 661def increment_image_processing( 662 verse_script, incremental_img_list, source_package_dir, 663 target_package_dir): 664 """ 665 Incremental image processing 666 :param verse_script: verse script 667 :param incremental_img_list: incremental image list 668 :param source_package_dir: source package path 669 :param target_package_dir: target package path 670 :return: 671 """ 672 script_check_cmd_list = [] 673 script_write_cmd_list = [] 674 patch_process = None 675 676 for each_img_name in OPTIONS_MANAGER.incremental_img_name_list: 677 each_img = each_img_name[:-4] 678 each_src_image_path = os.path.join(source_package_dir, '%s.img' % each_img) 679 each_src_map_path = os.path.join(source_package_dir, '%s.map' % each_img) 680 each_tgt_image_path = os.path.join(target_package_dir, '%s.img' % each_img) 681 each_tgt_map_path = os.path.join(target_package_dir, '%s.map' % each_img) 682 683 # This will store tuples of (start, end, length) for continuous ranges 684 non_continuous_blocks = [] 685 need_copy_blocks = [] 686 check_make_map_path(each_img) 687 688 # Call the new function to process image maps 689 continue_processing, src_generate_map, tgt_generate_map = process_image_maps( 690 each_img, each_src_image_path, each_src_map_path, 691 each_tgt_image_path, each_tgt_map_path) 692 if not continue_processing: 693 continue 694 695 # get the large of target image 696 if not get_large_of_target_image(each_tgt_image_path, each_img): 697 return False 698 699 if not src_generate_map or not tgt_generate_map: 700 if not handle_no_map_generation(each_img, each_src_image_path, each_tgt_image_path, \ 701 script_check_cmd_list, script_write_cmd_list, verse_script): 702 return False 703 continue 704 705 inc_image = OPTIONS_MANAGER.init.invoke_event(INC_IMAGE_EVENT) 706 if inc_image: 707 src_image_class, tgt_image_class = inc_image(each_src_image_path, each_src_map_path, 708 each_tgt_image_path, each_tgt_map_path) 709 else: 710 src_image_class = IncUpdateImage(each_src_image_path, each_src_map_path) 711 tgt_image_class = IncUpdateImage(each_tgt_image_path, each_tgt_map_path) 712 713 transfers_manager = TransfersManager(each_img, tgt_image_class, src_image_class) 714 transfers_manager.find_process_needs() 715 actions_list = transfers_manager.get_action_list() 716 717 graph_process = GigraphProcess(actions_list, src_image_class, tgt_image_class) 718 # Streaming update does not need to handle stash and free commands 719 if not OPTIONS_MANAGER.ab_partition_update: 720 graph_process.stash_process() 721 722 actions_list = graph_process.actions_list 723 patch_process = patch_package_process.PatchProcess(each_img, tgt_image_class, src_image_class, actions_list) 724 725 patch_process.patch_process(each_tgt_image_path) 726 727 # Add copy command for ab partition 728 copy_in_ab_process(patch_process, src_image_class, need_copy_blocks, 729 non_continuous_blocks, each_img) 730 731 patch_process.write_script(each_img, script_check_cmd_list, script_write_cmd_list, verse_script) 732 OPTIONS_MANAGER.incremental_block_file_obj_dict[each_img] = patch_process.package_patch_zip 733 if not OPTIONS_MANAGER.stream_update: 734 if not check_patch_file(patch_process): 735 UPDATE_LOGGER.print_log('Verify the incremental result failed!', UPDATE_LOGGER.ERROR_LOG) 736 raise RuntimeError 737 738 add_incremental_command(verse_script, script_check_cmd_list, script_write_cmd_list) 739 return True 740 741 742def get_large_of_target_image(each_tgt_image_path, each_img): 743 """ 744 Reads the target image content and stores it in OPTIONS_MANAGER.diff_image_new_data. 745 746 :param each_tgt_image_path: The path to the target image. 747 :param each_img: The name of the image (without extension). 748 :return: True if successful, False otherwise. 749 """ 750 try: 751 with open(each_tgt_image_path, 'rb') as f: 752 target_content = f.read() 753 OPTIONS_MANAGER.diff_image_new_data[each_img] = target_content 754 return True 755 except Exception as e: 756 print(f"Error reading target image {each_img}: {e}") 757 return False 758 759 760def process_image_maps(each_img, each_src_image_path, each_src_map_path, 761 each_tgt_image_path, each_tgt_map_path): 762 """ 763 Process source and target image maps for incremental updates. 764 765 :param each_img: The image name without extension 766 :param each_src_image_path: Path to the source image 767 :param each_src_map_path: Path to the source map 768 :param each_tgt_image_path: Path to the target image 769 :param each_tgt_map_path: Path to the target map 770 :return: A tuple (continue_processing, src_generate_map, tgt_generate_map) 771 - continue_processing: Whether to continue processing the current image 772 - src_generate_map: Whether the source map was successfully generated 773 - tgt_generate_map: Whether the target map was successfully generated 774 """ 775 # Check if source and target images are identical 776 if filecmp.cmp(each_src_image_path, each_tgt_image_path): 777 UPDATE_LOGGER.print_log( 778 "Source Image is the same as Target Image! src image path: %s, tgt image path: %s" 779 % (each_src_image_path, each_tgt_image_path), 780 UPDATE_LOGGER.INFO_LOG) 781 OPTIONS_MANAGER.incremental_img_list.remove(each_img) 782 return False, False, False # Stop further processing for this image 783 784 # Initialize map generation flags 785 src_generate_map = True 786 tgt_generate_map = True 787 788 # Generate source map if it does not exist 789 if not os.path.exists(each_src_map_path): 790 src_generate_map = generate_image_map_file(each_src_image_path, each_src_map_path, each_img) 791 792 # Generate target map if it does not exist 793 if not os.path.exists(each_tgt_map_path): 794 tgt_generate_map = generate_image_map_file(each_tgt_image_path, each_tgt_map_path, each_img) 795 796 return True, src_generate_map, tgt_generate_map 797 798 799def handle_no_map_generation(each_img, each_src_image_path, each_tgt_image_path, script_check_cmd_list, script_write_cmd_list, verse_script): 800 if OPTIONS_MANAGER.stream_update: 801 print(f'do no map process:{each_img}') 802 OPTIONS_MANAGER.no_map_image_exist = True 803 OPTIONS_MANAGER.no_map_file_list.append(each_img) 804 # Directly cut the target image into new commands 805 if not os.path.exists(each_tgt_image_path): 806 UPDATE_LOGGER.print_log( 807 "The file is missing " 808 "from the target package, " 809 "the component: %s cannot be full update processed. " % 810 each_tgt_image_path) 811 return False 812 with open(each_tgt_image_path, 'rb') as f: 813 tartget_no_map_content = f.read() 814 chunk, block_sets = split_image_file(each_img, tartget_no_map_content) 815 OPTIONS_MANAGER.image_chunk[each_img] = chunk 816 OPTIONS_MANAGER.image_block_sets[each_img] = block_sets 817 return True 818 # If it is not a streaming update and cannot generate map file,directly diff the image 819 elif increment_image_diff_processing(each_img, each_src_image_path, each_tgt_image_path, 820 script_check_cmd_list, script_write_cmd_list, verse_script) is True: 821 return True 822 UPDATE_LOGGER.print_log("increment_image_diff_processing %s failed" % each_img) 823 clear_resource(err_clear=True) 824 return False 825 826 827def check_patch_file(patch_process): 828 new_dat_file_obj, patch_dat_file_obj, transfer_list_file_obj = \ 829 patch_process.package_patch_zip.get_file_obj() 830 with open(transfer_list_file_obj.name) as f_t: 831 num = 0 832 diff_str = None 833 diff_num = 0 834 for line in f_t: 835 if line.startswith('new '): 836 each_line_list = \ 837 line.strip().replace("new ", "").split(",")[1:] 838 for idx in range(0, len(each_line_list), 2): 839 num += \ 840 int(each_line_list[idx + 1]) - int(each_line_list[idx]) 841 continue 842 if line.startswith('bsdiff ') or line.startswith('pkgdiff '): 843 diff_str = line 844 if diff_str: 845 diff_list = diff_str.split('\n')[0].split(' ') 846 diff_num = int(diff_list[1]) + int(diff_list[2]) 847 check_flag = \ 848 (os.path.getsize(new_dat_file_obj.name) == num * PER_BLOCK_SIZE) and \ 849 (os.path.getsize(patch_dat_file_obj.name) == diff_num) 850 return check_flag 851 852 853def check_make_map_path(each_img): 854 """ 855 If env does not exist, the command for map generation does not exist 856 in the environment variable, and False will be returned. 857 """ 858 try: 859 cmd = [E2FSDROID_PATH, " -h"] 860 subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, 861 stderr=subprocess.STDOUT) 862 except FileNotFoundError: 863 UPDATE_LOGGER.print_log( 864 "Command not found, need check the env! " 865 "Make %s.map failed!" % each_img, 866 UPDATE_LOGGER.ERROR_LOG) 867 clear_resource(err_clear=True) 868 raise RuntimeError 869 return True 870 871 872def incremental_processing(no_zip, partition_file, source_package, 873 verse_script): 874 """ 875 Incremental processing. 876 :param no_zip: no zip mode 877 :param partition_file: partition xml file path 878 :param source_package: source package path 879 :param verse_script: verse script obj 880 :return : processing result 881 """ 882 if len(OPTIONS_MANAGER.incremental_img_list) != 0: 883 if check_incremental_args(no_zip, partition_file, source_package, 884 OPTIONS_MANAGER.incremental_img_list) \ 885 is False: 886 return False 887 if increment_image_processing( 888 verse_script, OPTIONS_MANAGER.incremental_img_list, 889 OPTIONS_MANAGER.source_package_dir, 890 OPTIONS_MANAGER.target_package_dir) is False: 891 return False 892 else: 893 if source_package is not None: 894 UPDATE_LOGGER.print_log( 895 "There is no incremental image, " 896 "the - S parameter is not required!", 897 UPDATE_LOGGER.ERROR_LOG) 898 raise RuntimeError 899 900 901def check_args(private_key, source_package, target_package, update_package): 902 """ 903 Input args check. 904 :param private_key: private key path 905 :param source_package: source package path 906 :param target_package: target package path 907 :param update_package: output package path 908 :return : Judgment result 909 """ 910 if source_package is False or private_key is False or \ 911 target_package is False or update_package is False: 912 return False 913 if check_miss_private_key(private_key) is False: 914 return False 915 if check_target_package_path(target_package) is False: 916 return False 917 if get_update_info() is False: 918 return False 919 if check_images_list() is False: 920 return False 921 return True 922 923 924def unpack_package_processing(): 925 if OPTIONS_MANAGER.unpack_package_path: 926 package = UnpackPackage() 927 if not package.unpack_package(): 928 UPDATE_LOGGER.print_log("Unpack update package.bin failed!", UPDATE_LOGGER.ERROR_LOG) 929 clear_resource(err_clear=True) 930 sys.exit(1) 931 UPDATE_LOGGER.print_log("Unpack update package.bin success!") 932 clear_resource() 933 sys.exit(0) 934 935 936def group_numbers(arr): 937 """ 938 Groups and sorts a list of consecutive numbers, returning a list of block ranges. 939 :param arr: A list of integers representing block numbers. 940 :return: A list of integers representing the start and end of each block range. 941 """ 942 result = [] 943 current_group = [arr[0]] 944 transfer_content_blocks = [] 945 946 for i in range(1, len(arr)): 947 if arr[i] == arr[i - 1] + 1: 948 current_group.append(arr[i]) 949 else: 950 result.append({ 951 'min': min(current_group), 952 'max': max(current_group), 953 'length': len(current_group) 954 }) 955 transfer_content_blocks.append(min(current_group)) 956 transfer_content_blocks.append(max(current_group) + 1) 957 958 current_group = [arr[i]] 959 960 961 result.append({ 962 'min': min(current_group), 963 'max': max(current_group), 964 'length': len(current_group) 965 }) 966 transfer_content_blocks.append(min(current_group)) 967 transfer_content_blocks.append(max(current_group) + 1) 968 969 return transfer_content_blocks 970 971 972def parse_transfer_content(content): 973 """ 974 Parses the transfer content to extract block information and modified lines. 975 :param content: The string content to be parsed, typically containing block data. 976 :return: A tuple containing: 977 - all_blocks: A list of all extracted block numbers. 978 - modified_content: A modified version of the original content with adjustments made. 979 """ 980 all_blocks = [] 981 modified_lines = [] 982 lines = content.splitlines() 983 for line in lines[4:]: 984 if 'pkgdiff' in line: 985 modified_line, blocks = handle_pkgdiff(line) 986 modified_lines.append(modified_line) 987 all_blocks.extend(blocks) 988 elif 'zero' in line or 'erase' in line: 989 blocks = handle_zero_erase(line) 990 all_blocks.extend(blocks) 991 modified_lines.append(line) 992 elif 'move' in line or 'new' in line: 993 blocks = handle_move_new(line) 994 all_blocks.extend(blocks) 995 modified_lines.append(line) 996 elif 'free' in line: 997 print('no add free cmd') 998 modified_lines.append(line) 999 else: 1000 raise ValueError(f"no parse transfer_list info, got {lines}.") 1001 modified_content = '\n'.join(lines[:4] + modified_lines) 1002 return all_blocks, modified_content 1003 1004 1005def handle_pkgdiff(line): 1006 """ 1007 Handles the 'pkgdiff' line type. 1008 :param line: The line to process. 1009 :return: A tuple containing the modified line and the list of blocks. 1010 """ 1011 parts = line.replace(',', ' ').split() 1012 # Change the start offset address of pkgdiff to all zeros 1013 modified_line = line.replace(line.split()[1], '0', 1) 1014 length = int(parts[5]) 1015 if length % 2 != 0: 1016 raise ValueError(f"Length must be even, got {length}.") 1017 # Extract the range values of a pair of blocks 1018 pairs = list(map(int, parts[6:length + 6])) 1019 blocks = [] 1020 # List all blocks in a single pair of block range values 1021 for i in range(0, len(pairs), 2): 1022 start = pairs[i] 1023 print(f'start:{start}') 1024 end = pairs[i + 1] 1025 print(f'end:{end}') 1026 blocks.extend(range(start, end)) 1027 print(f'len(blocks):{len(blocks)}') 1028 return modified_line, blocks 1029 1030 1031def handle_zero_erase(line): 1032 """ 1033 Handles the 'zero' or 'erase' line type. 1034 :param line: The line to process. 1035 :return: The list of blocks extracted from the line. 1036 """ 1037 parts = line.replace(',', ' ').split() 1038 length = int(parts[1]) 1039 print(f'lenght:{length}') 1040 if length % 2 != 0: 1041 raise ValueError(f"Length must be even, got {length}.") 1042 pairs = list(map(int, parts[2:length + 2])) 1043 blocks = [] 1044 for i in range(0, len(pairs), 2): 1045 start = pairs[i] 1046 print(f'start:{start}') 1047 end = pairs[i + 1] 1048 print(f'end:{end}') 1049 blocks.extend(range(start, end)) 1050 print(f'len(blocks):{len(blocks)}') 1051 return blocks 1052 1053 1054def handle_move_new(line): 1055 """ 1056 Handles the 'move' or 'new' line type. 1057 :param line: The line to process. 1058 :return: The list of blocks extracted from the line. 1059 """ 1060 parts = line.replace(',', ' ').split() 1061 length = int(parts[2]) 1062 print(f'lenght:{length}') 1063 if length % 2 != 0: 1064 raise ValueError(f"Length must be even, got {length}.") 1065 pairs = list(map(int, parts[3:length + 3])) 1066 blocks = [] 1067 for i in range(0, len(pairs), 2): 1068 start = pairs[i] 1069 print(f'start:{start}') 1070 end = pairs[i + 1] 1071 print(f'end:{end}') 1072 blocks.extend(range(start, end)) 1073 print(f'len(blocks):{len(blocks)}') 1074 return blocks 1075 1076 1077def split_image_file(each_img, full_image_data): 1078 """ 1079 Splits the full image data into smaller chunks. 1080 :param each_img: The image to be split (not used in the current implementation). 1081 :param full_image_data: The complete image data to be split. 1082 :return: A tuple containing two lists: 1083 - chunks: A list of data chunks. 1084 - block_sets: A list of corresponding block sets for each chunk. 1085 """ 1086 # get the total block number of the image 1087 total_blocks = math.ceil(len(full_image_data) / 4096) 1088 print(f'total tgt blocks:{total_blocks}') 1089 # step 1：get the total size of the image data 1090 max_chunk_size = OPTIONS_MANAGER.chunk_limit * 4096 1091 total_size = len(full_image_data) 1092 print(f'total size:{total_size}') 1093 chunks = [] 1094 block_sets = [] 1095 # step 2：cut the image data into fixed block size 1096 num = math.ceil(total_size / max_chunk_size) 1097 print(f'num:{num}') 1098 for i in range(num): 1099 start_index = i * max_chunk_size 1100 end_index = min(start_index + max_chunk_size, total_size) 1101 chunk = full_image_data[start_index:end_index] 1102 # step 3：record the corresponding block set 1103 block_set = list(range(math.ceil(start_index / 4096), 1104 math.ceil(end_index / 4096))) 1105 chunks.append(chunk) 1106 block_sets.append(block_set) 1107 print(f"block {i + 1}: size = {len(chunk)} bytes, block sets = {block_set}") 1108 print(f'total size:{total_size}') 1109 print(f'total tgt blocks:{total_blocks}') 1110 return chunks, block_sets 1111 1112 1113create_entrance_args() 1114 1115 1116def main(): 1117 """ 1118 Entry function. 1119 """ 1120 parse_args() 1121 1122 OPTIONS_MANAGER.product = PRODUCT 1123 1124 source_package, target_package, update_package, no_zip, not_l2, \ 1125 partition_file, signing_algorithm, hash_algorithm, private_key = get_args() 1126 if not_l2: 1127 no_zip = True 1128 1129 # Unpack updater package 1130 unpack_package_processing() 1131 1132 if OPTIONS_MANAGER.sd_card: 1133 if source_package is not None or OPTIONS_MANAGER.xml_path is not None or partition_file is not None: 1134 UPDATE_LOGGER.print_log("SDCard updater:-S/-xp/-pf parameter is not required!", UPDATE_LOGGER.ERROR_LOG) 1135 raise RuntimeError 1136 if check_args(private_key, source_package, target_package, update_package) is False: 1137 clear_resource(err_clear=True) 1138 return 1139 1140 if not OPTIONS_MANAGER.sd_card: 1141 if check_userdata_image() is False: 1142 clear_resource(err_clear=True) 1143 return 1144 1145 # Create a Script object. 1146 prelude_script, verse_script, refrain_script, ending_script = get_script_obj() 1147 1148 # Create partition. 1149 if partition_file is not None: 1150 verse_script.add_command("\n# ---- do updater partitions ----\n") 1151 updater_partitions_cmd = verse_script.updater_partitions() 1152 verse_script.add_command(updater_partitions_cmd) 1153 1154 partition_file_obj, partitions_list, partitions_file_path_list = parse_partition_file_xml(partition_file) 1155 if partition_file_obj is False: 1156 clear_resource(err_clear=True) 1157 return False 1158 OPTIONS_MANAGER.partition_file_obj = partition_file_obj 1159 OPTIONS_MANAGER.full_img_list = partitions_list 1160 OPTIONS_MANAGER.full_image_path_list = partitions_file_path_list 1161 1162 # Incremental processing 1163 if incremental_processing(no_zip, partition_file, source_package, verse_script) is False: 1164 clear_resource(err_clear=True) 1165 return 1166 1167 # Full processing 1168 if len(OPTIONS_MANAGER.full_img_list) != 0: 1169 verse_script.add_command("\n# ---- full image ----\n") 1170 full_update_image = FullUpdateImage(OPTIONS_MANAGER.target_package_dir, 1171 OPTIONS_MANAGER.full_img_list, OPTIONS_MANAGER.full_img_name_list, verse_script, 1172 OPTIONS_MANAGER.full_image_path_list, no_zip=OPTIONS_MANAGER.no_zip) 1173 full_image_content_len_list, full_image_file_obj_list = full_update_image.update_full_image() 1174 if full_image_content_len_list is False or full_image_file_obj_list is False: 1175 clear_resource(err_clear=True) 1176 return 1177 # Full streaming process 1178 if OPTIONS_MANAGER.stream_update: 1179 for each_img_name in OPTIONS_MANAGER.full_img_name_list: 1180 print(f'full streaming process! current image name is:{each_img_name}') 1181 each_img = each_img_name[:-4] 1182 chunks, block_sets = split_image_file(each_img, 1183 OPTIONS_MANAGER.full_image_new_data[each_img]) 1184 OPTIONS_MANAGER.full_chunk[each_img] = chunks 1185 OPTIONS_MANAGER.full_block_sets[each_img] = block_sets 1186 1187 OPTIONS_MANAGER.full_image_content_len_list, OPTIONS_MANAGER.full_image_file_obj_list = \ 1188 full_image_content_len_list, full_image_file_obj_list 1189 1190 # Generate the update package. 1191 if build_update_package( 1192 no_zip, update_package, prelude_script, verse_script, refrain_script, ending_script) is False: 1193 clear_resource(err_clear=True) 1194 return 1195 # Clear resources. 1196 clear_resource() 1197 1198 1199if __name__ == '__main__': 1200 main() 1201