1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3 4# Copyright (c) 2024 Hunan OpenValley Digital Industry Development Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16""" 17Description : pack chunks to update.bin. 18""" 19import os 20import subprocess 21import build_module_img 22import tempfile 23from utils import OPTIONS_MANAGER 24from utils import DIFF_EXE_PATH 25from transfers_manager import ActionType 26 27DIFF_BLOCK_LIMIT = 10240 28 29 30class PatchPackageChunk: 31 32 def __init__(self, *args): 33 self.src_file, self.tgt_file, self.do_pkg_diff, self.transfer_content, self.diff_offset, self.patch_dat_file_obj,\ 34 self.src_img_obj, self.tgt_img_obj, each_action, self.chunk_data_list = args 35 self.chunk_src_obj = tempfile.NamedTemporaryFile(prefix="chunk_src_file", mode='wb') 36 self.chunk_tgt_obj = tempfile.NamedTemporaryFile(prefix="chunk_tft_file", mode='wb') 37 self.chunk_patch_obj = tempfile.NamedTemporaryFile(prefix="chunk_patch_file", mode='wb') 38 self.patch_obj = tempfile.NamedTemporaryFile(prefix="chunk_patch_file", mode='wb') 39 self.limit_size = OPTIONS_MANAGER.chunk_limit * build_module_img.BLOCK_SIZE 40 41 self.__apply_compute_patch(self.src_file, self.tgt_file, self.patch_obj, 42 4096) 43 print(os.stat(self.patch_obj.name).st_size) 44 print(f'src:{each_action.src_block_set.to_string_raw()}({each_action.src_block_set.size()}) tgt:{each_action.tgt_block_set.to_string_raw()}({each_action.tgt_block_set.size()})') 45 self.__apply_compute_patch(self.src_file, self.tgt_file, self.patch_obj, int(self.limit_size / DIFF_BLOCK_LIMIT)) # 45KB 46 47 self.__chunk_patch(self.patch_obj.name, int(self.limit_size / DIFF_BLOCK_LIMIT), each_action) 48 49 def split_into_closest_multiples_of_ten(self, n): 50 """ 51 Split an integer into two parts, such that the sum of the parts is a multiple of 10. 52 :param n: The integer to split. 53 :return: A tuple containing the two parts. 54 """ 55 # Check if the input is a multiple of 10 56 if n % 10 != 0: 57 raise ValueError("Input must be a multiple of 10.") 58 # Calculate the two parts 59 half = n // 2 60 # Ensure both parts are multiples of 10 61 part1 = (half // 10) * 10 # Nearest lower multiple of 10 62 part2 = n - part1 # Remaining part 63 # If part2 is not a multiple of 10, adjust part1 and part2 64 if part2 % 10 != 0: 65 part1 += 10 66 part2 = n - part1 67 return part1, part2 68 69 def split_old_file(self, each_action, start_blocks, blocks, src_file_blocks, src_file_obj, src_file_bytes): 70 """处理旧文件的分块逻辑""" 71 if (start_blocks + blocks) > each_action.src_block_set.size(): 72 src_file_blocks = each_action.src_block_set 73 temp_blocks = each_action.src_block_set.size() - blocks 74 if temp_blocks < 0: 75 temp_blocks = 0 76 else: 77 temp_src_blocks_to_write = src_file_blocks.get_first_block_obj(temp_blocks) 78 src_file_blocks = src_file_blocks.get_subtract_with_other(temp_src_blocks_to_write) 79 src_blocks_to_write = src_file_blocks.get_first_block_obj(blocks) 80 src_start = src_file_bytes - blocks * build_module_img.BLOCK_SIZE 81 if src_start < 0: 82 src_start = 0 83 else: 84 src_start = start_blocks * build_module_img.BLOCK_SIZE 85 src_blocks_to_write = src_file_blocks.get_first_block_obj(blocks) 86 src_file_blocks = src_file_blocks.get_subtract_with_other(src_blocks_to_write) 87 88 if src_file_bytes == 0: 89 print(f'error: {src_file_bytes}') 90 raise RuntimeError 91 92 src_file_obj.seek(src_start) 93 self.chunk_src_obj.seek(0) 94 self.chunk_src_obj.truncate(0) 95 bytes_obj = src_file_obj.read(int(blocks * build_module_img.BLOCK_SIZE)) 96 if len(bytes_obj) == 0: 97 src_total_size = each_action.src_block_set.size() 98 print(f'in: {src_total_size}') 99 raise RuntimeError 100 101 try: 102 self.chunk_src_obj.write(bytes_obj) 103 except Exception as e: 104 print(f'error:{e}') 105 raise RuntimeError 106 107 return src_file_blocks, src_blocks_to_write, src_start 108 109 def split_new_tgt_file(self, each_action, start_blocks, blocks, tgt_file_blocks, tgt_file_obj): 110 """处理新目标文件的分块逻辑""" 111 tgt_blocks_to_write = tgt_file_blocks.get_first_block_obj(blocks) 112 tgt_file_blocks = tgt_file_blocks.get_subtract_with_other(tgt_blocks_to_write) 113 tgt_file_obj.seek(int(start_blocks * build_module_img.BLOCK_SIZE)) 114 self.chunk_tgt_obj.seek(0) 115 self.chunk_tgt_obj.truncate(0) 116 bytes_obj = tgt_file_obj.read(int(blocks * build_module_img.BLOCK_SIZE)) 117 if len(bytes_obj) == 0: 118 tgt_total_size = each_action.tgt_block_set.size() 119 print(f'in: {tgt_total_size}') 120 raise RuntimeError 121 self.chunk_tgt_obj.write(bytes_obj) 122 123 return tgt_file_blocks, tgt_blocks_to_write 124 125 def process_patch_chunk(self, src_blocks_to_write, tgt_blocks_to_write, chunk_patch_size, start_blocks, blocks): 126 """ 127 处理每个 chunk patch 的逻辑，包括打印信息、更新状态、保存 patch 内容等。 128 """ 129 subfile_patch_totalsize = chunk_patch_size 130 start_blocks += blocks 131 132 # 打印 patch 文件的信息 133 print(f'self.chunk_patch_obj len: {os.stat(self.chunk_patch_obj.name).st_size} ' 134 f'old: {os.stat(self.patch_dat_file_obj.name).st_size} total: {self.diff_offset}') 135 136 # 读取 patch 文件内容并保存 137 with open(self.chunk_patch_obj.name, 'rb') as file_read: 138 patch_value = file_read.read() 139 self.patch_dat_file_obj.write(patch_value) 140 141 # 如果 patch 有内容，保存到 transfer_content 和其他相关变量 142 if len(patch_value) > 0: 143 diff_type = "pkgdiff" if self.do_pkg_diff else "bsdiff" 144 diff_str = ("%s %d %d %s %s %s %d %s\n" % ( 145 diff_type, 146 self.diff_offset, len(patch_value), 147 self.src_img_obj.range_sha256(src_blocks_to_write), 148 self.tgt_img_obj.range_sha256(tgt_blocks_to_write), 149 tgt_blocks_to_write.to_string_raw(), src_blocks_to_write.size(), src_blocks_to_write.to_string_raw())) 150 print(diff_str) 151 152 self.diff_offset += len(patch_value) 153 self.chunk_data_list.append(patch_value) 154 self.transfer_content.append(diff_str) 155 156 # 打印 transfer_content 长度信息 157 print(f'in transfer_content len: {len(self.transfer_content)} ' 158 f'self.chunk_patch_obj len: {os.stat(self.patch_dat_file_obj.name).st_size}') 159 160 # 返回更新后的 totalsize 和 start_blocks 161 return subfile_patch_totalsize, start_blocks 162 163 def cut_files(self, subblocks_list, each_action): 164 start_blocks = 0 165 subfile_patch_sizelist = [] 166 subfile_patch_totalsize = 0 167 src_file_blocks = each_action.src_block_set 168 tgt_file_blocks = each_action.tgt_block_set 169 170 src_file_obj = open(self.src_file, 'rb') 171 file_stat = os.stat(self.src_file) 172 src_file_bytes = file_stat.st_size 173 tgt_file_obj = open(self.tgt_file, 'rb') 174 file_stat = os.stat(self.tgt_file) 175 tgt_file_bytes = file_stat.st_size 176 src_start = 0 177 178 # 遍历 subblocks_list 179 i = 0 180 while i < len(subblocks_list): 181 blocks = subblocks_list[i] 182 183 # Store the original state for restoration 184 original_src_file_blocks = src_file_blocks 185 original_tgt_file_blocks = tgt_file_blocks 186 original_start_blocks = start_blocks 187 188 # Splite old files 189 src_file_blocks, src_blocks_to_write, src_start = self.split_old_file(each_action, start_blocks, blocks, src_file_blocks, src_file_obj, src_file_bytes) 190 191 # Splite new tgt file 192 tgt_file_blocks, tgt_blocks_to_write = self.split_new_tgt_file(each_action, start_blocks, blocks, tgt_file_blocks, tgt_file_obj) 193 194 # Here's a patch 195 self.chunk_patch_obj = self.__apply_compute_patch(self.chunk_src_obj.name, self.chunk_tgt_obj.name, self.chunk_patch_obj, 4096) 196 chunk_patch_size = os.stat(self.chunk_patch_obj.name).st_size 197 198 # 如果patch大小超过限制，将blocks切分为两部分插入subblocks_list中 199 if chunk_patch_size > self.limit_size: 200 print(f'Patch size {chunk_patch_size} exceeds limit {self.limit_size}, splitting blocks...') 201 block_one, block_two = self.split_into_closest_multiples_of_ten(blocks) 202 if block_one % 10 != 0 or block_two % 10 != 0 or block_one + block_two != blocks: 203 print(f"Error: blocks size split error.") 204 raise RuntimeError 205 206 # Restore the previous state 207 src_file_blocks = original_src_file_blocks 208 tgt_file_blocks = original_tgt_file_blocks 209 start_blocks = original_start_blocks 210 211 # 在当前位置插入两个新的blocks 212 subblocks_list[i: i + 1] = [block_one, block_two] 213 print(f'Inserted two new blocks: {block_one, block_two}') 214 print(f'Current subblocks_list: {subblocks_list}') 215 continue # 跳出这次循环，重新处理新分解的块 216 else: 217 print(f'Patch size {chunk_patch_size} within limit {self.limit_size}, no splitting needed.') 218 subfile_patch_sizelist.append(chunk_patch_size) 219 220 print(f'[{int(src_start)}, {int(os.stat(self.chunk_src_obj.name).st_size / build_module_img.BLOCK_SIZE)}] ' 221 f'diff [{start_blocks}, {int(os.stat(self.chunk_tgt_obj.name).st_size / build_module_img.BLOCK_SIZE)}] => {chunk_patch_size}') 222 223 subfile_patch_totalsize, start_blocks = self.process_patch_chunk( 224 src_blocks_to_write, tgt_blocks_to_write, chunk_patch_size, start_blocks, blocks) 225 # 继续处理下一个块 226 i += 1 227 228 src_file_obj.close() 229 tgt_file_obj.close() 230 231 return subfile_patch_sizelist, subfile_patch_totalsize 232 233 def __apply_compute_patch(self, src_file, tgt_file, patch_obj, limit): 234 """ 235 Add command content to the script. 236 :param src_file: source file name 237 :param tgt_file: target file name 238 :return: 239 """ 240 patch_obj.seek(0) 241 patch_obj.truncate(0) 242 cmd = [DIFF_EXE_PATH] if self.do_pkg_diff else [DIFF_EXE_PATH, '-b', '1'] 243 244 cmd.extend(['-s', src_file, '-d', tgt_file, 245 '-p', patch_obj.name, '-l', f'{limit}']) 246 sub_p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 247 stderr=subprocess.STDOUT) 248 output, _ = sub_p.communicate(timeout=300) 249 sub_p.wait() 250 patch_obj.seek(0) 251 252 if sub_p.returncode != 0: 253 raise ValueError(output) 254 255 return patch_obj 256 257 def __chunk_patch(self, patch_file_obj, file_limit_size, each_action): 258 # 1.Parase patch 259 patch_list = [] 260 file_limit_size = int(file_limit_size * DIFF_BLOCK_LIMIT / build_module_img.BLOCK_SIZE) 261 file_stat = os.stat(patch_file_obj) 262 file_size_bytes = file_stat.st_size 263 with open(patch_file_obj, 'rb') as file: 264 print(file.read(8)) # title 265 blocks = int.from_bytes(file.read(4), byteorder='little') 266 print({blocks}) 267 lastoffset = 0 268 for i in range(blocks): 269 file.read(20) # Ignore 20B 270 offset = int.from_bytes(file.read(8), byteorder='little') # patchOffset 271 if lastoffset == 0: 272 lastoffset = offset 273 else: 274 patch_list.append(offset - lastoffset) 275 lastoffset = offset 276 patch_list.append(file_size_bytes - lastoffset) 277 print(patch_list) 278 print(f'fileLen:{file_size_bytes}') 279 280 # 2.Split files 281 total = 0 282 blocks = 0 283 index = 0 284 subblocks_list = [] 285 for dt in patch_list: 286 total += dt 287 if total < 0: 288 total = 0 289 blocks += file_limit_size 290 if total > self.limit_size: 291 subblocks_list.append(blocks - file_limit_size) # 确保结果小于45 * 102 292 blocks = file_limit_size 293 total = dt 294 if blocks > 0: 295 subblocks_list.append(blocks) 296 print(subblocks_list) 297 # 3、Cut files 298 subfile_patch_sizelist, subfile_patch_totalsize = self.cut_files(subblocks_list, each_action) 299 300 # 4.Stats 301 print(f'\ndebug do over\n') 302 print(subfile_patch_sizelist) 303 print(subfile_patch_totalsize) 304 # 5.Comparison with Native 305 print(f'old:{file_stat.st_size} new:{subfile_patch_totalsize}') 306