• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright (C) 2022 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may not
6# use this file except in compliance with the License. You may obtain a copy of
7# the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations under
15# the License.
16#
17"""Functions for merging META/* files from partial builds.
18
19Expects items in OPTIONS prepared by merge_target_files.py.
20"""
21
22import logging
23import os
24import re
25import shutil
26
27import build_image
28import common
29import merge_utils
30import sparse_img
31import verity_utils
32
33from common import ExternalError
34
35logger = logging.getLogger(__name__)
36
37OPTIONS = common.OPTIONS
38
39# In apexkeys.txt or apkcerts.txt, we will find partition tags on each entry in
40# the file. We use these partition tags to filter the entries in those files
41# from the two different target files packages to produce a merged apexkeys.txt
42# or apkcerts.txt file. A partition tag (e.g., for the product partition) looks
43# like this: 'partition="product"'. We use the group syntax grab the value of
44# the tag. We use non-greedy matching in case there are other fields on the
45# same line.
46
47PARTITION_TAG_PATTERN = re.compile(r'partition="(.*?)"')
48
49# The sorting algorithm for apexkeys.txt and apkcerts.txt does not include the
50# ".apex" or ".apk" suffix, so we use the following pattern to extract a key.
51
52MODULE_KEY_PATTERN = re.compile(r'name="(.+)\.(apex|apk)"')
53
54
55def MergeMetaFiles(temp_dir, merged_dir):
56  """Merges various files in META/*."""
57
58  framework_meta_dir = os.path.join(temp_dir, 'framework_meta', 'META')
59  merge_utils.ExtractItems(
60      input_zip=OPTIONS.framework_target_files,
61      output_dir=os.path.dirname(framework_meta_dir),
62      extract_item_list=('META/*',))
63
64  vendor_meta_dir = os.path.join(temp_dir, 'vendor_meta', 'META')
65  merge_utils.ExtractItems(
66      input_zip=OPTIONS.vendor_target_files,
67      output_dir=os.path.dirname(vendor_meta_dir),
68      extract_item_list=('META/*',))
69
70  merged_meta_dir = os.path.join(merged_dir, 'META')
71
72  # Merge META/misc_info.txt into OPTIONS.merged_misc_info,
73  # but do not write it yet. The following functions may further
74  # modify this dict.
75  OPTIONS.merged_misc_info = MergeMiscInfo(
76      framework_meta_dir=framework_meta_dir,
77      vendor_meta_dir=vendor_meta_dir,
78      merged_meta_dir=merged_meta_dir)
79
80  CopyNamedFileContexts(
81      framework_meta_dir=framework_meta_dir,
82      vendor_meta_dir=vendor_meta_dir,
83      merged_meta_dir=merged_meta_dir)
84
85  if OPTIONS.merged_misc_info.get('use_dynamic_partitions') == 'true':
86    MergeDynamicPartitionsInfo(
87        framework_meta_dir=framework_meta_dir,
88        vendor_meta_dir=vendor_meta_dir,
89        merged_meta_dir=merged_meta_dir)
90
91  if OPTIONS.merged_misc_info.get('ab_update') == 'true':
92    MergeAbPartitions(
93        framework_meta_dir=framework_meta_dir,
94        vendor_meta_dir=vendor_meta_dir,
95        merged_meta_dir=merged_meta_dir)
96    UpdateCareMapImageSizeProps(images_dir=os.path.join(merged_dir, 'IMAGES'))
97
98  for file_name in ('apkcerts.txt', 'apexkeys.txt'):
99    MergePackageKeys(
100        framework_meta_dir=framework_meta_dir,
101        vendor_meta_dir=vendor_meta_dir,
102        merged_meta_dir=merged_meta_dir,
103        file_name=file_name)
104
105  # Write the now-finalized OPTIONS.merged_misc_info.
106  merge_utils.WriteSortedData(
107      data=OPTIONS.merged_misc_info,
108      path=os.path.join(merged_meta_dir, 'misc_info.txt'))
109
110
111def MergeAbPartitions(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
112  """Merges META/ab_partitions.txt.
113
114  The output contains the union of the partition names.
115  """
116  with open(os.path.join(framework_meta_dir, 'ab_partitions.txt')) as f:
117    framework_ab_partitions = f.read().splitlines()
118
119  with open(os.path.join(vendor_meta_dir, 'ab_partitions.txt')) as f:
120    vendor_ab_partitions = f.read().splitlines()
121
122  merge_utils.WriteSortedData(
123      data=set(framework_ab_partitions + vendor_ab_partitions),
124      path=os.path.join(merged_meta_dir, 'ab_partitions.txt'))
125
126
127def MergeMiscInfo(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
128  """Merges META/misc_info.txt.
129
130  The output contains a combination of key=value pairs from both inputs.
131  Most pairs are taken from the vendor input, while some are taken from
132  the framework input.
133  """
134
135  OPTIONS.framework_misc_info = common.LoadDictionaryFromFile(
136      os.path.join(framework_meta_dir, 'misc_info.txt'))
137  OPTIONS.vendor_misc_info = common.LoadDictionaryFromFile(
138      os.path.join(vendor_meta_dir, 'misc_info.txt'))
139
140  # Merged misc info is a combination of vendor misc info plus certain values
141  # from the framework misc info.
142
143  merged_dict = OPTIONS.vendor_misc_info
144  for key in OPTIONS.framework_misc_info_keys:
145    if key in OPTIONS.framework_misc_info:
146      merged_dict[key] = OPTIONS.framework_misc_info[key]
147
148  # If AVB is enabled then ensure that we build vbmeta.img.
149  # Partial builds with AVB enabled may set PRODUCT_BUILD_VBMETA_IMAGE=false to
150  # skip building an incomplete vbmeta.img.
151  if merged_dict.get('avb_enable') == 'true':
152    merged_dict['avb_building_vbmeta_image'] = 'true'
153
154  return merged_dict
155
156
157def MergeDynamicPartitionsInfo(framework_meta_dir, vendor_meta_dir,
158                               merged_meta_dir):
159  """Merge META/dynamic_partitions_info.txt."""
160  framework_dynamic_partitions_dict = common.LoadDictionaryFromFile(
161      os.path.join(framework_meta_dir, 'dynamic_partitions_info.txt'))
162  vendor_dynamic_partitions_dict = common.LoadDictionaryFromFile(
163      os.path.join(vendor_meta_dir, 'dynamic_partitions_info.txt'))
164
165  merged_dynamic_partitions_dict = common.MergeDynamicPartitionInfoDicts(
166      framework_dict=framework_dynamic_partitions_dict,
167      vendor_dict=vendor_dynamic_partitions_dict)
168
169  merge_utils.WriteSortedData(
170      data=merged_dynamic_partitions_dict,
171      path=os.path.join(merged_meta_dir, 'dynamic_partitions_info.txt'))
172
173  # Merge misc info keys used for Dynamic Partitions.
174  OPTIONS.merged_misc_info.update(merged_dynamic_partitions_dict)
175  # Ensure that add_img_to_target_files rebuilds super split images for
176  # devices that retrofit dynamic partitions. This flag may have been set to
177  # false in the partial builds to prevent duplicate building of super.img.
178  OPTIONS.merged_misc_info['build_super_partition'] = 'true'
179
180
181def MergePackageKeys(framework_meta_dir, vendor_meta_dir, merged_meta_dir,
182                     file_name):
183  """Merges APK/APEX key list files."""
184
185  if file_name not in ('apkcerts.txt', 'apexkeys.txt'):
186    raise ExternalError(
187        'Unexpected file_name provided to merge_package_keys_txt: %s',
188        file_name)
189
190  def read_helper(d):
191    temp = {}
192    with open(os.path.join(d, file_name)) as f:
193      for line in f.read().splitlines():
194        line = line.strip()
195        if line:
196          name_search = MODULE_KEY_PATTERN.search(line.split()[0])
197          temp[name_search.group(1)] = line
198    return temp
199
200  framework_dict = read_helper(framework_meta_dir)
201  vendor_dict = read_helper(vendor_meta_dir)
202  merged_dict = {}
203
204  def filter_into_merged_dict(item_dict, partition_set):
205    for key, value in item_dict.items():
206      tag_search = PARTITION_TAG_PATTERN.search(value)
207
208      if tag_search is None:
209        raise ValueError('Entry missing partition tag: %s' % value)
210
211      partition_tag = tag_search.group(1)
212
213      if partition_tag in partition_set:
214        if key in merged_dict:
215          if OPTIONS.allow_duplicate_apkapex_keys:
216            # TODO(b/150582573) Always raise on duplicates.
217            logger.warning('Duplicate key %s' % key)
218            continue
219          else:
220            raise ValueError('Duplicate key %s' % key)
221
222        merged_dict[key] = value
223
224  # Prioritize framework keys first.
225  # Duplicate keys from vendor are an error, or ignored.
226  filter_into_merged_dict(framework_dict, OPTIONS.framework_partition_set)
227  filter_into_merged_dict(vendor_dict, OPTIONS.vendor_partition_set)
228
229  # The following code is similar to WriteSortedData, but different enough
230  # that we couldn't use that function. We need the output to be sorted by the
231  # basename of the apex/apk (without the ".apex" or ".apk" suffix). This
232  # allows the sort to be consistent with the framework/vendor input data and
233  # eases comparison of input data with merged data.
234  with open(os.path.join(merged_meta_dir, file_name), 'w') as output:
235    for key, value in sorted(merged_dict.items()):
236      output.write(value + '\n')
237
238
239def CopyNamedFileContexts(framework_meta_dir, vendor_meta_dir, merged_meta_dir):
240  """Creates named copies of each partial build's file_contexts.bin.
241
242  Used when regenerating images from the partial build.
243  """
244
245  def copy_fc_file(source_dir, file_name):
246    for name in (file_name, 'file_contexts.bin'):
247      fc_path = os.path.join(source_dir, name)
248      if os.path.exists(fc_path):
249        shutil.copyfile(fc_path, os.path.join(merged_meta_dir, file_name))
250        return
251    raise ValueError('Missing file_contexts file from %s: %s', source_dir,
252                     file_name)
253
254  copy_fc_file(framework_meta_dir, 'framework_file_contexts.bin')
255  copy_fc_file(vendor_meta_dir, 'vendor_file_contexts.bin')
256
257  # Replace <image>_selinux_fc values with framework or vendor file_contexts.bin
258  # depending on which dictionary the key came from.
259  # Only the file basename is required because all selinux_fc properties are
260  # replaced with the full path to the file under META/ when misc_info.txt is
261  # loaded from target files for repacking. See common.py LoadInfoDict().
262  for key in OPTIONS.vendor_misc_info:
263    if key.endswith('_selinux_fc'):
264      OPTIONS.merged_misc_info[key] = 'vendor_file_contexts.bin'
265  for key in OPTIONS.framework_misc_info:
266    if key.endswith('_selinux_fc'):
267      OPTIONS.merged_misc_info[key] = 'framework_file_contexts.bin'
268
269
270def UpdateCareMapImageSizeProps(images_dir):
271  """Sets <partition>_image_size props in misc_info.
272
273  add_images_to_target_files uses these props to generate META/care_map.pb.
274  Regenerated images will have this property set during regeneration.
275
276  However, images copied directly from input partial target files packages
277  need this value calculated here.
278  """
279  for partition in common.PARTITIONS_WITH_CARE_MAP:
280    image_path = os.path.join(images_dir, '{}.img'.format(partition))
281    if os.path.exists(image_path):
282      partition_size = sparse_img.GetImagePartitionSize(image_path)
283      image_props = build_image.ImagePropFromGlobalDict(
284          OPTIONS.merged_misc_info, partition)
285      verity_image_builder = verity_utils.CreateVerityImageBuilder(image_props)
286      image_size = verity_image_builder.CalculateMaxImageSize(partition_size)
287      OPTIONS.merged_misc_info['{}_image_size'.format(partition)] = image_size
288