• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2020 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Test manifest split."""
15
16import json
17import mock
18import os
19import re
20import subprocess
21import tempfile
22import unittest
23import xml.etree.ElementTree as ET
24
25import manifest_split
26
27
28class ManifestSplitTest(unittest.TestCase):
29
30  def test_read_config(self):
31    with tempfile.NamedTemporaryFile('w+t') as test_config:
32      test_config.write("""
33        <config>
34          <add_project name="add1" />
35          <add_project name="add2" />
36          <remove_project name="remove1" />
37          <remove_project name="remove2" />
38          <path_mapping pattern="p1.*" sub="$0" />
39        </config>""")
40      test_config.flush()
41      config = manifest_split.ManifestSplitConfig.from_config_files(
42          [test_config.name])
43      self.assertEqual(config.remove_projects, {
44          'remove1': test_config.name,
45          'remove2': test_config.name
46      })
47      self.assertEqual(config.add_projects, {
48          'add1': test_config.name,
49          'add2': test_config.name
50      })
51      self.assertEqual(config.path_mappings, [
52          manifest_split.PathMappingConfig(re.compile('p1.*'), '$0'),
53      ])
54
55  def test_get_repo_projects_from_manifest(self):
56    manifest_contents = """
57      <manifest>
58        <project name="platform/project1" path="system/project1" />
59        <project name="platform/project2" path="system/project2" />
60        <project name="platform/project3" path="system/project3" />
61      </manifest>"""
62    manifest = ET.ElementTree(ET.fromstring(manifest_contents))
63    projects = manifest_split.get_repo_projects(
64        None, manifest, path_mappings=[])
65    self.assertDictEqual(
66        {
67            'system/project1': 'platform/project1',
68            'system/project2': 'platform/project2',
69            'system/project3': 'platform/project3',
70        }, projects)
71
72
73  def test_get_repo_projects(self):
74    with tempfile.NamedTemporaryFile('w+t') as repo_list_file:
75      repo_list_file.write("""
76        system/project1 : platform/project1
77        system/project2 : platform/project2""")
78      repo_list_file.flush()
79      repo_projects = manifest_split.get_repo_projects(
80          repo_list_file.name, None, path_mappings=[])
81      self.assertEqual(
82          repo_projects, {
83              'system/project1': 'platform/project1',
84              'system/project2': 'platform/project2',
85          })
86
87  def test_get_repo_projects_with_mappings(self):
88    with tempfile.NamedTemporaryFile('w+t') as repo_list_file:
89      repo_list_file.write("""
90        overlay/system/project1 : platform/project1
91        system/project2 : platform/project2
92        hide/this/one : platform/project3""")
93      repo_list_file.flush()
94      path_mappings = [
95          manifest_split.PathMappingConfig(re.compile('^overlay/(.*)'), '\\1'),
96          manifest_split.PathMappingConfig(re.compile('^hide/this/one.*'), ''),
97      ]
98
99      repo_projects = manifest_split.get_repo_projects(repo_list_file.name,
100                                                       None,
101                                                       path_mappings)
102      self.assertEqual(
103          repo_projects, {
104              'system/project1': 'platform/project1',
105              'system/project2': 'platform/project2',
106          })
107
108  def test_get_module_info(self):
109    with tempfile.NamedTemporaryFile('w+t') as module_info_file:
110      module_info_file.write("""{
111        "target1a": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target2"] },
112        "target1b": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target3", "target42"] },
113        "target2": { "class": ["SHARED_LIBRARIES"], "path": ["out/project2"], "dependencies": [] },
114        "target3": { "class": ["SHARED_LIBRARIES"], "path": ["vendor/google/project3"], "dependencies": ["x", "y", "z"] },
115        "target4a": { "class": ["APPS"], "path": ["system/project4"], "dependencies": ["out/target/common/obj/JAVA_LIBRARIES/target4b_intermediates/classes-header.jar"] },
116        "target4b": { "class": ["JAVA_LIBRARIES"],  "path": ["system/project4"], "dependencies": [] }
117      }""")
118      module_info_file.flush()
119      repo_projects = {
120          'system/project1': 'platform/project1',
121          'system/project4': 'platform/project4',
122          'vendor/google/project3': 'vendor/project3',
123      }
124      module_info = manifest_split.ModuleInfo(module_info_file.name,
125                                              repo_projects)
126      self.assertEqual(
127          module_info.project_modules, {
128              'platform/project1': set(['target1a', 'target1b']),
129              'platform/project4': set(['target4a', 'target4b']),
130              'vendor/project3': set(['target3']),
131          })
132      self.assertEqual(
133          module_info.module_project, {
134              'target1a': 'platform/project1',
135              'target1b': 'platform/project1',
136              'target3': 'vendor/project3',
137              'target4a': 'platform/project4',
138              'target4b': 'platform/project4',
139          })
140      self.assertEqual(
141          module_info.module_class, {
142              'target1a': 'EXECUTABLES',
143              'target1b': 'EXECUTABLES',
144              'target2': 'SHARED_LIBRARIES',
145              'target3': 'SHARED_LIBRARIES',
146              'target4a': 'APPS',
147              'target4b': 'JAVA_LIBRARIES',
148          })
149      self.assertEqual(
150          module_info.module_deps, {
151              'target1a': ['target2'],
152              'target1b': ['target3', 'target42'],
153              'target2': [],
154              'target3': ['x', 'y', 'z'],
155              'target4a': ['target4b'],
156              'target4b': [],
157          })
158
159  def test_get_module_info_raises_on_unknown_module_path(self):
160    with tempfile.NamedTemporaryFile('w+t') as module_info_file:
161      module_info_file.write("""{
162        "target1": { "class": ["EXECUTABLES"], "path": ["system/unknown/project1"], "dependencies": [] }
163      }""")
164      module_info_file.flush()
165      repo_projects = {}
166      with self.assertRaisesRegex(ValueError,
167                                  'Unknown module path for module target1'):
168        manifest_split.ModuleInfo(module_info_file.name, repo_projects)
169
170  @mock.patch.object(subprocess, 'check_output', autospec=True)
171  def test_get_ninja_inputs(self, mock_check_output):
172    mock_check_output.return_value = b"""
173    path/to/input1
174    path/to/input2
175    path/to/TEST_MAPPING
176    path/to/MODULE_LICENSE_GPL
177    """
178
179    inputs = manifest_split.get_ninja_inputs('unused', 'unused', ['droid'])
180    self.assertEqual(inputs, {'path/to/input1', 'path/to/input2'})
181
182  @mock.patch.object(subprocess, 'check_output', autospec=True)
183  def test_get_ninja_inputs_includes_test_mapping(self, mock_check_output):
184    mock_check_output.return_value = b"""
185    path/to/input1
186    path/to/input2
187    path/to/TEST_MAPPING
188    """
189
190    inputs = manifest_split.get_ninja_inputs('unused', 'unused',
191                                             ['droid', 'test_mapping'])
192    self.assertEqual(
193        inputs, {'path/to/input1', 'path/to/input2', 'path/to/TEST_MAPPING'})
194
195  @mock.patch.object(subprocess, 'check_output', autospec=True)
196  def test_get_kati_makefiles(self, mock_check_output):
197    with tempfile.TemporaryDirectory() as temp_dir:
198      os.chdir(temp_dir)
199
200      makefiles = [
201          'device/oem1/product1.mk',
202          'device/oem2/product2.mk',
203          'device/google/google_product.mk',
204          'overlays/oem_overlay/device/oem3/product3.mk',
205          'packages/apps/Camera/Android.mk',
206      ]
207      for makefile in makefiles:
208        os.makedirs(os.path.dirname(makefile))
209        os.mknod(makefile)
210
211      symlink_src = os.path.join(temp_dir, 'vendor/oem4/symlink_src.mk')
212      os.makedirs(os.path.dirname(symlink_src))
213      os.mknod(symlink_src)
214      symlink_dest = 'device/oem4/symlink_dest.mk'
215      os.makedirs(os.path.dirname(symlink_dest))
216      os.symlink(symlink_src, symlink_dest)
217      # Only append the symlink destination, not where the symlink points to.
218      # (The Kati stamp file does not resolve symlink sources.)
219      makefiles.append(symlink_dest)
220
221      # Mock the output of ckati_stamp_dump:
222      mock_check_output.return_value = '\n'.join(makefiles).encode()
223
224      kati_makefiles = manifest_split.get_kati_makefiles(
225          'stamp-file', ['overlays/oem_overlay/'])
226      self.assertEqual(
227          kati_makefiles,
228          set([
229              # Regular product makefiles
230              'device/oem1/product1.mk',
231              'device/oem2/product2.mk',
232              # Product makefile remapped from an overlay
233              'device/oem3/product3.mk',
234              # Product makefile symlink and its source
235              'device/oem4/symlink_dest.mk',
236              'vendor/oem4/symlink_src.mk',
237          ]))
238
239  def test_scan_repo_projects(self):
240    repo_projects = {
241        'system/project1': 'platform/project1',
242        'system/project2': 'platform/project2',
243    }
244    self.assertEqual(
245        manifest_split.scan_repo_projects(repo_projects,
246                                          'system/project1/path/to/file.h'),
247        'system/project1')
248    self.assertEqual(
249        manifest_split.scan_repo_projects(
250            repo_projects, 'system/project2/path/to/another_file.cc'),
251        'system/project2')
252    self.assertIsNone(
253        manifest_split.scan_repo_projects(
254            repo_projects, 'system/project3/path/to/unknown_file.h'))
255
256  def test_get_input_projects(self):
257    repo_projects = {
258        'system/project1': 'platform/project1',
259        'system/project2': 'platform/project2',
260        'system/project4': 'platform/project4',
261    }
262    inputs = [
263        'system/project1/path/to/file.h',
264        'out/path/to/out/file.h',
265        'system/project2/path/to/another_file.cc',
266        'system/project3/path/to/unknown_file.h',
267        '/tmp/absolute/path/file.java',
268    ]
269    self.assertEqual(
270        manifest_split.get_input_projects(repo_projects, inputs), {
271            'platform/project1': ['system/project1/path/to/file.h'],
272            'platform/project2': ['system/project2/path/to/another_file.cc'],
273        })
274
275  def test_update_manifest(self):
276    manifest_contents = """
277      <manifest>
278        <project name="platform/project1" path="system/project1" />
279        <project name="platform/project2" path="system/project2" />
280        <project name="platform/project3" path="system/project3" />
281      </manifest>"""
282    input_projects = set(['platform/project1', 'platform/project3'])
283    remove_projects = set(['platform/project3'])
284    manifest = manifest_split.update_manifest(
285        ET.ElementTree(ET.fromstring(manifest_contents)), input_projects,
286        remove_projects)
287
288    projects = manifest.getroot().findall('project')
289    self.assertEqual(len(projects), 1)
290    self.assertEqual(
291        ET.tostring(projects[0]).strip().decode(),
292        '<project name="platform/project1" path="system/project1" />')
293
294  @mock.patch.object(subprocess, 'check_output', autospec=True)
295  def test_create_split_manifest(self, mock_check_output):
296    with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \
297      tempfile.NamedTemporaryFile('w+t') as manifest_file, \
298      tempfile.NamedTemporaryFile('w+t') as module_info_file, \
299      tempfile.NamedTemporaryFile('w+t') as config_file, \
300      tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \
301      tempfile.TemporaryDirectory() as temp_dir:
302
303      os.chdir(temp_dir)
304
305      repo_list_file.write("""
306        system/project1 : platform/project1
307        system/project2 : platform/project2
308        system/project3 : platform/project3
309        system/project4 : platform/project4
310        system/project5 : platform/project5
311        system/project6 : platform/project6
312        system/project7 : platform/project7
313        system/project8 : platform/project8
314        system/project9 : platform/project9
315        vendor/project1 : vendor/project1""")
316      repo_list_file.flush()
317
318      manifest_file.write("""
319        <manifest>
320          <project name="platform/project1" path="system/project1" />
321          <project name="platform/project2" path="system/project2" />
322          <project name="platform/project3" path="system/project3" />
323          <project name="platform/project4" path="system/project4" />
324          <project name="platform/project5" path="system/project5" />
325          <project name="platform/project6" path="system/project6" />
326          <project name="platform/project7" path="system/project7" />
327          <project name="platform/project8" path="system/project8" />
328          <project name="platform/project9" path="system/project9" />
329          <project name="vendor/project1" path="vendor/project1" />
330        </manifest>""")
331      manifest_file.flush()
332
333      module_info_file.write("""{
334        "droid": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": [] },
335        "target_a": { "class": ["EXECUTABLES"], "path": ["out/project2"], "dependencies": ["unknown_module_a"] },
336        "target_b": { "class": ["EXECUTABLES"], "path": ["system/project3"], "dependencies": ["target_f", "unknown_module_b"] },
337        "target_c": { "class": ["EXECUTABLES"], "path": ["system/project4"], "dependencies": [] },
338        "target_d": { "class": ["EXECUTABLES"], "path": ["system/project5"], "dependencies": [] },
339        "target_e": { "class": ["EXECUTABLES"], "path": ["system/project6"], "dependencies": [] },
340        "target_f": { "class": ["HEADER_LIBRARIES"], "path": ["system/project7"], "dependencies": [] },
341        "target_g": { "class": ["SHARED_LIBRARIES"], "path": ["system/project8"], "dependencies": ["target_h"] },
342        "target_h": { "class": ["HEADER_LIBRARIES"], "path": ["system/project9"], "dependencies": [] }
343      }""")
344      module_info_file.flush()
345
346      # droid needs inputs from project1 and project3
347      ninja_inputs_droid = b"""
348      system/project1/file1
349      system/project1/file2
350      system/project3/file1
351      """
352
353      # target_b (indirectly included due to being in project3) needs inputs
354      # from project3 and project4
355      ninja_inputs_target_b = b"""
356      system/project3/file2
357      system/project4/file1
358      """
359
360      # target_c (indirectly included due to being in project4) needs inputs
361      # from only project4
362      ninja_inputs_target_c = b"""
363      system/project4/file2
364      system/project4/file3
365      """
366
367      product_makefile = 'vendor/project1/product.mk'
368      os.makedirs(os.path.dirname(product_makefile))
369      os.mknod(product_makefile)
370      kati_stamp_dump = product_makefile.encode()
371
372      mock_check_output.side_effect = [
373          ninja_inputs_droid,
374          kati_stamp_dump,
375          ninja_inputs_target_b,
376          ninja_inputs_target_c,
377      ]
378
379      # The config file says to manually include project6
380      config_file.write("""
381        <config>
382          <add_project name="platform/project6" />
383        </config>""")
384      config_file.flush()
385
386      debug_file = os.path.join(temp_dir, 'debug.json')
387
388      manifest_split.create_split_manifest(
389          ['droid'], manifest_file.name, split_manifest_file.name,
390          [config_file.name], repo_list_file.name, 'build-target.ninja',
391          'ninja', module_info_file.name, 'unused kati stamp',
392          ['unused overlay'], [], debug_file)
393      split_manifest = ET.parse(split_manifest_file.name)
394      split_manifest_projects = [
395          child.attrib['name']
396          for child in split_manifest.getroot().findall('project')
397      ]
398      self.assertEqual(
399          split_manifest_projects,
400          [
401              # From droid
402              'platform/project1',
403              # From droid
404              'platform/project3',
405              # From target_b (module within project3, indirect dependency)
406              'platform/project4',
407              # Manual inclusion from config file
408              'platform/project6',
409              # From target_b (depends on target_f header library)
410              'platform/project7',
411              # Inclusion from the Kati makefile stamp
412              'vendor/project1',
413          ])
414
415      with open(debug_file) as debug_fp:
416        debug_data = json.load(debug_fp)
417
418        # Dependency for droid, but no other adjacent modules
419        self.assertTrue(debug_data['platform/project1']['direct_input'])
420        self.assertFalse(debug_data['platform/project1']['adjacent_input'])
421        self.assertFalse(debug_data['platform/project1']['deps_input'])
422
423        # Dependency for droid and an adjacent module
424        self.assertTrue(debug_data['platform/project3']['direct_input'])
425        self.assertTrue(debug_data['platform/project3']['adjacent_input'])
426        self.assertFalse(debug_data['platform/project3']['deps_input'])
427
428        # Dependency only for an adjacent module
429        self.assertFalse(debug_data['platform/project4']['direct_input'])
430        self.assertTrue(debug_data['platform/project4']['adjacent_input'])
431        self.assertFalse(debug_data['platform/project4']['deps_input'])
432
433        # Included via header library
434        self.assertFalse(debug_data['platform/project7']['direct_input'])
435        self.assertFalse(debug_data['platform/project7']['adjacent_input'])
436        self.assertTrue(debug_data['platform/project7']['deps_input'])
437
438        # Included due to the config file
439        self.assertEqual(
440            debug_data['platform/project6']['manual_add_config'],
441            config_file.name)
442
443        # Included due to the Kati makefile stamp
444        self.assertEqual(debug_data['vendor/project1']['kati_makefiles'][0],
445                         product_makefile)
446
447  @mock.patch.object(manifest_split, 'get_ninja_inputs', autospec=True)
448  @mock.patch.object(manifest_split, 'get_kati_makefiles', autospec=True)
449  @mock.patch.object(manifest_split.ModuleInfo, '__init__', autospec=True)
450  def test_create_split_manifest_skip_kati_module_info(self, mock_init,
451                                                       mock_get_kati_makefiles,
452                                                       mock_get_ninja_inputs):
453    with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \
454            tempfile.NamedTemporaryFile('w+t') as manifest_file, \
455            tempfile.NamedTemporaryFile('w+t') as module_info_file, \
456            tempfile.NamedTemporaryFile('w+t') as config_file, \
457            tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \
458            tempfile.TemporaryDirectory() as temp_dir:
459
460      os.chdir(temp_dir)
461
462      manifest_file.write("""
463        <manifest>
464        </manifest>""")
465      manifest_file.flush()
466
467      manifest_split.create_split_manifest(
468          targets=['droid'],
469          manifest_file=manifest_file.name,
470          split_manifest_file=split_manifest_file.name,
471          config_files=[],
472          repo_list_file=repo_list_file.name,
473          ninja_build_file='build-target.ninja',
474          ninja_binary='ninja',
475          kati_stamp_file=None,
476          module_info_file=None,
477          overlays=[],
478          installed_prebuilts=[],
479          debug_file=None)
480
481    mock_get_ninja_inputs.assert_called_with(
482        'ninja', 'build-target.ninja', ['droid'])
483    mock_get_kati_makefiles.assert_not_called()
484    mock_init.assert_not_called()
485
486  @mock.patch.object(subprocess, 'check_output', autospec=True)
487  def test_create_split_manifest_installed_prebuilt(self, mock_check_output):
488
489    # The purpose of this test is to verify that create_split_manifests treats
490    # installed prebuilts as projects, even though the installed prebuilts are
491    # not in the manifest. This use case occurs when installed prebuilts
492    # contribute modules to the build, but the installed prebuilts themselves
493    # aren't sourced from the manifest.
494
495    with tempfile.NamedTemporaryFile('w+t') as repo_list_file, \
496      tempfile.NamedTemporaryFile('w+t') as manifest_file, \
497      tempfile.NamedTemporaryFile('w+t') as module_info_file, \
498      tempfile.NamedTemporaryFile('w+t') as split_manifest_file, \
499      tempfile.TemporaryDirectory() as temp_dir:
500
501      os.chdir(temp_dir)
502
503      repo_list_file.write("""
504        system/project1 : platform/project1
505        vendor/project1 : vendor/project1""")
506      repo_list_file.flush()
507
508      # Here we have small manifest that does not include "prebuilt/project3"
509      # or "prebuilt/project4".
510
511      manifest_file.write("""
512        <manifest>
513          <project name="platform/project1" path="system/project1" />
514          <project name="vendor/project1" path="vendor/project1" />
515        </manifest>""")
516      manifest_file.flush()
517
518      # Here's the module_info.json file. It contains modules whose paths are
519      # "prebuilt/project3" and "prebult/project4", which are not found in the
520      # manifest. Normally create_split_manifest doesn't tolerate a path that
521      # doesn't correspond to a manifest project. However, this test verifies
522      # that you can use these modules if you tell create_split_manifest about
523      # the installed prebuilts via a parameter.
524
525      module_info_file.write("""{
526        "droid": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": [] },
527        "target_a": { "class": ["EXECUTABLES"], "path": ["system/project1"], "dependencies": ["target_b", "target_c"] },
528        "target_b": { "class": ["SHARED_LIBRARIES"], "path": ["prebuilt/project3"], "dependencies": [] },
529        "target_c": { "class": ["SHARED_LIBRARIES"], "path": ["prebuilt/project4"], "dependencies": [] }
530      }""")
531      module_info_file.flush()
532
533      # droid needs inputs from project1
534      ninja_inputs_droid = b"""
535      system/project1/file1
536      """
537
538      # target_a needs inputs from prebuilt/project3 and prebuilt/project4
539      ninja_inputs_target_a = b"""
540      prebuilt/project3/file2
541      prebuilt/project4/file3
542      """
543
544      # target_b needs inputs from prebuilt/project3
545      ninja_inputs_target_b = b"""
546      prebuilt/project3/file4
547      """
548
549      # target_c needs inputs from prebuilt/project4
550      ninja_inputs_target_c = b"""
551      prebuilt/project4/file5
552      """
553
554      product_makefile = 'vendor/project1/product.mk'
555      os.makedirs(os.path.dirname(product_makefile))
556      os.mknod(product_makefile)
557      kati_stamp_dump = product_makefile.encode()
558
559      mock_check_output.side_effect = [
560          ninja_inputs_droid,
561          kati_stamp_dump,
562          ninja_inputs_target_a,
563          ninja_inputs_target_b,
564          ninja_inputs_target_c,
565      ]
566
567      debug_file = os.path.join(temp_dir, 'debug.json')
568
569      manifest_split.create_split_manifest(
570          targets=['droid'],
571          manifest_file=manifest_file.name,
572          split_manifest_file=split_manifest_file.name,
573          config_files=[],
574          repo_list_file=repo_list_file.name,
575          ninja_build_file='build-target.ninja',
576          ninja_binary='ninja',
577          module_info_file=module_info_file.name,
578          kati_stamp_file='unused kati stamp',
579          overlays=['unused overlay'],
580
581          # This is a key part of the test. Passing these two "projects" as
582          # prebuilts allows create_split_manifest to recognize them as
583          # projects even though they are not in the manifest.
584
585          installed_prebuilts=['prebuilt/project3', 'prebuilt/project4'],
586
587          debug_file = debug_file)
588
589      split_manifest = ET.parse(split_manifest_file.name)
590
591      split_manifest_projects = [
592          child.attrib['name']
593          for child in split_manifest.getroot().findall('project')
594      ]
595
596      # Note that the installed prebuilts do not appear in the final split
597      # manfiest output because they were not in the manifest to begin with.
598
599      self.assertEqual(
600          split_manifest_projects,
601          [
602              # From droid
603              'platform/project1',
604              # Inclusion from the Kati makefile stamp
605              'vendor/project1',
606          ])
607
608      with open(debug_file) as debug_fp:
609        debug_data = json.load(debug_fp)
610
611        # Dependency for droid, but no other adjacent modules
612        self.assertTrue(debug_data['platform/project1']['direct_input'])
613        self.assertFalse(debug_data['platform/project1']['adjacent_input'])
614        self.assertFalse(debug_data['platform/project1']['deps_input'])
615
616        # Included due to the Kati makefile stamp
617        self.assertEqual(debug_data['vendor/project1']['kati_makefiles'][0],
618                         product_makefile)
619
620
621if __name__ == '__main__':
622  unittest.main()
623