• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (C) 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15import dataclasses
16import enum
17import functools
18import io
19import logging
20import os
21import shutil
22import tempfile
23import textwrap
24import uuid
25from enum import Enum
26from pathlib import Path
27from typing import Callable, Optional
28from typing import Final
29from typing import TypeAlias
30
31import util
32import ui
33
34"""
35Provides some representative CUJs. If you wanted to manually run something but
36would like the metrics to be collated in the metrics.csv file, use
37`perf_metrics.py` as a stand-alone after your build.
38"""
39
40
41class BuildResult(Enum):
42  SUCCESS = enum.auto()
43  FAILED = enum.auto()
44  TEST_FAILURE = enum.auto()
45
46
47Action: TypeAlias = Callable[[], None]
48Verifier: TypeAlias = Callable[[], None]
49
50
51def skip_when_soong_only(func: Verifier) -> Verifier:
52  """A decorator for Verifiers that are not applicable to soong-only builds"""
53
54  def wrapper():
55    if InWorkspace.ws_counterpart(util.get_top_dir()).exists():
56      func()
57
58  return wrapper
59
60
61@skip_when_soong_only
62def verify_symlink_forest_has_only_symlink_leaves():
63  """Verifies that symlink forest has only symlinks or directories but no
64  files except for merged BUILD.bazel files"""
65
66  top_in_ws = InWorkspace.ws_counterpart(util.get_top_dir())
67
68  for root, dirs, files in os.walk(top_in_ws, topdown=True, followlinks=False):
69    for file in files:
70      if file == 'symlink_forest_version' and top_in_ws.samefile(root):
71        continue
72      f = Path(root).joinpath(file)
73      if file != 'BUILD.bazel' and not f.is_symlink():
74        raise AssertionError(f'{f} unexpected')
75
76  logging.info('VERIFIED Symlink Forest has no real files except BUILD.bazel')
77
78
79@dataclasses.dataclass(frozen=True)
80class CujStep:
81  verb: str
82  """a human-readable description"""
83  apply_change: Action
84  """user action(s) that are performed prior to a build attempt"""
85  verify: Verifier = verify_symlink_forest_has_only_symlink_leaves
86  """post-build assertions, i.e. tests.
87  Should raise `Exception` for failures.
88  """
89
90
91@dataclasses.dataclass(frozen=True)
92class CujGroup:
93  """A sequence of steps to be performed, such that at the end of all steps the
94  initial state of the source tree is attained.
95  NO attempt is made to achieve atomicity programmatically. It is left as the
96  responsibility of the user.
97  """
98  description: str
99  steps: list[CujStep]
100
101  def __str__(self) -> str:
102    if len(self.steps) < 2:
103      return f'{self.steps[0].verb} {self.description}'.strip()
104    return ' '.join(
105        [f'({chr(ord("a") + i)}) {step.verb} {self.description}'.strip() for
106         i, step in enumerate(self.steps)])
107
108
109Warmup: Final[CujGroup] = CujGroup('WARMUP',
110                                   [CujStep('no change', lambda: None)])
111
112
113class InWorkspace(Enum):
114  """For a given file in the source tree, the counterpart in the symlink forest
115   could be one of these kinds.
116  """
117  SYMLINK = enum.auto()
118  NOT_UNDER_SYMLINK = enum.auto()
119  UNDER_SYMLINK = enum.auto()
120  OMISSION = enum.auto()
121
122  @staticmethod
123  def ws_counterpart(src_path: Path) -> Path:
124    return util.get_out_dir().joinpath('soong/workspace').joinpath(
125        de_src(src_path))
126
127  def verifier(self, src_path: Path) -> Verifier:
128    @skip_when_soong_only
129    def f():
130      ws_path = InWorkspace.ws_counterpart(src_path)
131      actual: Optional[InWorkspace] = None
132      if ws_path.is_symlink():
133        actual = InWorkspace.SYMLINK
134        if not ws_path.exists():
135          logging.warning('Dangling symlink %s', ws_path)
136      elif not ws_path.exists():
137        actual = InWorkspace.OMISSION
138      else:
139        for p in ws_path.parents:
140          if not p.is_relative_to(util.get_out_dir()):
141            actual = InWorkspace.NOT_UNDER_SYMLINK
142            break
143          if p.is_symlink():
144            actual = InWorkspace.UNDER_SYMLINK
145            break
146
147      if self != actual:
148        raise AssertionError(
149            f'{ws_path} expected {self.name} but got {actual.name}')
150      logging.info(f'VERIFIED {de_src(ws_path)} {self.name}')
151
152    return f
153
154
155def de_src(p: Path) -> str:
156  return str(p.relative_to(util.get_top_dir()))
157
158
159def src(p: str) -> Path:
160  return util.get_top_dir().joinpath(p)
161
162
163def modify_revert(file: Path, text: str = '//BOGUS line\n') -> CujGroup:
164  """
165  :param file: the file to be modified and reverted
166  :param text: the text to be appended to the file to modify it
167  :return: A pair of CujSteps, where the first modifies the file and the
168  second reverts the modification
169  """
170  if not file.exists():
171    raise RuntimeError(f'{file} does not exist')
172
173  def add_line():
174    with open(file, mode="a") as f:
175      f.write(text)
176
177  def revert():
178    with open(file, mode="rb+") as f:
179      # assume UTF-8
180      f.seek(-len(text), io.SEEK_END)
181      f.truncate()
182
183  return CujGroup(de_src(file), [
184      CujStep('modify', add_line),
185      CujStep('revert', revert)
186  ])
187
188
189def create_delete(file: Path, ws: InWorkspace,
190    text: str = '//Test File: safe to delete\n') -> CujGroup:
191  """
192  :param file: the file to be created and deleted
193  :param ws: the expectation for the counterpart file in symlink
194  forest (aka the synthetic bazel workspace) when its created
195  :param text: the content of the file
196  :return: A pair of CujSteps, where the fist creates the file and the
197  second deletes it
198  """
199  missing_dirs = [f for f in file.parents if not f.exists()]
200  shallowest_missing_dir = missing_dirs[-1] if len(missing_dirs) else None
201
202  def create():
203    if file.exists():
204      raise RuntimeError(
205          f'File {file} already exists. Interrupted an earlier run?\n'
206          'TIP: `repo status` and revert changes!!!')
207    file.parent.mkdir(parents=True, exist_ok=True)
208    file.touch(exist_ok=False)
209    with open(file, mode="w") as f:
210      f.write(text)
211
212  def delete():
213    if shallowest_missing_dir:
214      shutil.rmtree(shallowest_missing_dir)
215    else:
216      file.unlink(missing_ok=False)
217
218  return CujGroup(de_src(file), [
219      CujStep('create', create, ws.verifier(file)),
220      CujStep('delete', delete, InWorkspace.OMISSION.verifier(file)),
221  ])
222
223
224def create_delete_bp(bp_file: Path) -> CujGroup:
225  """
226  This is basically the same as "create_delete" but with canned content for
227  an Android.bp file.
228  """
229  return create_delete(
230      bp_file, InWorkspace.SYMLINK,
231      'filegroup { name: "test-bogus-filegroup", srcs: ["**/*.md"] }')
232
233
234def delete_restore(original: Path, ws: InWorkspace) -> CujGroup:
235  """
236  :param original: The file to be deleted then restored
237  :param ws: When restored, expectation for the file's counterpart in the
238  symlink forest (aka synthetic bazel workspace)
239  :return: A pair of CujSteps, where the first deletes a file and the second
240  restores it
241  """
242  tempdir = Path(tempfile.gettempdir())
243  if tempdir.is_relative_to(util.get_top_dir()):
244    raise SystemExit(f'Temp dir {tempdir} is under source tree')
245  if tempdir.is_relative_to(util.get_out_dir()):
246    raise SystemExit(f'Temp dir {tempdir} is under '
247                     f'OUT dir {util.get_out_dir()}')
248  copied = tempdir.joinpath(f'{original.name}-{uuid.uuid4()}.bak')
249
250  def move_to_tempdir_to_mimic_deletion():
251    logging.warning('MOVING %s TO %s', de_src(original), copied)
252    original.rename(copied)
253
254  return CujGroup(de_src(original), [
255      CujStep('delete',
256              move_to_tempdir_to_mimic_deletion,
257              InWorkspace.OMISSION.verifier(original)),
258      CujStep('restore',
259              lambda: copied.rename(original),
260              ws.verifier(original))
261  ])
262
263
264def replace_link_with_dir(p: Path):
265  """Create a file, replace it with a non-empty directory, delete it"""
266  cd = create_delete(p, InWorkspace.SYMLINK)
267  create_file: CujStep
268  delete_file: CujStep
269  create_file, delete_file, *tail = cd.steps
270  assert len(tail) == 0
271
272  # an Android.bp is always a symlink in the workspace and thus its parent
273  # will be a directory in the workspace
274  create_dir: CujStep
275  delete_dir: CujStep
276  create_dir, delete_dir, *tail = create_delete_bp(
277      p.joinpath('Android.bp')).steps
278  assert len(tail) == 0
279
280  def replace_it():
281    delete_file.apply_change()
282    create_dir.apply_change()
283
284  return CujGroup(cd.description, [
285      create_file,
286      CujStep(f'{de_src(p)}/Android.bp instead of',
287              replace_it,
288              create_dir.verify),
289      delete_dir
290  ])
291
292
293def _sequence(*vs: Verifier) -> Verifier:
294  def f():
295    for v in vs:
296      v()
297
298  return f
299
300
301def content_verfiers(
302    ws_build_file: Path, content: str) -> (Verifier, Verifier):
303  def search() -> bool:
304    with open(ws_build_file, "r") as f:
305      for line in f:
306        if line == content:
307          return True
308    return False
309
310  @skip_when_soong_only
311  def contains():
312    if not search():
313      raise AssertionError(
314          f'{de_src(ws_build_file)} expected to contain {content}')
315    logging.info(f'VERIFIED {de_src(ws_build_file)} contains {content}')
316
317  @skip_when_soong_only
318  def does_not_contain():
319    if search():
320      raise AssertionError(
321          f'{de_src(ws_build_file)} not expected to contain {content}')
322    logging.info(f'VERIFIED {de_src(ws_build_file)} does not contain {content}')
323
324  return contains, does_not_contain
325
326
327def modify_revert_kept_build_file(build_file: Path) -> CujGroup:
328  content = f'//BOGUS {uuid.uuid4()}\n'
329  step1, step2, *tail = modify_revert(build_file, content).steps
330  assert len(tail) == 0
331  ws_build_file = InWorkspace.ws_counterpart(build_file).with_name(
332      'BUILD.bazel')
333  merge_prover, merge_disprover = content_verfiers(ws_build_file, content)
334  return CujGroup(de_src(build_file), [
335      CujStep(step1.verb,
336              step1.apply_change,
337              _sequence(step1.verify, merge_prover)),
338      CujStep(step2.verb,
339              step2.apply_change,
340              _sequence(step2.verify, merge_disprover))
341  ])
342
343
344def create_delete_kept_build_file(build_file: Path) -> CujGroup:
345  content = f'//BOGUS {uuid.uuid4()}\n'
346  ws_build_file = InWorkspace.ws_counterpart(build_file).with_name(
347      'BUILD.bazel')
348  if build_file.name == 'BUILD.bazel':
349    ws = InWorkspace.NOT_UNDER_SYMLINK
350  elif build_file.name == 'BUILD':
351    ws = InWorkspace.SYMLINK
352  else:
353    raise RuntimeError(f'Illegal name for a build file {build_file}')
354
355  merge_prover, merge_disprover = content_verfiers(ws_build_file, content)
356
357  step1: CujStep
358  step2: CujStep
359  step1, step2, *tail = create_delete(build_file, ws, content).steps
360  assert len(tail) == 0
361  return CujGroup(de_src(build_file), [
362      CujStep(step1.verb,
363              step1.apply_change,
364              _sequence(step1.verify, merge_prover)),
365      CujStep(step2.verb,
366              step2.apply_change,
367              _sequence(step2.verify, merge_disprover))
368  ])
369
370
371def create_delete_unkept_build_file(build_file) -> CujGroup:
372  content = f'//BOGUS {uuid.uuid4()}\n'
373  ws_build_file = InWorkspace.ws_counterpart(build_file).with_name(
374      'BUILD.bazel')
375  step1: CujStep
376  step2: CujStep
377  step1, step2, *tail = create_delete(
378      build_file, InWorkspace.SYMLINK, content).steps
379  assert len(tail) == 0
380  _, merge_disprover = content_verfiers(ws_build_file, content)
381  return CujGroup(de_src(build_file), [
382      CujStep(step1.verb,
383              step1.apply_change,
384              _sequence(step1.verify, merge_disprover)),
385      CujStep(step2.verb,
386              step2.apply_change,
387              _sequence(step2.verify, merge_disprover))
388  ])
389
390
391NON_LEAF = '*/*'
392"""If `a/*/*` is a valid path `a` is not a leaf directory"""
393LEAF = '!*/*'
394"""If `a/*/*` is not a valid path `a` is a leaf directory, i.e. has no other
395non-empty sub-directories"""
396PKG = ['Android.bp', '!BUILD', '!BUILD.bazel']
397"""limiting the candidate to Android.bp file with no sibling bazel files"""
398PKG_FREE = ['!**/Android.bp', '!**/BUILD', '!**/BUILD.bazel']
399"""no Android.bp or BUILD or BUILD.bazel file anywhere"""
400
401
402def _kept_build_cujs() -> list[CujGroup]:
403  # Bp2BuildKeepExistingBuildFile(build/bazel) is True(recursive)
404  kept = src('build/bazel')
405  pkg = util.any_dir_under(kept, *PKG)
406  examples = [pkg.joinpath('BUILD'),
407              pkg.joinpath('BUILD.bazel')]
408
409  return [
410      *[create_delete_kept_build_file(build_file) for build_file in examples],
411      create_delete(pkg.joinpath('BUILD/kept-dir'), InWorkspace.SYMLINK),
412      modify_revert_kept_build_file(util.any_file_under(kept, 'BUILD'))]
413
414
415def _unkept_build_cujs() -> list[CujGroup]:
416  # Bp2BuildKeepExistingBuildFile(bionic) is False(recursive)
417  unkept = src('bionic')
418  pkg = util.any_dir_under(unkept, *PKG)
419  return [
420      *[create_delete_unkept_build_file(build_file) for build_file in [
421          pkg.joinpath('BUILD'),
422          pkg.joinpath('BUILD.bazel'),
423      ]],
424      *[create_delete(build_file, InWorkspace.OMISSION) for build_file in [
425          unkept.joinpath('bogus-unkept/BUILD'),
426          unkept.joinpath('bogus-unkept/BUILD.bazel'),
427      ]],
428      create_delete(pkg.joinpath('BUILD/unkept-dir'), InWorkspace.SYMLINK)
429  ]
430
431
432@functools.cache
433def get_cujgroups() -> list[CujGroup]:
434  # we are choosing "package" directories that have Android.bp but
435  # not BUILD nor BUILD.bazel because
436  # we can't tell if ShouldKeepExistingBuildFile would be True or not
437  pkg, p_why = util.any_match(NON_LEAF, *PKG)
438  pkg_free, f_why = util.any_match(NON_LEAF, *PKG_FREE)
439  leaf_pkg_free, _ = util.any_match(LEAF, *PKG_FREE)
440  ancestor, a_why = util.any_match('!Android.bp', '!BUILD', '!BUILD.bazel',
441                                   '**/Android.bp')
442  logging.info(textwrap.dedent(f'''Choosing:
443            package: {de_src(pkg)} has {p_why}
444   package ancestor: {de_src(ancestor)} has {a_why} but no direct Android.bp
445       package free: {de_src(pkg_free)} has {f_why} but no Android.bp anywhere
446  leaf package free: {de_src(leaf_pkg_free)} has neither Android.bp nor sub-dirs
447  '''))
448
449  android_bp_cujs = [
450      modify_revert(src('Android.bp')),
451      *[create_delete_bp(d.joinpath('Android.bp')) for d in
452        [ancestor, pkg_free, leaf_pkg_free]]
453  ]
454  mixed_build_launch_cujs = [
455      modify_revert(src('bionic/libc/tzcode/asctime.c')),
456      modify_revert(src('bionic/libc/stdio/stdio.cpp')),
457      modify_revert(src('packages/modules/adb/daemon/main.cpp')),
458      modify_revert(src('frameworks/base/core/java/android/view/View.java')),
459  ]
460  unreferenced_file_cujs = [
461      *[create_delete(d.joinpath('unreferenced.txt'), InWorkspace.SYMLINK) for
462        d in [ancestor, pkg]],
463      *[create_delete(d.joinpath('unreferenced.txt'), InWorkspace.UNDER_SYMLINK)
464        for d
465        in [pkg_free, leaf_pkg_free]]
466  ]
467
468  def clean():
469    if ui.get_user_input().log_dir.is_relative_to(util.get_top_dir()):
470      raise AssertionError(
471          f'specify a different LOG_DIR: {ui.get_user_input().log_dir}')
472    if util.get_out_dir().exists():
473      shutil.rmtree(util.get_out_dir())
474
475  return [
476      CujGroup('', [CujStep('clean', clean)]),
477      Warmup,
478
479      create_delete(src('bionic/libc/tzcode/globbed.c'),
480                    InWorkspace.UNDER_SYMLINK),
481
482      # TODO (usta): find targets that should be affected
483      *[delete_restore(f, InWorkspace.SYMLINK) for f in [
484          util.any_file('version_script.txt'),
485          util.any_file('AndroidManifest.xml')]],
486
487      *unreferenced_file_cujs,
488      *mixed_build_launch_cujs,
489      *android_bp_cujs,
490      *_unkept_build_cujs(),
491      *_kept_build_cujs(),
492      replace_link_with_dir(pkg.joinpath('bogus.txt')),
493      # TODO(usta): add a dangling symlink
494  ]
495