• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env vpython3
2# Copyright 2020 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Regenerate all program and project configs and create commits.
6
7Example usage:
8./gen_all_configs.py -r src/program -b testbranch -j 8 -m "Regen configs."
9"""
10
11from typing import List
12
13import argparse
14import collections
15import functools
16import os
17import multiprocessing
18import subprocess
19import warnings
20
21# The root of the Chromiumos checkout.
22SRC_ROOT = os.path.realpath(os.path.join(__file__, '../../../../..'))
23
24# Describes a repo. Name is like 'chromeos/program/testprogram', path is like
25# 'src/program/testprogram'.
26RepoInfo = collections.namedtuple('RepoInfo', ['name', 'path'])
27
28
29class ConfigGenerationError(Exception):
30  """Exception raised for errors during config generation."""
31
32
33def get_repos(regexes: List[str]) -> List[RepoInfo]:
34  """Returns a list of RepoInfo for repos matching regexes.
35
36  Args:
37    regexes: A list of regexes, in format expected by `repo forall`.
38  """
39  cmd = "repo forall -r {} -c 'echo $REPO_PROJECT,$REPO_PATH'".format(
40      ' '.join(regexes))
41  repo_process = subprocess.run(
42      cmd, shell=True, check=True, capture_output=True)
43
44  repo_infos: List[RepoInfo] = []
45  for line in repo_process.stdout.decode().strip().split():
46    name, path = line.split(',')
47    repo_infos.append(RepoInfo(name, path))
48
49  if not repo_infos:
50    raise ValueError('No repos matched regexes {}'.format(regexes))
51
52  return repo_infos
53
54
55def regen_configs(path: str, branch: str, message: str):
56  """Generates all configs found under path and makes a commit if necessary.
57
58  Args:
59    path: A path, relative to the Chromiumos checkout, e.g.
60      'src/program/testprogram'.
61    branch: See add_argument help string.
62    message: See add_argument help string.
63  """
64  print('Generating configs for {}...'.format(path))
65  fullpath = os.path.join(SRC_ROOT, path)
66
67  # Change to the project directory, which will be necessary for `git commit`
68  # anyway.
69  os.chdir(fullpath)
70  subprocess.run(['repo', 'start', branch, '.'],
71                 check=True,
72                 capture_output=True)
73
74  config_paths = []
75  for dirpath, _, filenames in os.walk('.', followlinks=False):
76    if 'config.star' in filenames:
77      config_paths.append(os.path.join(dirpath, 'config.star'))
78
79  if not config_paths:
80    warnings.warn('No config.star files found in {}'.format(path))
81
82  for config in config_paths:
83    try:
84      subprocess.run(['./config/bin/gen_config', config], check=True)
85    except subprocess.CalledProcessError as ex:
86      raise ConfigGenerationError(
87          'Failed to generate config for {}'.format(path)) from ex
88
89  # Check if any files were changed.
90  status_process = subprocess.run(['git', 'status', '--porcelain'],
91                                  check=True,
92                                  capture_output=True)
93  if status_process.stdout:
94    print('Committing config changes in {}.'.format(path))
95    subprocess.run(['git', 'commit', '-a', '-m', message],
96                   check=True,
97                   capture_output=True)
98  else:
99    print('No config changes in {}'.format(path))
100
101
102def main():
103  parser = argparse.ArgumentParser(description=__doc__)
104  parser.add_argument(
105      '-r',
106      '--regex',
107      nargs='*',
108      default=['src/program', 'src/project'],
109      help=('Generate configs on projects matching regex. Defaults to'
110            "['src/program', 'src/project']. Format is as expected by `repo "
111            'forall`'))
112  parser.add_argument(
113      '-b',
114      '--branch',
115      required=True,
116      help=('Name of the branch to commit config changes to. Does not need to'
117            ' already exist. Passed to `repo start`.'))
118  parser.add_argument(
119      '-m',
120      '--message',
121      required=True,
122      help='Commit message for each of the config changes.')
123  parser.add_argument(
124      '--no-sync',
125      action='store_true',
126      help='Skip syncing before generating configs.')
127  parser.add_argument(
128      '-j',
129      '--jobs',
130      default=1,
131      type=int,
132      help='Number of commands to execute simultaneously')
133
134  args = parser.parse_args()
135
136  repo_infos = get_repos(args.regex)
137  repo_names, repo_paths = zip(*repo_infos)
138
139  # repo doesn't like when sync is run concurrently, so run in the main thread.
140  if not args.no_sync:
141    print('Syncing repos: {}'.format(', '.join(repo_names)))
142    subprocess.run(['repo', 'sync', '-j',
143                    str(args.jobs), *repo_names],
144                   check=True,
145                   capture_output=True)
146
147  pool = multiprocessing.Pool(args.jobs)
148
149  # map doesn't accept lambdas, so use functools.partial.
150  pool.map(
151      functools.partial(
152          regen_configs,
153          branch=args.branch,
154          message=args.message,
155      ),
156      repo_paths,
157  )
158
159  print("Upload changes with 'repo upload --br={} --ht={}'".format(
160      args.branch, args.branch))
161
162
163if __name__ == '__main__':
164  main()
165