• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2021 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Regenerate all project configs locally and generate a diff for them."""
6
7import argparse
8import atexit
9import collections
10import functools
11import glob
12import itertools
13import json
14import logging
15import multiprocessing
16import multiprocessing.pool
17import os
18import pathlib
19import shutil
20import subprocess
21import sys
22import tempfile
23import time
24
25from common import utilities
26
27# resolve relative directories
28this_dir = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
29path_projects = this_dir / "../../project"
30
31
32def run_config(config, logname=None):
33  """Run a single config job, return diff of current and new output.
34
35  Args:
36    config: (program, project) to configure
37    logname: Filename to redirect stderr to from config
38      default is to suppress the output
39  """
40
41  # open logfile as /dev/null by default so we don't have to check for it
42  logfile = open("/dev/null", "w")
43  if logname:
44    logfile = open(logname, "a")
45
46  def run_diff(cmd, current, output):
47    """Execute cmd and diff the current and output files"""
48    stdout = ""
49    try:
50      stdout = subprocess.run(
51          cmd,
52          stdout=subprocess.PIPE,
53          stderr=subprocess.STDOUT,
54          check=False,
55          text=True,
56          shell=True,
57      ).stdout
58    finally:
59      logfile.write(stdout)
60
61    # otherwise run diff
62    return utilities.jqdiff(
63        current if current.exists() else None,
64        output if output.exists() else None,
65    )
66
67  #### start of function body
68  program, project = config
69
70  logfile.write("== {}/{} ==================\n".format(program, project))
71
72  # path to project repo and config bundle
73  path_repo = (path_projects / program / project).resolve()
74  path_config = (path_repo / "generated/config.jsonproto").resolve()
75
76  # create temporary directory for output
77  diff = ""
78  with tempfile.TemporaryDirectory() as scratch:
79    scratch = pathlib.Path(scratch)
80
81    cmd = "cd {}; {} --no-proto --output-dir {} {}".format(
82        path_repo.resolve(),
83        (path_repo / "config/bin/gen_config").resolve(),
84        scratch,
85        path_repo / "config.star",
86    )
87
88    path_config_new = scratch / "generated/config.jsonproto"
89
90    logfile.write("repo path:  {}\n".format(path_repo))
91    logfile.write("old config: {}\n".format(path_config))
92    logfile.write("new config: {}\n".format(path_config_new))
93    logfile.write("running:    {}\n".format(cmd))
94    logfile.write("\n")
95
96    diff = run_diff(cmd, path_config, path_config_new)
97    logfile.write("\n\n")
98    logfile.close()
99
100  return ("{}-{}".format(program, project), diff)
101
102
103def run_configs(args, configs):
104  """Regenerate configuration for each project in configs.
105
106  Generate an über diff showing the changes that the current ToT
107  configuration code would generate vs what's currently committed.
108
109  Write the result to the output file specified on the command line.
110
111  Args:
112    args: command line arguments from argparse
113    configs: list of BackfillConfig instances to execute
114
115  Return:
116    nothing
117  """
118
119  # create a logfile if requested
120  kwargs = {}
121  if args.logfile:
122    # open and close the logfile to truncate it so backfills can append
123    # We can't pickle the file object and send it as an argument with
124    # multiprocessing, so this is a workaround for that limitation
125    with open(args.logfile, "w"):
126      kwargs["logname"] = args.logfile
127
128  nproc = 32
129  diffs = {}
130  nconfig = len(configs)
131  with multiprocessing.Pool(processes=nproc) as pool:
132    results = pool.imap_unordered(
133        functools.partial(run_config, **kwargs), configs, chunksize=1)
134    for ii, result in enumerate(results, 1):
135      sys.stderr.write(
136          utilities.clear_line("[{}/{}] Processing configs".format(ii,
137                                                                   nconfig)))
138
139      if result:
140        key, diff = result
141        diffs[key] = diff
142
143    sys.stderr.write(utilities.clear_line("[✔] Processing configs"))
144
145  # generate final über diff showing all the changes
146  with open(args.diff, "w") as ofile:
147    for name, result in sorted(diffs.items()):
148      ofile.write("## ---------------------\n")
149      ofile.write("## diff for {}\n".format(name))
150      ofile.write("\n")
151      ofile.write(result + "\n")
152
153
154def main():
155  parser = argparse.ArgumentParser(
156      description=__doc__,
157      formatter_class=argparse.RawTextHelpFormatter,
158  )
159
160  parser.add_argument(
161      "--diff",
162      type=str,
163      required=True,
164      help="target file for diff on config.jsonproto payload",
165  )
166
167  parser.add_argument(
168      "-l",
169      "--logfile",
170      type=str,
171      help="target file to log output from regens",
172  )
173  args = parser.parse_args()
174
175  # glob out all the config.stars in the project directory
176  strpath = str(path_projects)
177  config_stars = [
178      os.path.relpath(fname, strpath)
179      for fname in glob.glob(strpath + "/*/*/config.star")
180  ]
181
182  # break program/project out of path
183  configs = [
184      tuple(config_star.split("/")[0:2],) for config_star in config_stars
185  ]
186  run_configs(args, sorted(configs))
187
188
189if __name__ == "__main__":
190  main()
191