• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2021 The ChromiumOS Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Run an equivalent to the backfill pipeline locally and generate diffs.
6
7Parse the actual current builder configurations from BuildBucket and run
8the join_config_payloads.py script locally.  Generate a diff that shows any
9changes using the tip-of-tree code vs what's running in production.
10"""
11
12import argparse
13import collections
14import functools
15import itertools
16import json
17import logging
18import multiprocessing
19import multiprocessing.pool
20import os
21import pathlib
22import shutil
23import subprocess
24import sys
25import tempfile
26import time
27
28from common import utilities
29
30# resolve relative directories
31this_dir = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
32hwid_path = (this_dir / "../../platform/chromeos-hwid/v3").resolve()
33join_script = (this_dir / "../payload_utils/join_config_payloads.py").resolve()
34merge_script = (this_dir / "../payload_utils/aggregate_messages.py").resolve()
35public_path = (this_dir / "../../overlays").resolve()
36private_path = (this_dir / "../../private-overlays").resolve()
37project_path = (this_dir / "../../project").resolve()
38
39# record to store backfiller configuration in
40BackfillConfig = collections.namedtuple('BackfillConfig', [
41    'program',
42    'project',
43    'hwid_key',
44    'public_model',
45    'private_repo',
46    'private_model',
47])
48
49
50def parse_build_property(build, name):
51  """Parse out a property value from a build and return its value.
52
53  Properties are always JSON values, so we decode them and return the
54  resulting object
55
56  Args:
57    build (dict): json object containing BuildBucket properties
58    name (str): name of the property to look up
59
60  Return:
61    decoded property value or None if not found
62  """
63  return json.loads(build["config"]["properties"]).get(name)
64
65
66def run_backfill(config,
67                 args,
68                 logname=None,
69                 run_imported=True,
70                 run_joined=True):
71  """Run a single backfill job, return diff of current and new output.
72
73  Args:
74    config: BackfillConfig instance for the backfill operation.
75    args: Commandline arguments
76    logname: Filename to redirect stderr to from backfill
77      default is to suppress the output
78    run_imported: If True, generate a diff for the imported payload
79    run_joined: If True, generate a diff for the joined payload
80  """
81
82  def run_diff(cmd, current, output):
83    """Execute cmd and diff the current and output files"""
84    logfile.write("running: {}\n".format(" ".join(map(str, cmd))))
85
86    subprocess.run(cmd, stderr=logfile, check=True)
87
88    # if one or the other file doesn't exist, return the other as a diff
89    if current.exists() != output.exists():
90      if current.exists():
91        return open(current).read()
92      return open(output).read()
93
94    # otherwise run diff
95    return utilities.jqdiff(current, output)
96
97  #### start of function body
98
99  # path to project repo and config bundle
100  path_repo = project_path / config.program / config.project
101  path_config = path_repo / "generated/config.jsonproto"
102
103  logfile = subprocess.DEVNULL
104  if logname:
105    logfile = open(logname, "a")
106
107  # reef/fizz are currently broken because it _needs_ a real portage environment
108  # to pull in common code.
109  # TODO(https://crbug.com/1144956): fix when reef is corrected
110  if config.program in ["reef", "fizz"]:
111    return None
112
113  cmd = [join_script, "--l", "DEBUG"]
114  cmd.extend(["--program-name", config.program])
115  cmd.extend(["--project-name", config.project])
116
117  if path_config.exists():
118    cmd.extend(["--config-bundle", path_config])
119
120  if config.hwid_key:
121    cmd.extend(["--hwid", hwid_path / config.hwid_key])
122
123  if config.public_model:
124    cmd.extend(["--public-model", public_path / config.public_model])
125
126  if config.private_model:
127    overlay = config.private_repo.split('/')[-1]
128    cmd.extend(
129        ["--private-model", private_path / overlay / config.private_model])
130
131  # create output directory if it doesn't exist
132  if args.save_imported_payloads:
133    os.makedirs(
134        os.path.join(args.save_imported_payloads, config.project),
135        exist_ok=True,
136    )
137
138  if args.save_joined_payloads:
139    os.makedirs(
140        os.path.join(args.save_joined_payloads, config.project),
141        exist_ok=True,
142    )
143
144  # create temporary directory for output
145  diff_imported = ""
146  diff_joined = ""
147  with tempfile.TemporaryDirectory() as scratch:
148    scratch = pathlib.Path(scratch)
149
150    old_imported_prefix = path_repo / "generated"
151    if args.diff_imported_against:
152      old_imported_prefix = pathlib.Path(
153          os.path.join(
154              args.diff_imported_against,
155              config.project,
156          ))
157
158    # generate diff of imported payloads
159    path_imported_old = old_imported_prefix / "imported.jsonproto"
160    path_imported_new = scratch / "imported.jsonproto"
161
162    if run_imported:
163      diff_imported = run_diff(
164          cmd + ["--import-only", "--output", path_imported_new],
165          path_imported_old,
166          path_imported_new,
167      )
168
169      if args.save_imported_payloads:
170        shutil.copyfile(
171            path_imported_new,
172            os.path.join(
173                args.save_imported_payloads,
174                config.project,
175                "imported.jsonproto",
176            ),
177        )
178
179    old_joined_prefix = path_repo / "generated"
180    if args.diff_joined_against:
181      old_joined_prefix = pathlib.Path(
182          os.path.join(
183              args.diff_joined_against,
184              config.project,
185          ))
186
187    # generate diff of joined payloads
188    if run_joined and path_config.exists():
189      path_joined_old = old_joined_prefix / "joined.jsonproto"
190      path_joined_new = scratch / "joined.jsonproto"
191
192      diff_joined = run_diff(cmd + ["--output", path_joined_new],
193                             path_joined_old, path_joined_new)
194
195      if args.save_joined_payloads:
196        shutil.copyfile(
197            path_joined_new,
198            os.path.join(
199                args.save_joined_payloads,
200                config.project,
201                "joined.jsonproto",
202            ),
203        )
204
205  return ("{}-{}".format(config.program,
206                         config.project), diff_imported, diff_joined)
207
208
209def run_backfills(args, configs):
210  """Run backfill pipeline for each builder in configs.
211
212  Generate an über diff showing the changes that the current ToT
213  join_config_payloads code would generate vs what's currently committed.
214
215  Write the result to the output file specified on the command line.
216
217  Args:
218    args: command line arguments from argparse
219    configs: list of BackfillConfig instances to execute
220
221  Return:
222    nothing
223  """
224
225  # create a logfile if requested
226  kwargs = {}
227  kwargs["run_joined"] = args.joined_diff is not None
228  kwargs["args"] = args
229  if args.logfile:
230    # open and close the logfile to truncate it so backfills can append
231    # We can't pickle the file object and send it as an argument with
232    # multiprocessing, so this is a workaround for that limitation
233    with open(args.logfile, "w"):
234      kwargs["logname"] = args.logfile
235
236  nproc = 32
237  nconfig = len(configs)
238  imported_diffs = {}
239  joined_diffs = {}
240  with multiprocessing.Pool(processes=nproc) as pool:
241    results = pool.imap_unordered(
242        functools.partial(run_backfill, **kwargs), configs, chunksize=1)
243    for ii, result in enumerate(results, 1):
244      sys.stderr.write(
245          utilities.clear_line("[{}/{}] Processing backfills".format(
246              ii, nconfig)))
247
248      if result:
249        key, imported, joined = result
250        imported_diffs[key] = imported
251        joined_diffs[key] = joined
252
253    sys.stderr.write(utilities.clear_line("Processing backfills"))
254
255  # generate final über diff showing all the changes
256  with open(args.imported_diff, "w") as ofile:
257    for name, result in sorted(imported_diffs.items()):
258      ofile.write("## ---------------------\n")
259      ofile.write("## diff for {}\n".format(name))
260      ofile.write("\n")
261      ofile.write(result + "\n")
262
263  if args.joined_diff:
264    with open(args.joined_diff, "w") as ofile:
265      for name, result in sorted(joined_diffs.items()):
266        ofile.write("## ---------------------\n")
267        ofile.write("## diff for {}\n".format(name))
268        ofile.write("\n")
269        ofile.write(result + "\n")
270
271
272def main():
273  parser = argparse.ArgumentParser(
274      description=__doc__,
275      formatter_class=argparse.RawTextHelpFormatter,
276  )
277
278  parser.add_argument(
279      "--imported-diff",
280      type=str,
281      required=True,
282      help="target file for diff on imported.jsonproto payload",
283  )
284
285  parser.add_argument(
286      "--save-imported-payloads",
287      type=str,
288      help="target directory to save individual imported.jsonproto payloads",
289  )
290
291  parser.add_argument(
292      "--diff-imported-against",
293      type=str,
294      help="source directory of individual imported.jsonproto payloads",
295  )
296
297  parser.add_argument(
298      "--joined-diff",
299      type=str,
300      help="target file for diff on joined.jsonproto payload",
301  )
302
303  parser.add_argument(
304      "--save-joined-payloads",
305      type=str,
306      help="target directory to save individual joined.jsonproto payloads",
307  )
308
309  parser.add_argument(
310      "--diff-joined-against",
311      type=str,
312      help="source directory of individual joined.jsonproto payloads",
313  )
314
315  parser.add_argument(
316      "-l",
317      "--logfile",
318      type=str,
319      help="target file to log output from backfills",
320  )
321  args = parser.parse_args()
322
323  # query BuildBucket for current builder configurations in the infra bucket
324  data, status = utilities.call_and_spin(
325      "Listing backfill builder",
326      json.dumps({
327          "id": {
328              "project": "chromeos",
329              "bucket": "infra",
330              "builder": "backfiller"
331          }
332      }),
333      "prpc",
334      "call",
335      "cr-buildbucket.appspot.com",
336      "buildbucket.v2.Builders.GetBuilder",
337  )
338
339  if status != 0:
340    print(
341        "Error executing prpc call to list builders.  Try 'prpc login' first.",
342        file=sys.stderr,
343    )
344    sys.exit(status)
345
346  builder = json.loads(data)
347
348  # construct backfill config from the configured builder properties
349  configs = []
350  for builder_config in parse_build_property(builder, "configs"):
351    config = BackfillConfig(
352        program=builder_config["program_name"],
353        project=builder_config["project_name"],
354        hwid_key=builder_config.get("hwid_key"),
355        public_model=builder_config.get("public_yaml_path"),
356        private_repo=builder_config.get("private_yaml", {}).get("repo"),
357        private_model=builder_config.get("private_yaml", {}).get("path"),
358    )
359
360    path_repo = project_path / config.program / config.project
361    if not path_repo.exists():
362      logging.warning("{}/{} does not exist locally, skipping".format(
363          config.program, config.project))
364      continue
365
366    configs.append(config)
367
368  run_backfills(args, configs)
369
370
371if __name__ == "__main__":
372  main()
373