• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python3
2# Copyright 2017 The PDFium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""Looks for performance regressions on all pushes since the last run.
6
7Run this nightly to have a periodical check for performance regressions.
8
9Stores the results for each run and last checkpoint in a results directory.
10"""
11
12import argparse
13import datetime
14import json
15import os
16import sys
17
18from common import PrintWithTime
19from common import RunCommandPropagateErr
20from githelper import GitHelper
21from safetynet_conclusions import PrintConclusionsDictHumanReadable
22
23
24class JobContext:
25  """Context for a single run, including name and directory paths."""
26
27  def __init__(self, args):
28    self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
29    self.results_dir = args.results_dir
30    self.last_revision_covered_file = os.path.join(self.results_dir,
31                                                   'last_revision_covered')
32    self.run_output_dir = os.path.join(self.results_dir,
33                                       'profiles_%s' % self.datetime)
34    self.run_output_log_file = os.path.join(self.results_dir,
35                                            '%s.log' % self.datetime)
36
37
38class JobRun:
39  """A single run looking for regressions since the last one."""
40
41  def __init__(self, args, context):
42    """Constructor.
43
44    Args:
45      args: Namespace with arguments passed to the script.
46      context: JobContext for this run.
47    """
48    self.git = GitHelper()
49    self.args = args
50    self.context = context
51
52  def Run(self):
53    """Searches for regressions.
54
55    Will only write a checkpoint when first run, and on all subsequent runs
56    a comparison is done against the last checkpoint.
57
58    Returns:
59      Exit code for the script: 0 if no significant changes are found; 1 if
60      there was an error in the comparison; 3 if there was a regression; 4 if
61      there was an improvement and no regression.
62    """
63    pdfium_src_dir = os.path.join(
64        os.path.dirname(__file__), os.path.pardir, os.path.pardir)
65    os.chdir(pdfium_src_dir)
66
67    branch_to_restore = self.git.GetCurrentBranchName()
68
69    if not self.args.no_checkout:
70      self.git.FetchOriginMaster()
71      self.git.Checkout('origin/main')
72
73    # Make sure results dir exists
74    if not os.path.exists(self.context.results_dir):
75      os.makedirs(self.context.results_dir)
76
77    if not os.path.exists(self.context.last_revision_covered_file):
78      result = self._InitialRun()
79    else:
80      with open(self.context.last_revision_covered_file) as f:
81        last_revision_covered = f.read().strip()
82      result = self._IncrementalRun(last_revision_covered)
83
84    self.git.Checkout(branch_to_restore)
85    return result
86
87  def _InitialRun(self):
88    """Initial run, just write a checkpoint.
89
90    Returns:
91      Exit code for the script.
92    """
93    current = self.git.GetCurrentBranchHash()
94
95    PrintWithTime('Initial run, current is %s' % current)
96
97    self._WriteCheckpoint(current)
98
99    PrintWithTime('All set up, next runs will be incremental and perform '
100                  'comparisons')
101    return 0
102
103  def _IncrementalRun(self, last_revision_covered):
104    """Incremental run, compare against last checkpoint and update it.
105
106    Args:
107      last_revision_covered: String with hash for last checkpoint.
108
109    Returns:
110      Exit code for the script.
111    """
112    current = self.git.GetCurrentBranchHash()
113
114    PrintWithTime('Incremental run, current is %s, last is %s' %
115                  (current, last_revision_covered))
116
117    if not os.path.exists(self.context.run_output_dir):
118      os.makedirs(self.context.run_output_dir)
119
120    if current == last_revision_covered:
121      PrintWithTime('No changes seen, finishing job')
122      output_info = {
123          'metadata':
124              self._BuildRunMetadata(last_revision_covered, current, False)
125      }
126      self._WriteRawJson(output_info)
127      return 0
128
129    # Run compare
130    cmd = [
131        'testing/tools/safetynet_compare.py', '--this-repo',
132        '--machine-readable',
133        '--branch-before=%s' % last_revision_covered,
134        '--output-dir=%s' % self.context.run_output_dir
135    ]
136    cmd.extend(self.args.input_paths)
137
138    json_output = RunCommandPropagateErr(cmd)
139
140    if json_output is None:
141      return 1
142
143    output_info = json.loads(json_output)
144
145    run_metadata = self._BuildRunMetadata(last_revision_covered, current, True)
146    output_info.setdefault('metadata', {}).update(run_metadata)
147    self._WriteRawJson(output_info)
148
149    PrintConclusionsDictHumanReadable(
150        output_info,
151        colored=(not self.args.output_to_log and not self.args.no_color),
152        key='after')
153
154    status = 0
155
156    if output_info['summary']['improvement']:
157      PrintWithTime('Improvement detected.')
158      status = 4
159
160    if output_info['summary']['regression']:
161      PrintWithTime('Regression detected.')
162      status = 3
163
164    if status == 0:
165      PrintWithTime('Nothing detected.')
166
167    self._WriteCheckpoint(current)
168
169    return status
170
171  def _WriteRawJson(self, output_info):
172    json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
173    with open(json_output_file, 'w') as f:
174      json.dump(output_info, f)
175
176  def _BuildRunMetadata(self, revision_before, revision_after,
177                        comparison_performed):
178    return {
179        'datetime': self.context.datetime,
180        'revision_before': revision_before,
181        'revision_after': revision_after,
182        'comparison_performed': comparison_performed,
183    }
184
185  def _WriteCheckpoint(self, checkpoint):
186    if not self.args.no_checkpoint:
187      with open(self.context.last_revision_covered_file, 'w') as f:
188        f.write(checkpoint + '\n')
189
190
191def main():
192  parser = argparse.ArgumentParser()
193  parser.add_argument('results_dir', help='where to write the job results')
194  parser.add_argument(
195      'input_paths',
196      nargs='+',
197      help='pdf files or directories to search for pdf files '
198      'to run as test cases')
199  parser.add_argument(
200      '--no-checkout',
201      action='store_true',
202      help='whether to skip checking out origin/main. Use '
203      'for script debugging.')
204  parser.add_argument(
205      '--no-checkpoint',
206      action='store_true',
207      help='whether to skip writing the new checkpoint. Use '
208      'for script debugging.')
209  parser.add_argument(
210      '--no-color',
211      action='store_true',
212      help='whether to write output without color escape '
213      'codes.')
214  parser.add_argument(
215      '--output-to-log',
216      action='store_true',
217      help='whether to write output to a log file')
218  args = parser.parse_args()
219
220  job_context = JobContext(args)
221
222  if args.output_to_log:
223    log_file = open(job_context.run_output_log_file, 'w')
224    sys.stdout = log_file
225    sys.stderr = log_file
226
227  run = JobRun(args, job_context)
228  result = run.Run()
229
230  if args.output_to_log:
231    log_file.close()
232
233  return result
234
235
236if __name__ == '__main__':
237  sys.exit(main())
238