• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 The PDFium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Looks for performance regressions on all pushes since the last run.
7
8Run this nightly to have a periodical check for performance regressions.
9
10Stores the results for each run and last checkpoint in a results directory.
11"""
12
13import argparse
14import datetime
15import json
16import os
17import sys
18
19from common import PrintWithTime
20from common import RunCommandPropagateErr
21from githelper import GitHelper
22from safetynet_conclusions import PrintConclusionsDictHumanReadable
23
24
25class JobContext(object):
26  """Context for a single run, including name and directory paths."""
27
28  def __init__(self, args):
29    self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
30    self.results_dir = args.results_dir
31    self.last_revision_covered_file = os.path.join(self.results_dir,
32                                                   'last_revision_covered')
33    self.run_output_dir = os.path.join(self.results_dir,
34                                       'profiles_%s' % self.datetime)
35    self.run_output_log_file = os.path.join(self.results_dir,
36                                            '%s.log' % self.datetime)
37
38
39class JobRun(object):
40  """A single run looking for regressions since the last one."""
41
42  def __init__(self, args, context):
43    """Constructor.
44
45    Args:
46      args: Namespace with arguments passed to the script.
47      context: JobContext for this run.
48    """
49    self.git = GitHelper()
50    self.args = args
51    self.context = context
52
53  def Run(self):
54    """Searches for regressions.
55
56    Will only write a checkpoint when first run, and on all subsequent runs
57    a comparison is done against the last checkpoint.
58
59    Returns:
60      Exit code for the script: 0 if no significant changes are found; 1 if
61      there was an error in the comparison; 3 if there was a regression; 4 if
62      there was an improvement and no regression.
63    """
64    pdfium_src_dir = os.path.join(
65        os.path.dirname(__file__),
66        os.path.pardir,
67        os.path.pardir)
68    os.chdir(pdfium_src_dir)
69
70    if not self.git.IsCurrentBranchClean() and not self.args.no_checkout:
71      PrintWithTime('Current branch is not clean, aborting')
72      return 1
73
74    branch_to_restore = self.git.GetCurrentBranchName()
75
76    if not self.args.no_checkout:
77      self.git.FetchOriginMaster()
78      self.git.Checkout('origin/master')
79
80    # Make sure results dir exists
81    if not os.path.exists(self.context.results_dir):
82      os.makedirs(self.context.results_dir)
83
84    if not os.path.exists(self.context.last_revision_covered_file):
85      result = self._InitialRun()
86    else:
87      with open(self.context.last_revision_covered_file) as f:
88        last_revision_covered = f.read().strip()
89      result = self._IncrementalRun(last_revision_covered)
90
91    self.git.Checkout(branch_to_restore)
92    return result
93
94  def _InitialRun(self):
95    """Initial run, just write a checkpoint.
96
97    Returns:
98      Exit code for the script.
99    """
100    current = self.git.GetCurrentBranchHash()
101
102    PrintWithTime('Initial run, current is %s' % current)
103
104    self._WriteCheckpoint(current)
105
106    PrintWithTime('All set up, next runs will be incremental and perform '
107                  'comparisons')
108    return 0
109
110  def _IncrementalRun(self, last_revision_covered):
111    """Incremental run, compare against last checkpoint and update it.
112
113    Args:
114      last_revision_covered: String with hash for last checkpoint.
115
116    Returns:
117      Exit code for the script.
118    """
119    current = self.git.GetCurrentBranchHash()
120
121    PrintWithTime('Incremental run, current is %s, last is %s'
122                  % (current, last_revision_covered))
123
124    if not os.path.exists(self.context.run_output_dir):
125      os.makedirs(self.context.run_output_dir)
126
127    if current == last_revision_covered:
128      PrintWithTime('No changes seen, finishing job')
129      output_info = {
130          'metadata': self._BuildRunMetadata(last_revision_covered,
131                                             current,
132                                             False)}
133      self._WriteRawJson(output_info)
134      return 0
135
136    # Run compare
137    cmd = ['testing/tools/safetynet_compare.py',
138           '--this-repo',
139           '--machine-readable',
140           '--branch-before=%s' % last_revision_covered,
141           '--output-dir=%s' % self.context.run_output_dir]
142    cmd.extend(self.args.input_paths)
143
144    json_output = RunCommandPropagateErr(cmd)
145
146    if json_output is None:
147      return 1
148
149    output_info = json.loads(json_output)
150
151    run_metadata = self._BuildRunMetadata(last_revision_covered,
152                                          current,
153                                          True)
154    output_info.setdefault('metadata', {}).update(run_metadata)
155    self._WriteRawJson(output_info)
156
157    PrintConclusionsDictHumanReadable(output_info,
158                                      colored=(not self.args.output_to_log
159                                               and not self.args.no_color),
160                                      key='after')
161
162    status = 0
163
164    if output_info['summary']['improvement']:
165      PrintWithTime('Improvement detected.')
166      status = 4
167
168    if output_info['summary']['regression']:
169      PrintWithTime('Regression detected.')
170      status = 3
171
172    if status == 0:
173      PrintWithTime('Nothing detected.')
174
175    self._WriteCheckpoint(current)
176
177    return status
178
179  def _WriteRawJson(self, output_info):
180    json_output_file = os.path.join(self.context.run_output_dir, 'raw.json')
181    with open(json_output_file, 'w') as f:
182      json.dump(output_info, f)
183
184  def _BuildRunMetadata(self, revision_before, revision_after,
185                        comparison_performed):
186    return {
187        'datetime': self.context.datetime,
188        'revision_before': revision_before,
189        'revision_after': revision_after,
190        'comparison_performed': comparison_performed,
191    }
192
193  def _WriteCheckpoint(self, checkpoint):
194    if not self.args.no_checkpoint:
195      with open(self.context.last_revision_covered_file, 'w') as f:
196        f.write(checkpoint + '\n')
197
198
199def main():
200  parser = argparse.ArgumentParser()
201  parser.add_argument('results_dir',
202                      help='where to write the job results')
203  parser.add_argument('input_paths', nargs='+',
204                      help='pdf files or directories to search for pdf files '
205                           'to run as test cases')
206  parser.add_argument('--no-checkout', action='store_true',
207                      help='whether to skip checking out origin/master. Use '
208                           'for script debugging.')
209  parser.add_argument('--no-checkpoint', action='store_true',
210                      help='whether to skip writing the new checkpoint. Use '
211                           'for script debugging.')
212  parser.add_argument('--no-color', action='store_true',
213                      help='whether to write output without color escape '
214                           'codes.')
215  parser.add_argument('--output-to-log', action='store_true',
216                      help='whether to write output to a log file')
217  args = parser.parse_args()
218
219  job_context = JobContext(args)
220
221  if args.output_to_log:
222    log_file = open(job_context.run_output_log_file, 'w')
223    sys.stdout = log_file
224    sys.stderr = log_file
225
226  run = JobRun(args, job_context)
227  result = run.Run()
228
229  if args.output_to_log:
230    log_file.close()
231
232  return result
233
234
235if __name__ == '__main__':
236  sys.exit(main())
237
238