1#!/usr/bin/env python 2# Copyright 2017 The PDFium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Looks for performance regressions on all pushes since the last run. 7 8Run this nightly to have a periodical check for performance regressions. 9 10Stores the results for each run and last checkpoint in a results directory. 11""" 12 13import argparse 14import datetime 15import json 16import os 17import sys 18 19from common import PrintWithTime 20from common import RunCommandPropagateErr 21from githelper import GitHelper 22from safetynet_conclusions import PrintConclusionsDictHumanReadable 23 24 25class JobContext(object): 26 """Context for a single run, including name and directory paths.""" 27 28 def __init__(self, args): 29 self.datetime = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') 30 self.results_dir = args.results_dir 31 self.last_revision_covered_file = os.path.join(self.results_dir, 32 'last_revision_covered') 33 self.run_output_dir = os.path.join(self.results_dir, 34 'profiles_%s' % self.datetime) 35 self.run_output_log_file = os.path.join(self.results_dir, 36 '%s.log' % self.datetime) 37 38 39class JobRun(object): 40 """A single run looking for regressions since the last one.""" 41 42 def __init__(self, args, context): 43 """Constructor. 44 45 Args: 46 args: Namespace with arguments passed to the script. 47 context: JobContext for this run. 48 """ 49 self.git = GitHelper() 50 self.args = args 51 self.context = context 52 53 def Run(self): 54 """Searches for regressions. 55 56 Will only write a checkpoint when first run, and on all subsequent runs 57 a comparison is done against the last checkpoint. 58 59 Returns: 60 Exit code for the script: 0 if no significant changes are found; 1 if 61 there was an error in the comparison; 3 if there was a regression; 4 if 62 there was an improvement and no regression. 63 """ 64 pdfium_src_dir = os.path.join( 65 os.path.dirname(__file__), 66 os.path.pardir, 67 os.path.pardir) 68 os.chdir(pdfium_src_dir) 69 70 if not self.git.IsCurrentBranchClean() and not self.args.no_checkout: 71 PrintWithTime('Current branch is not clean, aborting') 72 return 1 73 74 branch_to_restore = self.git.GetCurrentBranchName() 75 76 if not self.args.no_checkout: 77 self.git.FetchOriginMaster() 78 self.git.Checkout('origin/master') 79 80 # Make sure results dir exists 81 if not os.path.exists(self.context.results_dir): 82 os.makedirs(self.context.results_dir) 83 84 if not os.path.exists(self.context.last_revision_covered_file): 85 result = self._InitialRun() 86 else: 87 with open(self.context.last_revision_covered_file) as f: 88 last_revision_covered = f.read().strip() 89 result = self._IncrementalRun(last_revision_covered) 90 91 self.git.Checkout(branch_to_restore) 92 return result 93 94 def _InitialRun(self): 95 """Initial run, just write a checkpoint. 96 97 Returns: 98 Exit code for the script. 99 """ 100 current = self.git.GetCurrentBranchHash() 101 102 PrintWithTime('Initial run, current is %s' % current) 103 104 self._WriteCheckpoint(current) 105 106 PrintWithTime('All set up, next runs will be incremental and perform ' 107 'comparisons') 108 return 0 109 110 def _IncrementalRun(self, last_revision_covered): 111 """Incremental run, compare against last checkpoint and update it. 112 113 Args: 114 last_revision_covered: String with hash for last checkpoint. 115 116 Returns: 117 Exit code for the script. 118 """ 119 current = self.git.GetCurrentBranchHash() 120 121 PrintWithTime('Incremental run, current is %s, last is %s' 122 % (current, last_revision_covered)) 123 124 if not os.path.exists(self.context.run_output_dir): 125 os.makedirs(self.context.run_output_dir) 126 127 if current == last_revision_covered: 128 PrintWithTime('No changes seen, finishing job') 129 output_info = { 130 'metadata': self._BuildRunMetadata(last_revision_covered, 131 current, 132 False)} 133 self._WriteRawJson(output_info) 134 return 0 135 136 # Run compare 137 cmd = ['testing/tools/safetynet_compare.py', 138 '--this-repo', 139 '--machine-readable', 140 '--branch-before=%s' % last_revision_covered, 141 '--output-dir=%s' % self.context.run_output_dir] 142 cmd.extend(self.args.input_paths) 143 144 json_output = RunCommandPropagateErr(cmd) 145 146 if json_output is None: 147 return 1 148 149 output_info = json.loads(json_output) 150 151 run_metadata = self._BuildRunMetadata(last_revision_covered, 152 current, 153 True) 154 output_info.setdefault('metadata', {}).update(run_metadata) 155 self._WriteRawJson(output_info) 156 157 PrintConclusionsDictHumanReadable(output_info, 158 colored=(not self.args.output_to_log 159 and not self.args.no_color), 160 key='after') 161 162 status = 0 163 164 if output_info['summary']['improvement']: 165 PrintWithTime('Improvement detected.') 166 status = 4 167 168 if output_info['summary']['regression']: 169 PrintWithTime('Regression detected.') 170 status = 3 171 172 if status == 0: 173 PrintWithTime('Nothing detected.') 174 175 self._WriteCheckpoint(current) 176 177 return status 178 179 def _WriteRawJson(self, output_info): 180 json_output_file = os.path.join(self.context.run_output_dir, 'raw.json') 181 with open(json_output_file, 'w') as f: 182 json.dump(output_info, f) 183 184 def _BuildRunMetadata(self, revision_before, revision_after, 185 comparison_performed): 186 return { 187 'datetime': self.context.datetime, 188 'revision_before': revision_before, 189 'revision_after': revision_after, 190 'comparison_performed': comparison_performed, 191 } 192 193 def _WriteCheckpoint(self, checkpoint): 194 if not self.args.no_checkpoint: 195 with open(self.context.last_revision_covered_file, 'w') as f: 196 f.write(checkpoint + '\n') 197 198 199def main(): 200 parser = argparse.ArgumentParser() 201 parser.add_argument('results_dir', 202 help='where to write the job results') 203 parser.add_argument('input_paths', nargs='+', 204 help='pdf files or directories to search for pdf files ' 205 'to run as test cases') 206 parser.add_argument('--no-checkout', action='store_true', 207 help='whether to skip checking out origin/master. Use ' 208 'for script debugging.') 209 parser.add_argument('--no-checkpoint', action='store_true', 210 help='whether to skip writing the new checkpoint. Use ' 211 'for script debugging.') 212 parser.add_argument('--no-color', action='store_true', 213 help='whether to write output without color escape ' 214 'codes.') 215 parser.add_argument('--output-to-log', action='store_true', 216 help='whether to write output to a log file') 217 args = parser.parse_args() 218 219 job_context = JobContext(args) 220 221 if args.output_to_log: 222 log_file = open(job_context.run_output_log_file, 'w') 223 sys.stdout = log_file 224 sys.stderr = log_file 225 226 run = JobRun(args, job_context) 227 result = run.Run() 228 229 if args.output_to_log: 230 log_file.close() 231 232 return result 233 234 235if __name__ == '__main__': 236 sys.exit(main()) 237 238