1# -*- coding: utf-8 -*- 2# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""The experiment file module. It manages the input file of crosperf.""" 7 8from __future__ import print_function 9import os.path 10import re 11from settings_factory import SettingsFactory 12 13 14class ExperimentFile(object): 15 """Class for parsing the experiment file format. 16 17 The grammar for this format is: 18 19 experiment = { _FIELD_VALUE_RE | settings } 20 settings = _OPEN_SETTINGS_RE 21 { _FIELD_VALUE_RE } 22 _CLOSE_SETTINGS_RE 23 24 Where the regexes are terminals defined below. This results in an format 25 which looks something like: 26 27 field_name: value 28 settings_type: settings_name { 29 field_name: value 30 field_name: value 31 } 32 """ 33 34 # Field regex, e.g. "iterations: 3" 35 _FIELD_VALUE_RE = re.compile(r'(\+)?\s*(\w+?)(?:\.(\S+))?\s*:\s*(.*)') 36 # Open settings regex, e.g. "label {" 37 _OPEN_SETTINGS_RE = re.compile(r'(?:([\w.-]+):)?\s*([\w.-]+)\s*{') 38 # Close settings regex. 39 _CLOSE_SETTINGS_RE = re.compile(r'}') 40 41 def __init__(self, experiment_file, overrides=None): 42 """Construct object from file-like experiment_file. 43 44 Args: 45 experiment_file: file-like object with text description of experiment. 46 overrides: A settings object that will override fields in other settings. 47 48 Raises: 49 Exception: if invalid build type or description is invalid. 50 """ 51 self.all_settings = [] 52 self.global_settings = SettingsFactory().GetSettings('global', 'global') 53 self.all_settings.append(self.global_settings) 54 55 self._Parse(experiment_file) 56 57 for settings in self.all_settings: 58 settings.Inherit() 59 settings.Validate() 60 if overrides: 61 settings.Override(overrides) 62 63 def GetSettings(self, settings_type): 64 """Return nested fields from the experiment file.""" 65 res = [] 66 for settings in self.all_settings: 67 if settings.settings_type == settings_type: 68 res.append(settings) 69 return res 70 71 def GetGlobalSettings(self): 72 """Return the global fields from the experiment file.""" 73 return self.global_settings 74 75 def _ParseField(self, reader): 76 """Parse a key/value field.""" 77 line = reader.CurrentLine().strip() 78 match = ExperimentFile._FIELD_VALUE_RE.match(line) 79 append, name, _, text_value = match.groups() 80 return (name, text_value, append) 81 82 def _ParseSettings(self, reader): 83 """Parse a settings block.""" 84 line = reader.CurrentLine().strip() 85 match = ExperimentFile._OPEN_SETTINGS_RE.match(line) 86 settings_type = match.group(1) 87 if settings_type is None: 88 settings_type = '' 89 settings_name = match.group(2) 90 settings = SettingsFactory().GetSettings(settings_name, settings_type) 91 settings.SetParentSettings(self.global_settings) 92 93 while reader.NextLine(): 94 line = reader.CurrentLine().strip() 95 96 if not line: 97 continue 98 elif ExperimentFile._FIELD_VALUE_RE.match(line): 99 field = self._ParseField(reader) 100 settings.SetField(field[0], field[1], field[2]) 101 elif ExperimentFile._CLOSE_SETTINGS_RE.match(line): 102 return settings, settings_type 103 104 raise EOFError('Unexpected EOF while parsing settings block.') 105 106 def _Parse(self, experiment_file): 107 """Parse experiment file and create settings.""" 108 reader = ExperimentFileReader(experiment_file) 109 settings_names = {} 110 try: 111 while reader.NextLine(): 112 line = reader.CurrentLine().strip() 113 114 if not line: 115 continue 116 elif ExperimentFile._OPEN_SETTINGS_RE.match(line): 117 new_settings, settings_type = self._ParseSettings(reader) 118 # We will allow benchmarks with duplicated settings name for now. 119 # Further decision will be made when parsing benchmark details in 120 # ExperimentFactory.GetExperiment(). 121 if settings_type != 'benchmark': 122 if new_settings.name in settings_names: 123 raise SyntaxError( 124 "Duplicate settings name: '%s'." % new_settings.name) 125 settings_names[new_settings.name] = True 126 self.all_settings.append(new_settings) 127 elif ExperimentFile._FIELD_VALUE_RE.match(line): 128 field = self._ParseField(reader) 129 self.global_settings.SetField(field[0], field[1], field[2]) 130 else: 131 raise IOError('Unexpected line.') 132 except Exception as err: 133 raise RuntimeError('Line %d: %s\n==> %s' % (reader.LineNo(), str(err), 134 reader.CurrentLine(False))) 135 136 def Canonicalize(self): 137 """Convert parsed experiment file back into an experiment file.""" 138 res = '' 139 board = '' 140 for field_name in self.global_settings.fields: 141 field = self.global_settings.fields[field_name] 142 if field.assigned: 143 res += '%s: %s\n' % (field.name, field.GetString()) 144 if field.name == 'board': 145 board = field.GetString() 146 res += '\n' 147 148 for settings in self.all_settings: 149 if settings.settings_type != 'global': 150 res += '%s: %s {\n' % (settings.settings_type, settings.name) 151 for field_name in settings.fields: 152 field = settings.fields[field_name] 153 if field.assigned: 154 res += '\t%s: %s\n' % (field.name, field.GetString()) 155 if field.name == 'chromeos_image': 156 real_file = ( 157 os.path.realpath(os.path.expanduser(field.GetString()))) 158 if real_file != field.GetString(): 159 res += '\t#actual_image: %s\n' % real_file 160 if field.name == 'build': 161 chromeos_root_field = settings.fields['chromeos_root'] 162 if chromeos_root_field: 163 chromeos_root = chromeos_root_field.GetString() 164 value = field.GetString() 165 autotest_field = settings.fields['autotest_path'] 166 autotest_path = '' 167 if autotest_field.assigned: 168 autotest_path = autotest_field.GetString() 169 debug_field = settings.fields['debug_path'] 170 debug_path = '' 171 if debug_field.assigned: 172 debug_path = autotest_field.GetString() 173 # Do not download the debug symbols since this function is for 174 # canonicalizing experiment file. 175 downlad_debug = False 176 image_path, autotest_path, debug_path = settings.GetXbuddyPath( 177 value, autotest_path, debug_path, board, chromeos_root, 178 'quiet', downlad_debug) 179 res += '\t#actual_image: %s\n' % image_path 180 if not autotest_field.assigned: 181 res += '\t#actual_autotest_path: %s\n' % autotest_path 182 if not debug_field.assigned: 183 res += '\t#actual_debug_path: %s\n' % debug_path 184 185 res += '}\n\n' 186 187 return res 188 189 190class ExperimentFileReader(object): 191 """Handle reading lines from an experiment file.""" 192 193 def __init__(self, file_object): 194 self.file_object = file_object 195 self.current_line = None 196 self.current_line_no = 0 197 198 def CurrentLine(self, strip_comment=True): 199 """Return the next line from the file, without advancing the iterator.""" 200 if strip_comment: 201 return self._StripComment(self.current_line) 202 return self.current_line 203 204 def NextLine(self, strip_comment=True): 205 """Advance the iterator and return the next line of the file.""" 206 self.current_line_no += 1 207 self.current_line = self.file_object.readline() 208 return self.CurrentLine(strip_comment) 209 210 def _StripComment(self, line): 211 """Strip comments starting with # from a line.""" 212 if '#' in line: 213 line = line[:line.find('#')] + line[-1] 214 return line 215 216 def LineNo(self): 217 """Return the current line number.""" 218 return self.current_line_no 219