1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6'''The 'grit build' tool along with integration for this tool with the 7SCons build system. 8''' 9 10import filecmp 11import getopt 12import os 13import shutil 14import sys 15 16from grit import grd_reader 17from grit import util 18from grit.tool import interface 19from grit import shortcuts 20 21 22# It would be cleaner to have each module register itself, but that would 23# require importing all of them on every run of GRIT. 24'''Map from <output> node types to modules under grit.format.''' 25_format_modules = { 26 'android': 'android_xml', 27 'c_format': 'c_format', 28 'chrome_messages_json': 'chrome_messages_json', 29 'data_package': 'data_pack', 30 'js_map_format': 'js_map_format', 31 'rc_all': 'rc', 32 'rc_translateable': 'rc', 33 'rc_nontranslateable': 'rc', 34 'rc_header': 'rc_header', 35 'resource_map_header': 'resource_map', 36 'resource_map_source': 'resource_map', 37 'resource_file_map_source': 'resource_map', 38} 39_format_modules.update((type, 'policy_templates.template_formatter') 40 for type in 'adm plist plist_strings admx adml doc json reg'.split()) 41 42 43def GetFormatter(type): 44 modulename = 'grit.format.' + _format_modules[type] 45 __import__(modulename) 46 module = sys.modules[modulename] 47 try: 48 return module.Format 49 except AttributeError: 50 return module.GetFormatter(type) 51 52 53class RcBuilder(interface.Tool): 54 '''A tool that builds RC files and resource header files for compilation. 55 56Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]* 57 58All output options for this tool are specified in the input file (see 59'grit help' for details on how to specify the input file - it is a global 60option). 61 62Options: 63 64 -o OUTPUTDIR Specify what directory output paths are relative to. 65 Defaults to the current directory. 66 67 -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional 68 value VAL (defaults to 1) which will be used to control 69 conditional inclusion of resources. 70 71 -E NAME=VALUE Set environment variable NAME to VALUE (within grit). 72 73 -f FIRSTIDSFILE Path to a python file that specifies the first id of 74 value to use for resources. A non-empty value here will 75 override the value specified in the <grit> node's 76 first_ids_file. 77 78 -w WHITELISTFILE Path to a file containing the string names of the 79 resources to include. Anything not listed is dropped. 80 81 -t PLATFORM Specifies the platform the build is targeting; defaults 82 to the value of sys.platform. The value provided via this 83 flag should match what sys.platform would report for your 84 target platform; see grit.node.base.EvaluateCondition. 85 86Conditional inclusion of resources only affects the output of files which 87control which resources get linked into a binary, e.g. it affects .rc files 88meant for compilation but it does not affect resource header files (that define 89IDs). This helps ensure that values of IDs stay the same, that all messages 90are exported to translation interchange files (e.g. XMB files), etc. 91''' 92 93 def ShortDescription(self): 94 return 'A tool that builds RC files for compilation.' 95 96 def Run(self, opts, args): 97 self.output_directory = '.' 98 first_ids_file = None 99 whitelist_filenames = [] 100 target_platform = None 101 dep_dir = None 102 (own_opts, args) = getopt.getopt(args, 'o:D:E:f:w:t:', ('dep-dir=',)) 103 for (key, val) in own_opts: 104 if key == '-o': 105 self.output_directory = val 106 elif key == '-D': 107 name, val = util.ParseDefine(val) 108 self.defines[name] = val 109 elif key == '-E': 110 (env_name, env_value) = val.split('=', 1) 111 os.environ[env_name] = env_value 112 elif key == '-f': 113 # TODO(joi@chromium.org): Remove this override once change 114 # lands in WebKit.grd to specify the first_ids_file in the 115 # .grd itself. 116 first_ids_file = val 117 elif key == '-w': 118 whitelist_filenames.append(val) 119 elif key == '-t': 120 target_platform = val 121 elif key == '--dep-dir': 122 dep_dir = val 123 124 if len(args): 125 print 'This tool takes no tool-specific arguments.' 126 return 2 127 self.SetOptions(opts) 128 if self.scons_targets: 129 self.VerboseOut('Using SCons targets to identify files to output.\n') 130 else: 131 self.VerboseOut('Output directory: %s (absolute path: %s)\n' % 132 (self.output_directory, 133 os.path.abspath(self.output_directory))) 134 135 if whitelist_filenames: 136 self.whitelist_names = set() 137 for whitelist_filename in whitelist_filenames: 138 self.VerboseOut('Using whitelist: %s\n' % whitelist_filename); 139 whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT) 140 self.whitelist_names.update(whitelist_contents.strip().split('\n')) 141 142 self.res = grd_reader.Parse(opts.input, 143 debug=opts.extra_verbose, 144 first_ids_file=first_ids_file, 145 defines=self.defines, 146 target_platform=target_platform) 147 # Set an output context so that conditionals can use defines during the 148 # gathering stage; we use a dummy language here since we are not outputting 149 # a specific language. 150 self.res.SetOutputLanguage('en') 151 self.res.RunGatherers() 152 self.Process() 153 154 if dep_dir: 155 self.GenerateDepfile(opts.input, dep_dir) 156 157 return 0 158 159 def __init__(self, defines=None): 160 # Default file-creation function is built-in open(). Only done to allow 161 # overriding by unit test. 162 self.fo_create = open 163 164 # key/value pairs of C-preprocessor like defines that are used for 165 # conditional output of resources 166 self.defines = defines or {} 167 168 # self.res is a fully-populated resource tree if Run() 169 # has been called, otherwise None. 170 self.res = None 171 172 # Set to a list of filenames for the output nodes that are relative 173 # to the current working directory. They are in the same order as the 174 # output nodes in the file. 175 self.scons_targets = None 176 177 # The set of names that are whitelisted to actually be included in the 178 # output. 179 self.whitelist_names = None 180 181 182 @staticmethod 183 def AddWhitelistTags(start_node, whitelist_names): 184 # Walk the tree of nodes added attributes for the nodes that shouldn't 185 # be written into the target files (skip markers). 186 from grit.node import include 187 from grit.node import message 188 for node in start_node: 189 # Same trick data_pack.py uses to see what nodes actually result in 190 # real items. 191 if (isinstance(node, include.IncludeNode) or 192 isinstance(node, message.MessageNode)): 193 text_ids = node.GetTextualIds() 194 # Mark the item to be skipped if it wasn't in the whitelist. 195 if text_ids and text_ids[0] not in whitelist_names: 196 node.SetWhitelistMarkedAsSkip(True) 197 198 @staticmethod 199 def ProcessNode(node, output_node, outfile): 200 '''Processes a node in-order, calling its formatter before and after 201 recursing to its children. 202 203 Args: 204 node: grit.node.base.Node subclass 205 output_node: grit.node.io.OutputNode 206 outfile: open filehandle 207 ''' 208 base_dir = util.dirname(output_node.GetOutputFilename()) 209 210 formatter = GetFormatter(output_node.GetType()) 211 formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir) 212 outfile.writelines(formatted) 213 214 215 def Process(self): 216 # Update filenames with those provided by SCons if we're being invoked 217 # from SCons. The list of SCons targets also includes all <structure> 218 # node outputs, but it starts with our output files, in the order they 219 # occur in the .grd 220 if self.scons_targets: 221 assert len(self.scons_targets) >= len(self.res.GetOutputFiles()) 222 outfiles = self.res.GetOutputFiles() 223 for ix in range(len(outfiles)): 224 outfiles[ix].output_filename = os.path.abspath( 225 self.scons_targets[ix]) 226 else: 227 for output in self.res.GetOutputFiles(): 228 output.output_filename = os.path.abspath(os.path.join( 229 self.output_directory, output.GetFilename())) 230 231 # If there are whitelisted names, tag the tree once up front, this way 232 # while looping through the actual output, it is just an attribute check. 233 if self.whitelist_names: 234 self.AddWhitelistTags(self.res, self.whitelist_names) 235 236 for output in self.res.GetOutputFiles(): 237 self.VerboseOut('Creating %s...' % output.GetFilename()) 238 239 # Microsoft's RC compiler can only deal with single-byte or double-byte 240 # files (no UTF-8), so we make all RC files UTF-16 to support all 241 # character sets. 242 if output.GetType() in ('rc_header', 'resource_map_header', 243 'resource_map_source', 'resource_file_map_source'): 244 encoding = 'cp1252' 245 elif output.GetType() in ('android', 'c_format', 'js_map_format', 'plist', 246 'plist_strings', 'doc', 'json'): 247 encoding = 'utf_8' 248 elif output.GetType() in ('chrome_messages_json'): 249 # Chrome Web Store currently expects BOM for UTF-8 files :-( 250 encoding = 'utf-8-sig' 251 else: 252 # TODO(gfeher) modify here to set utf-8 encoding for admx/adml 253 encoding = 'utf_16' 254 255 # Set the context, for conditional inclusion of resources 256 self.res.SetOutputLanguage(output.GetLanguage()) 257 self.res.SetOutputContext(output.GetContext()) 258 self.res.SetDefines(self.defines) 259 260 # Make the output directory if it doesn't exist. 261 self.MakeDirectoriesTo(output.GetOutputFilename()) 262 263 # Write the results to a temporary file and only overwrite the original 264 # if the file changed. This avoids unnecessary rebuilds. 265 outfile = self.fo_create(output.GetOutputFilename() + '.tmp', 'wb') 266 267 if output.GetType() != 'data_package': 268 outfile = util.WrapOutputStream(outfile, encoding) 269 270 # Iterate in-order through entire resource tree, calling formatters on 271 # the entry into a node and on exit out of it. 272 with outfile: 273 self.ProcessNode(self.res, output, outfile) 274 275 # Now copy from the temp file back to the real output, but on Windows, 276 # only if the real output doesn't exist or the contents of the file 277 # changed. This prevents identical headers from being written and .cc 278 # files from recompiling (which is painful on Windows). 279 if not os.path.exists(output.GetOutputFilename()): 280 os.rename(output.GetOutputFilename() + '.tmp', 281 output.GetOutputFilename()) 282 else: 283 # CHROMIUM SPECIFIC CHANGE. 284 # This clashes with gyp + vstudio, which expect the output timestamp 285 # to change on a rebuild, even if nothing has changed. 286 #files_match = filecmp.cmp(output.GetOutputFilename(), 287 # output.GetOutputFilename() + '.tmp') 288 #if (output.GetType() != 'rc_header' or not files_match 289 # or sys.platform != 'win32'): 290 shutil.copy2(output.GetOutputFilename() + '.tmp', 291 output.GetOutputFilename()) 292 os.remove(output.GetOutputFilename() + '.tmp') 293 294 self.VerboseOut(' done.\n') 295 296 # Print warnings if there are any duplicate shortcuts. 297 warnings = shortcuts.GenerateDuplicateShortcutsWarnings( 298 self.res.UberClique(), self.res.GetTcProject()) 299 if warnings: 300 print '\n'.join(warnings) 301 302 # Print out any fallback warnings, and missing translation errors, and 303 # exit with an error code if there are missing translations in a non-pseudo 304 # and non-official build. 305 warnings = (self.res.UberClique().MissingTranslationsReport(). 306 encode('ascii', 'replace')) 307 if warnings: 308 self.VerboseOut(warnings) 309 if self.res.UberClique().HasMissingTranslations(): 310 print self.res.UberClique().missing_translations_ 311 sys.exit(-1) 312 313 def GenerateDepfile(self, input_filename, dep_dir): 314 '''Generate a depfile that contains the imlicit dependencies of the input 315 grd. The depfile will be in the same format as a makefile, and will contain 316 references to files relative to |dep_dir|. It will be put in the same 317 directory as the generated outputs. 318 319 For example, supposing we have three files in a directory src/ 320 321 src/ 322 blah.grd <- depends on input{1,2}.xtb 323 input1.xtb 324 input2.xtb 325 326 and we run 327 328 grit -i blah.grd -o ../out/gen --dep-dir ../out 329 330 from the directory src/ we will generate a depfile ../out/gen/blah.grd.d 331 that has the contents 332 333 gen/blah.grd.d: ../src/input1.xtb ../src/input2.xtb 334 335 Note that all paths in the depfile are relative to ../out, the dep-dir. 336 ''' 337 depsfile_basename = os.path.basename(input_filename + '.d') 338 depfile = os.path.abspath( 339 os.path.join(self.output_directory, depsfile_basename)) 340 dep_dir = os.path.abspath(dep_dir) 341 # The path prefix to prepend to dependencies in the depfile. 342 prefix = os.path.relpath(os.getcwd(), dep_dir) 343 # The path that the depfile refers to itself by. 344 self_ref_depfile = os.path.relpath(depfile, dep_dir) 345 infiles = self.res.GetInputFiles() 346 deps_text = ' '.join([os.path.join(prefix, i) for i in infiles]) 347 depfile_contents = self_ref_depfile + ': ' + deps_text 348 self.MakeDirectoriesTo(depfile) 349 outfile = self.fo_create(depfile, 'wb') 350 outfile.writelines(depfile_contents) 351 352 @staticmethod 353 def MakeDirectoriesTo(file): 354 '''Creates directories necessary to contain |file|.''' 355 dir = os.path.split(file)[0] 356 if not os.path.exists(dir): 357 os.makedirs(dir) 358