1#!/usr/bin/env python 2# Copyright (c) 2012 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6'''The 'grit build' tool along with integration for this tool with the 7SCons build system. 8''' 9 10import filecmp 11import getopt 12import os 13import shutil 14import sys 15 16from grit import grd_reader 17from grit import util 18from grit.tool import interface 19from grit import shortcuts 20 21 22# It would be cleaner to have each module register itself, but that would 23# require importing all of them on every run of GRIT. 24'''Map from <output> node types to modules under grit.format.''' 25_format_modules = { 26 'android': 'android_xml', 27 'c_format': 'c_format', 28 'chrome_messages_json': 'chrome_messages_json', 29 'data_package': 'data_pack', 30 'js_map_format': 'js_map_format', 31 'rc_all': 'rc', 32 'rc_translateable': 'rc', 33 'rc_nontranslateable': 'rc', 34 'rc_header': 'rc_header', 35 'resource_map_header': 'resource_map', 36 'resource_map_source': 'resource_map', 37 'resource_file_map_source': 'resource_map', 38} 39_format_modules.update( 40 (type, 'policy_templates.template_formatter') for type in 41 [ 'adm', 'admx', 'adml', 'reg', 'doc', 'json', 42 'plist', 'plist_strings', 'ios_plist' ]) 43 44 45def GetFormatter(type): 46 modulename = 'grit.format.' + _format_modules[type] 47 __import__(modulename) 48 module = sys.modules[modulename] 49 try: 50 return module.Format 51 except AttributeError: 52 return module.GetFormatter(type) 53 54 55class RcBuilder(interface.Tool): 56 '''A tool that builds RC files and resource header files for compilation. 57 58Usage: grit build [-o OUTPUTDIR] [-D NAME[=VAL]]* 59 60All output options for this tool are specified in the input file (see 61'grit help' for details on how to specify the input file - it is a global 62option). 63 64Options: 65 66 -o OUTPUTDIR Specify what directory output paths are relative to. 67 Defaults to the current directory. 68 69 -D NAME[=VAL] Specify a C-preprocessor-like define NAME with optional 70 value VAL (defaults to 1) which will be used to control 71 conditional inclusion of resources. 72 73 -E NAME=VALUE Set environment variable NAME to VALUE (within grit). 74 75 -f FIRSTIDSFILE Path to a python file that specifies the first id of 76 value to use for resources. A non-empty value here will 77 override the value specified in the <grit> node's 78 first_ids_file. 79 80 -w WHITELISTFILE Path to a file containing the string names of the 81 resources to include. Anything not listed is dropped. 82 83 -t PLATFORM Specifies the platform the build is targeting; defaults 84 to the value of sys.platform. The value provided via this 85 flag should match what sys.platform would report for your 86 target platform; see grit.node.base.EvaluateCondition. 87 88 -h HEADERFORMAT Custom format string to use for generating rc header files. 89 The string should have two placeholders: {textual_id} 90 and {numeric_id}. E.g. "#define {textual_id} {numeric_id}" 91 Otherwise it will use the default "#define SYMBOL 1234" 92 93Conditional inclusion of resources only affects the output of files which 94control which resources get linked into a binary, e.g. it affects .rc files 95meant for compilation but it does not affect resource header files (that define 96IDs). This helps ensure that values of IDs stay the same, that all messages 97are exported to translation interchange files (e.g. XMB files), etc. 98''' 99 100 def ShortDescription(self): 101 return 'A tool that builds RC files for compilation.' 102 103 def Run(self, opts, args): 104 self.output_directory = '.' 105 first_ids_file = None 106 whitelist_filenames = [] 107 target_platform = None 108 depfile = None 109 depdir = None 110 rc_header_format = None 111 (own_opts, args) = getopt.getopt(args, 'o:D:E:f:w:t:h:', ('depdir=','depfile=')) 112 for (key, val) in own_opts: 113 if key == '-o': 114 self.output_directory = val 115 elif key == '-D': 116 name, val = util.ParseDefine(val) 117 self.defines[name] = val 118 elif key == '-E': 119 (env_name, env_value) = val.split('=', 1) 120 os.environ[env_name] = env_value 121 elif key == '-f': 122 # TODO(joi@chromium.org): Remove this override once change 123 # lands in WebKit.grd to specify the first_ids_file in the 124 # .grd itself. 125 first_ids_file = val 126 elif key == '-w': 127 whitelist_filenames.append(val) 128 elif key == '-t': 129 target_platform = val 130 elif key == '-h': 131 rc_header_format = val 132 elif key == '--depdir': 133 depdir = val 134 elif key == '--depfile': 135 depfile = val 136 137 if len(args): 138 print 'This tool takes no tool-specific arguments.' 139 return 2 140 self.SetOptions(opts) 141 if self.scons_targets: 142 self.VerboseOut('Using SCons targets to identify files to output.\n') 143 else: 144 self.VerboseOut('Output directory: %s (absolute path: %s)\n' % 145 (self.output_directory, 146 os.path.abspath(self.output_directory))) 147 148 if whitelist_filenames: 149 self.whitelist_names = set() 150 for whitelist_filename in whitelist_filenames: 151 self.VerboseOut('Using whitelist: %s\n' % whitelist_filename); 152 whitelist_contents = util.ReadFile(whitelist_filename, util.RAW_TEXT) 153 self.whitelist_names.update(whitelist_contents.strip().split('\n')) 154 155 self.res = grd_reader.Parse(opts.input, 156 debug=opts.extra_verbose, 157 first_ids_file=first_ids_file, 158 defines=self.defines, 159 target_platform=target_platform) 160 # Set an output context so that conditionals can use defines during the 161 # gathering stage; we use a dummy language here since we are not outputting 162 # a specific language. 163 self.res.SetOutputLanguage('en') 164 if rc_header_format: 165 self.res.AssignRcHeaderFormat(rc_header_format) 166 self.res.RunGatherers() 167 self.Process() 168 169 if depfile and depdir: 170 self.GenerateDepfile(opts.input, depfile, depdir) 171 172 return 0 173 174 def __init__(self, defines=None): 175 # Default file-creation function is built-in open(). Only done to allow 176 # overriding by unit test. 177 self.fo_create = open 178 179 # key/value pairs of C-preprocessor like defines that are used for 180 # conditional output of resources 181 self.defines = defines or {} 182 183 # self.res is a fully-populated resource tree if Run() 184 # has been called, otherwise None. 185 self.res = None 186 187 # Set to a list of filenames for the output nodes that are relative 188 # to the current working directory. They are in the same order as the 189 # output nodes in the file. 190 self.scons_targets = None 191 192 # The set of names that are whitelisted to actually be included in the 193 # output. 194 self.whitelist_names = None 195 196 @staticmethod 197 def AddWhitelistTags(start_node, whitelist_names): 198 # Walk the tree of nodes added attributes for the nodes that shouldn't 199 # be written into the target files (skip markers). 200 from grit.node import include 201 from grit.node import message 202 for node in start_node: 203 # Same trick data_pack.py uses to see what nodes actually result in 204 # real items. 205 if (isinstance(node, include.IncludeNode) or 206 isinstance(node, message.MessageNode)): 207 text_ids = node.GetTextualIds() 208 # Mark the item to be skipped if it wasn't in the whitelist. 209 if text_ids and text_ids[0] not in whitelist_names: 210 node.SetWhitelistMarkedAsSkip(True) 211 212 @staticmethod 213 def ProcessNode(node, output_node, outfile): 214 '''Processes a node in-order, calling its formatter before and after 215 recursing to its children. 216 217 Args: 218 node: grit.node.base.Node subclass 219 output_node: grit.node.io.OutputNode 220 outfile: open filehandle 221 ''' 222 base_dir = util.dirname(output_node.GetOutputFilename()) 223 224 formatter = GetFormatter(output_node.GetType()) 225 formatted = formatter(node, output_node.GetLanguage(), output_dir=base_dir) 226 outfile.writelines(formatted) 227 228 229 def Process(self): 230 # Update filenames with those provided by SCons if we're being invoked 231 # from SCons. The list of SCons targets also includes all <structure> 232 # node outputs, but it starts with our output files, in the order they 233 # occur in the .grd 234 if self.scons_targets: 235 assert len(self.scons_targets) >= len(self.res.GetOutputFiles()) 236 outfiles = self.res.GetOutputFiles() 237 for ix in range(len(outfiles)): 238 outfiles[ix].output_filename = os.path.abspath( 239 self.scons_targets[ix]) 240 else: 241 for output in self.res.GetOutputFiles(): 242 output.output_filename = os.path.abspath(os.path.join( 243 self.output_directory, output.GetFilename())) 244 245 # If there are whitelisted names, tag the tree once up front, this way 246 # while looping through the actual output, it is just an attribute check. 247 if self.whitelist_names: 248 self.AddWhitelistTags(self.res, self.whitelist_names) 249 250 for output in self.res.GetOutputFiles(): 251 self.VerboseOut('Creating %s...' % output.GetFilename()) 252 253 # Microsoft's RC compiler can only deal with single-byte or double-byte 254 # files (no UTF-8), so we make all RC files UTF-16 to support all 255 # character sets. 256 if output.GetType() in ('rc_header', 'resource_map_header', 257 'resource_map_source', 'resource_file_map_source'): 258 encoding = 'cp1252' 259 elif output.GetType() in ('android', 'c_format', 'js_map_format', 'plist', 260 'plist_strings', 'doc', 'json'): 261 encoding = 'utf_8' 262 elif output.GetType() in ('chrome_messages_json'): 263 # Chrome Web Store currently expects BOM for UTF-8 files :-( 264 encoding = 'utf-8-sig' 265 else: 266 # TODO(gfeher) modify here to set utf-8 encoding for admx/adml 267 encoding = 'utf_16' 268 269 # Set the context, for conditional inclusion of resources 270 self.res.SetOutputLanguage(output.GetLanguage()) 271 self.res.SetOutputContext(output.GetContext()) 272 self.res.SetDefines(self.defines) 273 274 # Make the output directory if it doesn't exist. 275 self.MakeDirectoriesTo(output.GetOutputFilename()) 276 277 # Write the results to a temporary file and only overwrite the original 278 # if the file changed. This avoids unnecessary rebuilds. 279 outfile = self.fo_create(output.GetOutputFilename() + '.tmp', 'wb') 280 281 if output.GetType() != 'data_package': 282 outfile = util.WrapOutputStream(outfile, encoding) 283 284 # Iterate in-order through entire resource tree, calling formatters on 285 # the entry into a node and on exit out of it. 286 with outfile: 287 self.ProcessNode(self.res, output, outfile) 288 289 # Now copy from the temp file back to the real output, but on Windows, 290 # only if the real output doesn't exist or the contents of the file 291 # changed. This prevents identical headers from being written and .cc 292 # files from recompiling (which is painful on Windows). 293 if not os.path.exists(output.GetOutputFilename()): 294 os.rename(output.GetOutputFilename() + '.tmp', 295 output.GetOutputFilename()) 296 else: 297 # CHROMIUM SPECIFIC CHANGE. 298 # This clashes with gyp + vstudio, which expect the output timestamp 299 # to change on a rebuild, even if nothing has changed. 300 #files_match = filecmp.cmp(output.GetOutputFilename(), 301 # output.GetOutputFilename() + '.tmp') 302 #if (output.GetType() != 'rc_header' or not files_match 303 # or sys.platform != 'win32'): 304 shutil.copy2(output.GetOutputFilename() + '.tmp', 305 output.GetOutputFilename()) 306 os.remove(output.GetOutputFilename() + '.tmp') 307 308 self.VerboseOut(' done.\n') 309 310 # Print warnings if there are any duplicate shortcuts. 311 warnings = shortcuts.GenerateDuplicateShortcutsWarnings( 312 self.res.UberClique(), self.res.GetTcProject()) 313 if warnings: 314 print '\n'.join(warnings) 315 316 # Print out any fallback warnings, and missing translation errors, and 317 # exit with an error code if there are missing translations in a non-pseudo 318 # and non-official build. 319 warnings = (self.res.UberClique().MissingTranslationsReport(). 320 encode('ascii', 'replace')) 321 if warnings: 322 self.VerboseOut(warnings) 323 if self.res.UberClique().HasMissingTranslations(): 324 print self.res.UberClique().missing_translations_ 325 sys.exit(-1) 326 327 def GenerateDepfile(self, input_filename, depfile, depdir): 328 '''Generate a depfile that contains the imlicit dependencies of the input 329 grd. The depfile will be in the same format as a makefile, and will contain 330 references to files relative to |depdir|. It will be put in |depfile|. 331 332 For example, supposing we have three files in a directory src/ 333 334 src/ 335 blah.grd <- depends on input{1,2}.xtb 336 input1.xtb 337 input2.xtb 338 339 and we run 340 341 grit -i blah.grd -o ../out/gen --depdir ../out --depfile ../out/gen/blah.rd.d 342 343 from the directory src/ we will generate a depfile ../out/gen/blah.grd.d 344 that has the contents 345 346 gen/blah.grd.d: ../src/input1.xtb ../src/input2.xtb 347 348 Note that all paths in the depfile are relative to ../out, the depdir. 349 ''' 350 depfile = os.path.abspath(depfile) 351 depdir = os.path.abspath(depdir) 352 # The path prefix to prepend to dependencies in the depfile. 353 prefix = os.path.relpath(os.getcwd(), depdir) 354 # The path that the depfile refers to itself by. 355 self_ref_depfile = os.path.relpath(depfile, depdir) 356 infiles = self.res.GetInputFiles() 357 deps_text = ' '.join([os.path.join(prefix, i) for i in infiles]) 358 depfile_contents = self_ref_depfile + ': ' + deps_text 359 self.MakeDirectoriesTo(depfile) 360 outfile = self.fo_create(depfile, 'wb') 361 outfile.writelines(depfile_contents) 362 363 @staticmethod 364 def MakeDirectoriesTo(file): 365 '''Creates directories necessary to contain |file|.''' 366 dir = os.path.split(file)[0] 367 if not os.path.exists(dir): 368 os.makedirs(dir) 369