1#!/usr/bin/python 2# 3# Copyright (C) 2010 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17# Generate ICU dat files for locale relevant resources. 18# 19# Usage: 20# icu_dat_generator.py [-v] [-h] 21# 22# Sample usage: 23# $ANDROID_BUILD_TOP/external/icu4c/stubdata$ ./icu_dat_generator.py --verbose 24# 25# Add new dat file: 26# 1. Add icudtxxl-<datname>.txt to $ANDROID_BUILD_TOP/external/icu4c/stubdata. 27# Check the example file under 28# $ANDROID_BUILD_TOP/external/icu4c/stubdata/icudt48l-us.txt 29# 2. Add an entry to main() --> datlist[] 30# 3. Run this script to generate dat files. 31# 32# For ICU upgrade 33# We cannot get CLDR version from dat file unless calling ICU function. 34# If there is a CLDR version change, please modify "global CLDR_VERSION". 35 36import getopt 37import glob 38import os.path 39import re 40import shutil 41import subprocess 42import sys 43 44 45def PrintHelpAndExit(): 46 print "Usage:" 47 print " icu_dat_generator.py [-v|--verbose] [-h|--help]" 48 print "Example:" 49 print " $ANDROID_BUILD_TOP/external/icu4c/stubdata$ ./icu_dat_generator.py" 50 sys.exit(1) 51 52 53def InvokeIcuTool(tool, working_dir, args): 54 command_list = [os.path.join(ICU_PREBUILT_DIR, tool)] 55 command_list.extend(args) 56 57 if VERBOSE: 58 command = "[%s] %s" % (working_dir, " ".join(command_list)) 59 print command 60 61 ret = subprocess.call(command_list, cwd=working_dir) 62 if ret != 0: 63 sys.exit(command_list[0:]) 64 65 66def ExtractAllResourceToTempDir(): 67 # copy icudtxxl-all.dat to icudtxxl.dat 68 src_dat = os.path.join(ICU4C_DIR, "stubdata", ICUDATA + "-all.dat") 69 dst_dat = os.path.join(ICU4C_DIR, "stubdata", ICUDATA + ".dat") 70 shutil.copyfile(src_dat, dst_dat) 71 InvokeIcuTool("icupkg", None, [dst_dat, "-x", "*", "-d", TMP_DAT_PATH]) 72 73 74def MakeDat(input_file, stubdata_dir): 75 print "------ Processing '%s'..." % (input_file) 76 if not os.path.isfile(input_file): 77 print "%s not a file!" % input_file 78 sys.exit(1) 79 GenResIndex(input_file) 80 CopyAndroidCnvFiles(stubdata_dir) 81 # Run "icupkg -tl -s icudt48l -a icudt48l-us.txt new icudt48l.dat". 82 args = ["-tl", "-s", TMP_DAT_PATH, "-a", input_file, "new", ICUDATA + ".dat"] 83 InvokeIcuTool("icupkg", TMP_DAT_PATH, args) 84 85 86def WriteIndex(path, locales, cldr_version=None): 87 empty_value = " {\"\"}\n" # key-value pair for all locale entries 88 89 f = open(path, "w") 90 f.write("res_index:table(nofallback) {\n") 91 if cldr_version: 92 f.write(" CLDRVersion { %s }\n" % cldr_version) 93 f.write(" InstalledLocales {\n") 94 for locale in locales: 95 f.write(locale + empty_value) 96 97 f.write(" }\n") 98 f.write("}\n") 99 f.close() 100 101 102def AddResFile(collection, path): 103 # There are two consumers of the the input .txt file: this script and 104 # icupkg. We only care about .res files, but icupkg needs files they depend 105 # on too, so it's not an error to have to ignore non-.res files here. 106 end = path.find(".res") 107 if end > 0: 108 collection.add(path[path.find("/")+1:end]) 109 return 110 111 112# Open input file (such as icudt48l-us.txt). 113# Go through the list and generate res_index.txt for locales, brkitr, 114# coll, et cetera. 115def GenResIndex(input_file): 116 res_index = "res_index.txt" 117 118 brkitrs = set() 119 colls = set() 120 currs = set() 121 langs = set() 122 locales = set() 123 regions = set() 124 zones = set() 125 126 for line in open(input_file, "r"): 127 if "root." in line or "res_index" in line or "_.res" in line: 128 continue 129 if "brkitr/" in line: 130 AddResFile(brkitrs, line) 131 elif "coll/" in line: 132 AddResFile(colls, line) 133 elif "curr/" in line: 134 AddResFile(currs, line) 135 elif "lang/" in line: 136 AddResFile(langs, line) 137 elif "region/" in line: 138 AddResFile(regions, line) 139 elif "zone/" in line: 140 AddResFile(zones, line) 141 elif ".res" in line: 142 # We need to determine the resource is locale resource or misc resource. 143 # To determine the locale resource, we assume max script length is 3. 144 end = line.find(".res") 145 if end <= 3 or (line.find("_") <= 3 and line.find("_") > 0): 146 locales.add(line[:end]) 147 148 kind_to_locales = { 149 "brkitr": brkitrs, 150 "coll": colls, 151 "curr": currs, 152 "lang": langs, 153 "locales": locales, 154 "region": regions, 155 "zone": zones 156 } 157 158 # Find every locale we've mentioned, for whatever reason. 159 every_locale = set() 160 for locales in kind_to_locales.itervalues(): 161 every_locale = every_locale.union(locales) 162 163 if VERBOSE: 164 for kind, locales in kind_to_locales.items(): 165 print "%s=%s" % (kind, sorted(locales)) 166 167 # Print a human-readable list of the languages supported. 168 every_language = set() 169 for locale in every_locale: 170 language = re.sub(r"(_.*)", "", locale) 171 if language != "pool" and language != "supplementalData": 172 every_language.add(language) 173 input_basename = os.path.basename(input_file) 174 print "%s includes %s." % (input_basename, ", ".join(sorted(every_language))) 175 176 # Find cases where we've included only part of a locale's data. 177 missing_files = [] 178 for locale in every_locale: 179 for kind, locales in kind_to_locales.items(): 180 p = os.path.join(ICU4C_DIR, "data", kind, locale + ".txt") 181 if not locale in locales and os.path.exists(p): 182 missing_files.append(p) 183 184 # Warn about the missing files. 185 for missing_file in sorted(missing_files): 186 relative_path = "/".join(missing_file.split("/")[-2:]) 187 print "warning: missing data for supported locale: %s" % relative_path 188 189 # Write the genrb input files. 190 WriteIndex(os.path.join(TMP_DAT_PATH, res_index), locales, CLDR_VERSION) 191 for kind, locales in kind_to_locales.items(): 192 if kind == "locales": 193 continue 194 WriteIndex(os.path.join(TMP_DAT_PATH, kind, res_index), locales) 195 196 # Call genrb to generate new res_index.res. 197 InvokeIcuTool("genrb", TMP_DAT_PATH, [res_index]) 198 for kind, locales in kind_to_locales.items(): 199 if kind == "locales": 200 continue 201 InvokeIcuTool("genrb", os.path.join(TMP_DAT_PATH, kind), [res_index]) 202 203 204def CopyAndroidCnvFiles(stubdata_dir): 205 android_specific_cnv = ["gsm-03.38-2000.cnv", 206 "iso-8859_16-2001.cnv", 207 "docomo-shift_jis-2007.cnv", 208 "kddi-jisx-208-2007.cnv", 209 "kddi-shift_jis-2007.cnv", 210 "softbank-jisx-208-2007.cnv", 211 "softbank-shift_jis-2007.cnv"] 212 for cnv_file in android_specific_cnv: 213 src_path = os.path.join(stubdata_dir, "cnv", cnv_file) 214 dst_path = os.path.join(TMP_DAT_PATH, cnv_file) 215 shutil.copyfile(src_path, dst_path) 216 if VERBOSE: 217 print "copy " + src_path + " " + dst_path 218 219 220def main(): 221 global ANDROID_BUILD_TOP # $ANDROID_BUILD_TOP 222 global ICU4C_DIR # $ANDROID_BUILD_TOP/external/icu4c 223 global ICU_PREBUILT_DIR # Directory containing pre-built ICU tools. 224 global ICUDATA # e.g. "icudt48l" 225 global CLDR_VERSION # CLDR version. The value varies between ICU releases. 226 global TMP_DAT_PATH # temp directory to store all resource files and 227 # intermediate dat files. 228 global VERBOSE 229 230 CLDR_VERSION = "2.0.1" 231 VERBOSE = False 232 233 show_help = False 234 try: 235 opts, args = getopt.getopt(sys.argv[1:], "hv", ["help", "verbose"]) 236 except getopt.error: 237 PrintHelpAndExit() 238 for opt, _ in opts: 239 if opt in ("-h", "--help"): 240 show_help = True 241 elif opt in ("-v", "--verbose"): 242 VERBOSE = True 243 if args: 244 show_help = True 245 246 if show_help: 247 PrintHelpAndExit() 248 249 ANDROID_BUILD_TOP = os.environ.get("ANDROID_BUILD_TOP") 250 if not ANDROID_BUILD_TOP: 251 print "$ANDROID_BUILD_TOP not set! Run 'env_setup.sh'." 252 sys.exit(1) 253 ICU4C_DIR = os.path.join(ANDROID_BUILD_TOP, "external", "icu4c") 254 stubdata_dir = os.path.join(ICU4C_DIR, "stubdata") 255 256 # Find all the input files. 257 input_files = glob.glob(os.path.join(stubdata_dir, "icudt[0-9][0-9]l-*.txt")) 258 259 # Work out the ICU version from the input filenames, so we can find the 260 # appropriate pre-built ICU tools. 261 icu_version = re.sub(r"([^0-9])", "", os.path.basename(input_files[0])) 262 ICU_PREBUILT_DIR = os.path.join(os.environ.get("ANDROID_BUILD_TOP"), 263 "prebuilt", "linux-x86_64", "icu-%s.%s" % (icu_version[0], icu_version[1])) 264 if not os.path.exists(ICU_PREBUILT_DIR): 265 print "%s does not exist!" % ICU_PREBUILT_DIR 266 267 ICUDATA = "icudt" + icu_version + "l" 268 269 # Check that -all.dat exists (since we build the other .dat files from that). 270 full_data_filename = os.path.join(stubdata_dir, ICUDATA + "-all.dat") 271 if not os.path.isfile(full_data_filename): 272 print "%s not present." % full_data_filename 273 sys.exit(1) 274 275 # Create a temporary working directory. 276 TMP_DAT_PATH = os.path.join(ICU4C_DIR, "tmp") 277 if os.path.exists(TMP_DAT_PATH): 278 shutil.rmtree(TMP_DAT_PATH) 279 os.mkdir(TMP_DAT_PATH) 280 281 # Extract resource files from icudtxxl-all.dat to TMP_DAT_PATH. 282 ExtractAllResourceToTempDir() 283 284 # Process each input file in turn. 285 for input_file in sorted(input_files): 286 output_file = input_file[:-3] + "dat" 287 MakeDat(input_file, stubdata_dir) 288 shutil.copyfile(os.path.join(TMP_DAT_PATH, ICUDATA + ".dat"), output_file) 289 print "Generated ICU data: %s" % output_file 290 291 # Cleanup temporary working directory and icudtxxl.dat 292 shutil.rmtree(TMP_DAT_PATH) 293 os.remove(os.path.join(stubdata_dir, ICUDATA + ".dat")) 294 295if __name__ == "__main__": 296 main() 297