1#!/usr/bin/python -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Utility methods associated with ICU source and builds.""" 18 19from __future__ import print_function 20 21import filecmp 22import glob 23import os 24import shutil 25import subprocess 26import sys 27 28import i18nutil 29import ziputil 30 31def cldrDir(): 32 """Returns the location of CLDR in the Android source tree.""" 33 android_build_top = i18nutil.GetAndroidRootOrDie() 34 cldr_dir = os.path.realpath('%s/external/cldr' % android_build_top) 35 i18nutil.CheckDirExists(cldr_dir, 'external/cldr') 36 return cldr_dir 37 38 39def icuDir(): 40 """Returns the location of ICU in the Android source tree.""" 41 android_build_top = i18nutil.GetAndroidRootOrDie() 42 icu_dir = os.path.realpath('%s/external/icu' % android_build_top) 43 i18nutil.CheckDirExists(icu_dir, 'external/icu') 44 return icu_dir 45 46 47def icu4cDir(): 48 """Returns the location of ICU4C in the Android source tree.""" 49 icu4c_dir = os.path.realpath('%s/icu4c/source' % icuDir()) 50 i18nutil.CheckDirExists(icu4c_dir, 'external/icu/icu4c/source') 51 return icu4c_dir 52 53 54def icu4jDir(): 55 """Returns the location of ICU4J in the Android source tree.""" 56 icu4j_dir = os.path.realpath('%s/icu4j' % icuDir()) 57 i18nutil.CheckDirExists(icu4j_dir, 'external/icu/icu4j') 58 return icu4j_dir 59 60 61def datFile(icu_build_dir): 62 """Returns the location of the ICU .dat file in the specified ICU build dir.""" 63 dat_file_pattern = '%s/data/out/tmp/icudt??l.dat' % icu_build_dir 64 dat_files = glob.glob(dat_file_pattern) 65 if len(dat_files) != 1: 66 print('ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)) 67 sys.exit(1) 68 dat_file = dat_files[0] 69 return dat_file 70 71 72def PrepareIcuBuild(icu_build_dir): 73 """Sets up an ICU build in the specified (non-existent) directory. 74 75 Creates the directory and runs "runConfigureICU Linux" 76 """ 77 # Keep track of the original cwd so we can go back to it at the end. 78 original_working_dir = os.getcwd() 79 80 # Create a directory to run 'make' from. 81 os.mkdir(icu_build_dir) 82 os.chdir(icu_build_dir) 83 84 # Build the ICU tools. 85 print('Configuring ICU tools...') 86 subprocess.check_call(['env', 'ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data', '%s/runConfigureICU' % icu4cDir(), 'Linux']) 87 88 os.chdir(original_working_dir) 89 90 91def MakeTzDataFiles(icu_build_dir, iana_tar_file): 92 """Builds and runs the ICU tools in ${icu_Build_dir}/tools/tzcode. 93 94 The tools are run against the specified IANA tzdata .tar.gz. 95 The resulting zoneinfo64.txt is copied into the src directories. 96 """ 97 tzcode_working_dir = '%s/tools/tzcode' % icu_build_dir 98 99 # Fix missing files. 100 # The tz2icu tool only picks up icuregions and icuzones if they are in the CWD 101 for icu_data_file in [ 'icuregions', 'icuzones']: 102 icu_data_file_source = '%s/tools/tzcode/%s' % (icu4cDir(), icu_data_file) 103 icu_data_file_symlink = '%s/%s' % (tzcode_working_dir, icu_data_file) 104 os.symlink(icu_data_file_source, icu_data_file_symlink) 105 106 iana_tar_filename = os.path.basename(iana_tar_file) 107 working_iana_tar_file = '%s/%s' % (tzcode_working_dir, iana_tar_filename) 108 shutil.copyfile(iana_tar_file, working_iana_tar_file) 109 110 print('Making ICU tz data files...') 111 # The Makefile assumes the existence of the bin directory. 112 os.mkdir('%s/bin' % icu_build_dir) 113 114 # -j1 is needed because the build is not parallelizable. http://b/109641429 115 subprocess.check_call(['make', '-j1', '-C', tzcode_working_dir]) 116 117 # Copy the source file to its ultimate destination. 118 zoneinfo_file = '%s/zoneinfo64.txt' % tzcode_working_dir 119 icu_txt_data_dir = '%s/data/misc' % icu4cDir() 120 print('Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir) 121 shutil.copy(zoneinfo_file, icu_txt_data_dir) 122 123 124def MakeAndCopyIcuDataFiles(icu_build_dir): 125 """Builds the ICU .dat and .jar files using the current src data. 126 127 The files are copied back into the expected locations in the src tree. 128 """ 129 # Keep track of the original cwd so we can go back to it at the end. 130 original_working_dir = os.getcwd() 131 132 # Regenerate the .dat file. 133 os.chdir(icu_build_dir) 134 subprocess.check_call(['make', '-j32']) 135 136 # Copy the .dat file to its ultimate destination. 137 icu_dat_data_dir = '%s/stubdata' % icu4cDir() 138 dat_file = datFile(icu_build_dir) 139 140 print('Copying %s to %s ...' % (dat_file, icu_dat_data_dir)) 141 shutil.copy(dat_file, icu_dat_data_dir) 142 143 # Generate the ICU4J .jar files 144 subprocess.check_call(['make', '-j32', 'icu4j-data']) 145 146 # Generate the test data in icu4c/source/test/testdata/out 147 subprocess.check_call(['make', '-j32', 'tests']) 148 149 # Copy the ICU4J .jar files to their ultimate destination. 150 icu_jar_data_dir = '%s/main/shared/data' % icu4jDir() 151 jarfiles = glob.glob('data/out/icu4j/*.jar') 152 if len(jarfiles) != 3: 153 print('ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)) 154 sys.exit(1) 155 for jarfile in jarfiles: 156 icu_jarfile = os.path.join(icu_jar_data_dir, os.path.basename(jarfile)) 157 if ziputil.ZipCompare(jarfile, icu_jarfile): 158 print('Ignoring %s which is identical to %s ...' % (jarfile, icu_jarfile)) 159 else: 160 print('Copying %s to %s ...' % (jarfile, icu_jar_data_dir)) 161 shutil.copy(jarfile, icu_jar_data_dir) 162 163 testdata_out_dir = '%s/test/testdata/out' % icu4cDir() 164 print('Copying test data to %s ' % testdata_out_dir) 165 if os.path.exists(testdata_out_dir): 166 shutil.rmtree(testdata_out_dir) 167 shutil.copytree('test/testdata/out', testdata_out_dir) 168 169 # Switch back to the original working cwd. 170 os.chdir(original_working_dir) 171 172 173def MakeAndCopyOverlayTzIcuData(icu_build_dir, dest_file): 174 """Makes a .dat file containing just time-zone data. 175 176 The overlay file can be used as an overlay of a full ICU .dat file 177 to provide newer time zone data. Some strings like translated 178 time zone names will be missing, but rules will be correct.""" 179 # Keep track of the original cwd so we can go back to it at the end. 180 original_working_dir = os.getcwd() 181 182 # Regenerate the .res files. 183 os.chdir(icu_build_dir) 184 subprocess.check_call(['make', '-j32']) 185 186 # The list of ICU resources needed for time zone data overlays. 187 tz_res_names = [ 188 'metaZones.res', 189 'timezoneTypes.res', 190 'windowsZones.res', 191 'zoneinfo64.res', 192 ] 193 194 dat_file = datFile(icu_build_dir) 195 icu_package_dat = os.path.basename(dat_file) 196 if not icu_package_dat.endswith('.dat'): 197 print('%s does not end with .dat' % icu_package_dat) 198 sys.exit(1) 199 icu_package = icu_package_dat[:-4] 200 201 # Create a staging directory to hold the files to go into the overlay .dat 202 res_staging_dir = '%s/overlay_res' % icu_build_dir 203 os.mkdir(res_staging_dir) 204 205 # Copy all the .res files we need from, e.g. ./data/out/build/icudt55l, to the staging directory 206 res_src_dir = '%s/data/out/build/%s' % (icu_build_dir, icu_package) 207 for tz_res_name in tz_res_names: 208 shutil.copy('%s/%s' % (res_src_dir, tz_res_name), res_staging_dir) 209 210 # Create a .lst file to pass to pkgdata. 211 tz_files_file = '%s/tzdata.lst' % res_staging_dir 212 with open(tz_files_file, "a") as tz_files: 213 for tz_res_name in tz_res_names: 214 tz_files.write('%s\n' % tz_res_name) 215 216 icu_lib_dir = '%s/lib' % icu_build_dir 217 pkg_data_bin = '%s/bin/pkgdata' % icu_build_dir 218 219 # Run pkgdata to create a .dat file. 220 icu_env = os.environ.copy() 221 icu_env["LD_LIBRARY_PATH"] = icu_lib_dir 222 223 # pkgdata treats the .lst file it is given as relative to CWD, and the path also affects the 224 # resource names in the .dat file produced so we change the CWD. 225 os.chdir(res_staging_dir) 226 227 # -F : force rebuilding all data 228 # -m common : create a .dat 229 # -v : verbose 230 # -T . : use "." as a temp dir 231 # -d . : use "." as the dest dir 232 # -p <name> : Set the "data name" 233 p = subprocess.Popen( 234 [pkg_data_bin, '-F', '-m', 'common', '-v', '-T', '.', '-d', '.', '-p', 235 icu_package, tz_files_file], 236 env=icu_env) 237 p.wait() 238 if p.returncode != 0: 239 print('pkgdata failed with status code: %s' % p.returncode) 240 241 # Copy the .dat to the chosen place / name. 242 generated_dat_file = '%s/%s' % (res_staging_dir, icu_package_dat) 243 shutil.copyfile(generated_dat_file, dest_file) 244 print('ICU overlay .dat can be found here: %s' % dest_file) 245 246 # Switch back to the original working cwd. 247 os.chdir(original_working_dir) 248 249def RequiredToMakeLangInfo(): 250 """ Returns true if icu4c/source/data/misc/langInfo.txt has been re-generated. 251 Returns false if re-generation is not needed. 252 """ 253 254 # Generate icu4c/source/data/misc/langInfo.txt by a ICU4J tool 255 langInfo_dst_path = os.path.join(icu4cDir(), 'data/misc/langInfo.txt') 256 print('Building %s' % langInfo_dst_path) 257 langInfo_out_path = '/tmp/langInfo.txt' # path hardcoded in the LocaleDistanceBuilder tool 258 if os.path.exists(langInfo_out_path): 259 os.remove(langInfo_out_path) 260 261 icu4j_dir = icu4jDir() 262 os.chdir(icu4j_dir) 263 subprocess.check_call(['ant', 'icu4jJar']) 264 os.chdir(os.path.join(icu4j_dir, 'tools', 'misc')) 265 subprocess.check_call(['ant', 'jar']) 266 subprocess.check_call([ 267 'java', 268 '-cp', 269 'out/lib/icu4j-tools.jar:../../icu4j.jar', 270 'com.ibm.icu.dev.tool.locale.LocaleDistanceBuilder', 271 ]) 272 if (filecmp.cmp(langInfo_dst_path, langInfo_out_path)): 273 print('The files {src} and {dst} are the same'.format(src=langInfo_out_path, dst=langInfo_dst_path)) 274 return False 275 276 print('Copying {src} to {dst}'.format(src=langInfo_out_path, dst=langInfo_dst_path)) 277 shutil.copyfile(langInfo_out_path, langInfo_dst_path) 278 return True 279 280def CopyLicenseFiles(target_dir): 281 """Copies ICU license files to the target_dir""" 282 283 license_file = '%s/main/shared/licenses/LICENSE' % icu4jDir() 284 print('Copying %s to %s ...' % (license_file, target_dir)) 285 shutil.copy(license_file, target_dir) 286 287