1#!/usr/bin/python -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Utility methods associated with ICU source and builds.""" 18 19from __future__ import print_function 20 21import glob 22import os 23import shutil 24import subprocess 25import sys 26 27import i18nutil 28import ziputil 29 30def cldrDir(): 31 """Returns the location of CLDR in the Android source tree.""" 32 android_build_top = i18nutil.GetAndroidRootOrDie() 33 cldr_dir = os.path.realpath('%s/external/cldr' % android_build_top) 34 i18nutil.CheckDirExists(cldr_dir, 'external/cldr') 35 return cldr_dir 36 37 38def icuDir(): 39 """Returns the location of ICU in the Android source tree.""" 40 android_build_top = i18nutil.GetAndroidRootOrDie() 41 icu_dir = os.path.realpath('%s/external/icu' % android_build_top) 42 i18nutil.CheckDirExists(icu_dir, 'external/icu') 43 return icu_dir 44 45 46def icu4cDir(): 47 """Returns the location of ICU4C in the Android source tree.""" 48 icu4c_dir = os.path.realpath('%s/icu4c/source' % icuDir()) 49 i18nutil.CheckDirExists(icu4c_dir, 'external/icu/icu4c/source') 50 return icu4c_dir 51 52 53def icu4jDir(): 54 """Returns the location of ICU4J in the Android source tree.""" 55 icu4j_dir = os.path.realpath('%s/icu4j' % icuDir()) 56 i18nutil.CheckDirExists(icu4j_dir, 'external/icu/icu4j') 57 return icu4j_dir 58 59 60def datFile(icu_build_dir): 61 """Returns the location of the ICU .dat file in the specified ICU build dir.""" 62 dat_file_pattern = '%s/data/out/tmp/icudt??l.dat' % icu_build_dir 63 dat_files = glob.glob(dat_file_pattern) 64 if len(dat_files) != 1: 65 print('ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles)) 66 sys.exit(1) 67 dat_file = dat_files[0] 68 return dat_file 69 70 71def PrepareIcuBuild(icu_build_dir): 72 """Sets up an ICU build in the specified (non-existent) directory. 73 74 Creates the directory and runs "runConfigureICU Linux" 75 """ 76 # Keep track of the original cwd so we can go back to it at the end. 77 original_working_dir = os.getcwd() 78 79 # Create a directory to run 'make' from. 80 os.mkdir(icu_build_dir) 81 os.chdir(icu_build_dir) 82 83 # Build the ICU tools. 84 print('Configuring ICU tools...') 85 subprocess.check_call(['%s/runConfigureICU' % icu4cDir(), 'Linux']) 86 87 os.chdir(original_working_dir) 88 89 90def MakeTzDataFiles(icu_build_dir, iana_tar_file): 91 """Builds and runs the ICU tools in ${icu_Build_dir}/tools/tzcode. 92 93 The tools are run against the specified IANA tzdata .tar.gz. 94 The resulting zoneinfo64.txt is copied into the src directories. 95 """ 96 tzcode_working_dir = '%s/tools/tzcode' % icu_build_dir 97 98 # Fix missing files. 99 # The tz2icu tool only picks up icuregions and icuzones if they are in the CWD 100 for icu_data_file in [ 'icuregions', 'icuzones']: 101 icu_data_file_source = '%s/tools/tzcode/%s' % (icu4cDir(), icu_data_file) 102 icu_data_file_symlink = '%s/%s' % (tzcode_working_dir, icu_data_file) 103 os.symlink(icu_data_file_source, icu_data_file_symlink) 104 105 iana_tar_filename = os.path.basename(iana_tar_file) 106 working_iana_tar_file = '%s/%s' % (tzcode_working_dir, iana_tar_filename) 107 shutil.copyfile(iana_tar_file, working_iana_tar_file) 108 109 print('Making ICU tz data files...') 110 # The Makefile assumes the existence of the bin directory. 111 os.mkdir('%s/bin' % icu_build_dir) 112 113 # -j1 is needed because the build is not parallelizable. http://b/109641429 114 subprocess.check_call(['make', '-j1', '-C', tzcode_working_dir]) 115 116 # Copy the source file to its ultimate destination. 117 zoneinfo_file = '%s/zoneinfo64.txt' % tzcode_working_dir 118 icu_txt_data_dir = '%s/data/misc' % icu4cDir() 119 print('Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir) 120 shutil.copy(zoneinfo_file, icu_txt_data_dir) 121 122 123def MakeAndCopyIcuDataFiles(icu_build_dir): 124 """Builds the ICU .dat and .jar files using the current src data. 125 126 The files are copied back into the expected locations in the src tree. 127 """ 128 # Keep track of the original cwd so we can go back to it at the end. 129 original_working_dir = os.getcwd() 130 131 # Regenerate the .dat file. 132 os.chdir(icu_build_dir) 133 subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32']) 134 135 # Copy the .dat file to its ultimate destination. 136 icu_dat_data_dir = '%s/stubdata' % icu4cDir() 137 dat_file = datFile(icu_build_dir) 138 139 print('Copying %s to %s ...' % (dat_file, icu_dat_data_dir)) 140 shutil.copy(dat_file, icu_dat_data_dir) 141 142 # Generate the ICU4J .jar files 143 os.chdir('%s/data' % icu_build_dir) 144 subprocess.check_call(['make', '-j32', 'icu4j-data']) 145 146 # Copy the ICU4J .jar files to their ultimate destination. 147 icu_jar_data_dir = '%s/main/shared/data' % icu4jDir() 148 jarfiles = glob.glob('out/icu4j/*.jar') 149 if len(jarfiles) != 2: 150 print('ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles)) 151 sys.exit(1) 152 for jarfile in jarfiles: 153 icu_jarfile = os.path.join(icu_jar_data_dir, os.path.basename(jarfile)) 154 if ziputil.ZipCompare(jarfile, icu_jarfile): 155 print('Ignoring %s which is identical to %s ...' % (jarfile, icu_jarfile)) 156 else: 157 print('Copying %s to %s ...' % (jarfile, icu_jar_data_dir)) 158 shutil.copy(jarfile, icu_jar_data_dir) 159 160 # Switch back to the original working cwd. 161 os.chdir(original_working_dir) 162 163 164def MakeAndCopyOverlayTzIcuData(icu_build_dir, dest_file): 165 """Makes a .dat file containing just time-zone data. 166 167 The overlay file can be used as an overlay of a full ICU .dat file 168 to provide newer time zone data. Some strings like translated 169 time zone names will be missing, but rules will be correct.""" 170 # Keep track of the original cwd so we can go back to it at the end. 171 original_working_dir = os.getcwd() 172 173 # Regenerate the .res files. 174 os.chdir(icu_build_dir) 175 subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32']) 176 177 # The list of ICU resources needed for time zone data overlays. 178 tz_res_names = [ 179 'metaZones.res', 180 'timezoneTypes.res', 181 'windowsZones.res', 182 'zoneinfo64.res', 183 ] 184 185 dat_file = datFile(icu_build_dir) 186 icu_package_dat = os.path.basename(dat_file) 187 if not icu_package_dat.endswith('.dat'): 188 print('%s does not end with .dat' % icu_package_dat) 189 sys.exit(1) 190 icu_package = icu_package_dat[:-4] 191 192 # Create a staging directory to hold the files to go into the overlay .dat 193 res_staging_dir = '%s/overlay_res' % icu_build_dir 194 os.mkdir(res_staging_dir) 195 196 # Copy all the .res files we need from, e.g. ./data/out/build/icudt55l, to the staging directory 197 res_src_dir = '%s/data/out/build/%s' % (icu_build_dir, icu_package) 198 for tz_res_name in tz_res_names: 199 shutil.copy('%s/%s' % (res_src_dir, tz_res_name), res_staging_dir) 200 201 # Create a .lst file to pass to pkgdata. 202 tz_files_file = '%s/tzdata.lst' % res_staging_dir 203 with open(tz_files_file, "a") as tz_files: 204 for tz_res_name in tz_res_names: 205 tz_files.write('%s\n' % tz_res_name) 206 207 icu_lib_dir = '%s/lib' % icu_build_dir 208 pkg_data_bin = '%s/bin/pkgdata' % icu_build_dir 209 210 # Run pkgdata to create a .dat file. 211 icu_env = os.environ.copy() 212 icu_env["LD_LIBRARY_PATH"] = icu_lib_dir 213 214 # pkgdata treats the .lst file it is given as relative to CWD, and the path also affects the 215 # resource names in the .dat file produced so we change the CWD. 216 os.chdir(res_staging_dir) 217 218 # -F : force rebuilding all data 219 # -m common : create a .dat 220 # -v : verbose 221 # -T . : use "." as a temp dir 222 # -d . : use "." as the dest dir 223 # -p <name> : Set the "data name" 224 p = subprocess.Popen( 225 [pkg_data_bin, '-F', '-m', 'common', '-v', '-T', '.', '-d', '.', '-p', 226 icu_package, tz_files_file], 227 env=icu_env) 228 p.wait() 229 if p.returncode != 0: 230 print('pkgdata failed with status code: %s' % p.returncode) 231 232 # Copy the .dat to the chosen place / name. 233 generated_dat_file = '%s/%s' % (res_staging_dir, icu_package_dat) 234 shutil.copyfile(generated_dat_file, dest_file) 235 print('ICU overlay .dat can be found here: %s' % dest_file) 236 237 # Switch back to the original working cwd. 238 os.chdir(original_working_dir) 239 240 241def CopyLicenseFiles(target_dir): 242 """Copies ICU license files to the target_dir""" 243 244 license_file = '%s/main/shared/licenses/LICENSE' % icu4jDir() 245 print('Copying %s to %s ...' % (license_file, target_dir)) 246 shutil.copy(license_file, target_dir) 247 248