1#!/usr/bin/python -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Utility methods associated with ICU source and builds.""" 18 19import glob 20import os 21import shutil 22import subprocess 23import sys 24 25import i18nutil 26 27def icuDir(): 28 """Returns the location of ICU in the Android source tree.""" 29 android_build_top = i18nutil.GetAndroidRootOrDie() 30 icu_dir = os.path.realpath('%s/external/icu' % android_build_top) 31 i18nutil.CheckDirExists(icu_dir, 'external/icu') 32 return icu_dir 33 34 35def icu4cDir(): 36 """Returns the location of ICU4C in the Android source tree.""" 37 icu4c_dir = os.path.realpath('%s/icu4c/source' % icuDir()) 38 i18nutil.CheckDirExists(icu4c_dir, 'external/icu/icu4c/source') 39 return icu4c_dir 40 41 42def icu4jDir(): 43 """Returns the location of ICU4J in the Android source tree.""" 44 icu4j_dir = os.path.realpath('%s/icu4j' % icuDir()) 45 i18nutil.CheckDirExists(icu4j_dir, 'external/icu/icu4j') 46 return icu4j_dir 47 48 49def datFile(icu_build_dir): 50 """Returns the location of the ICU .dat file in the specified ICU build dir.""" 51 dat_file_pattern = '%s/data/out/tmp/icudt??l.dat' % icu_build_dir 52 dat_files = glob.glob(dat_file_pattern) 53 if len(dat_files) != 1: 54 print 'ERROR: Unexpectedly found %d .dat files (%s). Halting.' % (len(datfiles), datfiles) 55 sys.exit(1) 56 dat_file = dat_files[0] 57 return dat_file 58 59 60def PrepareIcuBuild(icu_build_dir): 61 """Sets up an ICU build in the specified (non-existent) directory. 62 63 Creates the directory and runs "runConfigureICU Linux" 64 """ 65 # Keep track of the original cwd so we can go back to it at the end. 66 original_working_dir = os.getcwd() 67 68 # Create a directory to run 'make' from. 69 os.mkdir(icu_build_dir) 70 os.chdir(icu_build_dir) 71 72 # Build the ICU tools. 73 print 'Configuring ICU tools...' 74 subprocess.check_call(['%s/runConfigureICU' % icu4cDir(), 'Linux']) 75 76 os.chdir(original_working_dir) 77 78 79def MakeTzDataFiles(icu_build_dir, iana_tar_file): 80 """Builds and runs the ICU tools in ${icu_Build_dir}/tools/tzcode. 81 82 The tools are run against the specified IANA tzdata .tar.gz. 83 The resulting zoneinfo64.txt is copied into the src directories. 84 """ 85 tzcode_working_dir = '%s/tools/tzcode' % icu_build_dir 86 87 # Fix missing files. 88 # The tz2icu tool only picks up icuregions and icuzones if they are in the CWD 89 for icu_data_file in [ 'icuregions', 'icuzones']: 90 icu_data_file_source = '%s/tools/tzcode/%s' % (icu4cDir(), icu_data_file) 91 icu_data_file_symlink = '%s/%s' % (tzcode_working_dir, icu_data_file) 92 os.symlink(icu_data_file_source, icu_data_file_symlink) 93 94 iana_tar_filename = os.path.basename(iana_tar_file) 95 working_iana_tar_file = '%s/%s' % (tzcode_working_dir, iana_tar_filename) 96 shutil.copyfile(iana_tar_file, working_iana_tar_file) 97 98 print 'Making ICU tz data files...' 99 # The Makefile assumes the existence of the bin directory. 100 os.mkdir('%s/bin' % icu_build_dir) 101 102 subprocess.check_call(['make', '-C', tzcode_working_dir]) 103 104 # Copy the source file to its ultimate destination. 105 zoneinfo_file = '%s/zoneinfo64.txt' % tzcode_working_dir 106 icu_txt_data_dir = '%s/data/misc' % icu4cDir() 107 print 'Copying zoneinfo64.txt to %s ...' % icu_txt_data_dir 108 shutil.copy(zoneinfo_file, icu_txt_data_dir) 109 110 111def MakeAndCopyIcuDataFiles(icu_build_dir): 112 """Builds the ICU .dat and .jar files using the current src data. 113 114 The files are copied back into the expected locations in the src tree. 115 """ 116 # Keep track of the original cwd so we can go back to it at the end. 117 original_working_dir = os.getcwd() 118 119 # Regenerate the .dat file. 120 os.chdir(icu_build_dir) 121 subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32']) 122 123 # Copy the .dat file to its ultimate destination. 124 icu_dat_data_dir = '%s/stubdata' % icu4cDir() 125 dat_file = datFile(icu_build_dir) 126 127 print 'Copying %s to %s ...' % (dat_file, icu_dat_data_dir) 128 shutil.copy(dat_file, icu_dat_data_dir) 129 130 # Generate the ICU4J .jar files 131 os.chdir('%s/data' % icu_build_dir) 132 subprocess.check_call(['make', 'icu4j-data']) 133 134 # Copy the ICU4J .jar files to their ultimate destination. 135 icu_jar_data_dir = '%s/main/shared/data' % icu4jDir() 136 jarfiles = glob.glob('out/icu4j/*.jar') 137 if len(jarfiles) != 2: 138 print 'ERROR: Unexpectedly found %d .jar files (%s). Halting.' % (len(jarfiles), jarfiles) 139 sys.exit(1) 140 for jarfile in jarfiles: 141 print 'Copying %s to %s ...' % (jarfile, icu_jar_data_dir) 142 shutil.copy(jarfile, icu_jar_data_dir) 143 144 # Switch back to the original working cwd. 145 os.chdir(original_working_dir) 146 147 148def MakeAndCopyOverlayTzIcuData(icu_build_dir, dest_file): 149 """Makes a .dat file containing just time-zone data. 150 151 The overlay file can be used as an overlay of a full ICU .dat file 152 to provide newer time zone data. Some strings like translated 153 time zone names will be missing, but rules will be correct.""" 154 # Keep track of the original cwd so we can go back to it at the end. 155 original_working_dir = os.getcwd() 156 157 # Regenerate the .res files. 158 os.chdir(icu_build_dir) 159 subprocess.check_call(['make', 'INCLUDE_UNI_CORE_DATA=1', '-j32']) 160 161 # The list of ICU resources needed for time zone data overlays. 162 tz_res_names = [ 163 'metaZones.res', 164 'timezoneTypes.res', 165 'windowsZones.res', 166 'zoneinfo64.res', 167 ] 168 169 dat_file = datFile(icu_build_dir) 170 icu_package_dat = os.path.basename(dat_file) 171 if not icu_package_dat.endswith('.dat'): 172 print '%s does not end with .dat' % icu_package_dat 173 sys.exit(1) 174 icu_package = icu_package_dat[:-4] 175 176 # Create a staging directory to hold the files to go into the overlay .dat 177 res_staging_dir = '%s/overlay_res' % icu_build_dir 178 os.mkdir(res_staging_dir) 179 180 # Copy all the .res files we need from, e.g. ./data/out/build/icudt55l, to the staging directory 181 res_src_dir = '%s/data/out/build/%s' % (icu_build_dir, icu_package) 182 for tz_res_name in tz_res_names: 183 shutil.copy('%s/%s' % (res_src_dir, tz_res_name), res_staging_dir) 184 185 # Create a .lst file to pass to pkgdata. 186 tz_files_file = '%s/tzdata.lst' % res_staging_dir 187 with open(tz_files_file, "a") as tz_files: 188 for tz_res_name in tz_res_names: 189 tz_files.write('%s\n' % tz_res_name) 190 191 icu_lib_dir = '%s/lib' % icu_build_dir 192 pkg_data_bin = '%s/bin/pkgdata' % icu_build_dir 193 194 # Run pkgdata to create a .dat file. 195 icu_env = os.environ.copy() 196 icu_env["LD_LIBRARY_PATH"] = icu_lib_dir 197 198 # pkgdata treats the .lst file it is given as relative to CWD, and the path also affects the 199 # resource names in the .dat file produced so we change the CWD. 200 os.chdir(res_staging_dir) 201 202 # -F : force rebuilding all data 203 # -m common : create a .dat 204 # -v : verbose 205 # -T . : use "." as a temp dir 206 # -d . : use "." as the dest dir 207 # -p <name> : Set the "data name" 208 p = subprocess.Popen( 209 [pkg_data_bin, '-F', '-m', 'common', '-v', '-T', '.', '-d', '.', '-p', 210 icu_package, tz_files_file], 211 env=icu_env) 212 p.wait() 213 if p.returncode != 0: 214 print 'pkgdata failed with status code: %s' % p.returncode 215 216 # Copy the .dat to the chosen place / name. 217 generated_dat_file = '%s/%s' % (res_staging_dir, icu_package_dat) 218 shutil.copyfile(generated_dat_file, dest_file) 219 print 'ICU overlay .dat can be found here: %s' % dest_file 220 221 # Switch back to the original working cwd. 222 os.chdir(original_working_dir) 223 224