1#!/usr/bin/python3 -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Generates the timezone data files used by Android.""" 18 19import argparse 20import glob 21import os 22import re 23import subprocess 24import sys 25import tarfile 26import tempfile 27 28sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) 29import i18nutil 30import icuutil 31import tzdatautil 32 33 34# Calculate the paths that are referred to by multiple functions. 35android_build_top = i18nutil.GetAndroidRootOrDie() 36timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) 37i18nutil.CheckDirExists(timezone_dir, 'system/timezone') 38 39android_host_out = i18nutil.GetAndroidHostOutOrDie() 40 41zone_compactor_dir = os.path.realpath('%s/system/timezone/input_tools/android' % android_build_top) 42i18nutil.CheckDirExists(zone_compactor_dir, 'system/timezone/input_tools/android') 43 44timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) 45timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) 46 47timezone_output_data_dir = '%s/output_data' % timezone_dir 48i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') 49 50tmp_dir = tempfile.mkdtemp('-tzdata') 51 52def GenerateZicInputFile(extracted_iana_data_dir): 53 # Android APIs assume DST means "summer time" so we follow the rearguard format 54 # introduced in 2018e. 55 zic_input_file_name = 'rearguard.zi' 56 57 # 'NDATA=' is used to remove unnecessary rules files. 58 subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) 59 60 zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) 61 if not os.path.exists(zic_input_file): 62 print('Could not find %s' % zic_input_file) 63 sys.exit(1) 64 return zic_input_file 65 66 67def WriteSetupFile(zic_input_file): 68 """Writes the list of zones that ZoneCompactor should process.""" 69 links = [] 70 zones = [] 71 for line in open(zic_input_file): 72 fields = line.split() 73 if fields: 74 line_type = fields[0] 75 if line_type == 'Link': 76 # Each "Link" line requires the creation of a link from an old tz ID to 77 # a new tz ID, and implies the existence of a zone with the old tz ID. 78 # 79 # IANA terminology: 80 # TARGET = the new tz ID, LINK-NAME = the old tz ID 81 target = fields[1] 82 link_name = fields[2] 83 links.append('Link %s %s' % (target, link_name)) 84 zones.append('Zone %s' % link_name) 85 elif line_type == 'Zone': 86 # Each "Zone" line indicates the existence of a tz ID. 87 # 88 # IANA terminology: 89 # NAME is the tz ID, other fields like STDOFF, RULES, FORMAT,[UNTIL] are 90 # ignored. 91 name = fields[1] 92 zones.append('Zone %s' % name) 93 94 zone_compactor_setup_file = '%s/setup' % tmp_dir 95 setup = open(zone_compactor_setup_file, 'w') 96 97 # Ordering requirement from ZoneCompactor: Links must come first. 98 for link in sorted(set(links)): 99 setup.write('%s\n' % link) 100 for zone in sorted(set(zones)): 101 setup.write('%s\n' % zone) 102 setup.close() 103 return zone_compactor_setup_file 104 105 106def BuildIcuData(iana_data_tar_file): 107 icu_build_dir = '%s/icu' % tmp_dir 108 109 icuutil.PrepareIcuBuild(icu_build_dir) 110 icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) 111 112 # Create ICU system image files. 113 icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) 114 115 # Create the ICU overlay time zone file. 116 icu_overlay_dir = '%s/icu_overlay' % timezone_output_data_dir 117 icu_overlay_dat_file = '%s/icu_tzdata.dat' % icu_overlay_dir 118 icuutil.MakeAndCopyOverlayTzIcuData(icu_build_dir, icu_overlay_dat_file) 119 120 # There are files in ICU which generation depends on ICU itself, 121 # so multiple builds might be needed. 122 icuutil.GenerateIcuDataFiles() 123 124 # Copy ICU license file(s) 125 icuutil.CopyLicenseFiles(icu_overlay_dir) 126 127 128def GetIanaVersion(iana_tar_file): 129 iana_tar_filename = os.path.basename(iana_tar_file) 130 iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) 131 return iana_version 132 133 134def ExtractTarFile(tar_file, dir): 135 print('Extracting %s...' % tar_file) 136 if not os.path.exists(dir): 137 os.mkdir(dir) 138 tar = tarfile.open(tar_file, 'r') 139 tar.extractall(dir) 140 141 142def BuildZic(iana_tools_dir): 143 iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode') 144 iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) 145 iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata') 146 iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) 147 148 print('Found IANA zic release %s/%s in %s/%s ...' \ 149 % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, 150 iana_zic_data_tar_file)) 151 152 zic_build_dir = '%s/zic' % tmp_dir 153 ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) 154 ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) 155 156 # zic 157 print('Building zic...') 158 # VERSION_DEPS= is to stop the build process looking for files that might not 159 # be present across different versions. 160 subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) 161 162 zic_binary_file = '%s/zic' % zic_build_dir 163 if not os.path.exists(zic_binary_file): 164 print('Could not find %s' % zic_binary_file) 165 sys.exit(1) 166 return zic_binary_file 167 168 169def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): 170 print('Generating zic input file...') 171 zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) 172 173 print('Calling zic...') 174 zic_output_dir = '%s/data' % tmp_dir 175 os.mkdir(zic_output_dir) 176 zic_cmd = [zic_binary_file, '-b', 'fat', '-d', zic_output_dir, zic_input_file] 177 subprocess.check_call(zic_cmd) 178 179 # ZoneCompactor 180 zone_compactor_setup_file = WriteSetupFile(zic_input_file) 181 182 print('Calling ZoneCompactor to update tzdata to %s...' % iana_data_version) 183 184 tzdatautil.InvokeSoong(android_build_top, ['zone_compactor']) 185 186 # Create args for ZoneCompactor 187 header_string = 'tzdata%s' % iana_data_version 188 189 print('Executing ZoneCompactor...') 190 command = '%s/bin/zone_compactor' % android_host_out 191 iana_output_data_dir = '%s/iana' % timezone_output_data_dir 192 subprocess.check_call([command, zone_compactor_setup_file, zic_output_dir, iana_output_data_dir, 193 header_string]) 194 195 196def BuildTzlookupAndTzIds(iana_data_dir): 197 countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir 198 tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 199 tzids_dest_file = '%s/android/tzids.prototxt' % timezone_output_data_dir 200 201 print('Calling TzLookupGenerator to create tzlookup.xml / tzids.prototxt...') 202 tzdatautil.InvokeSoong(android_build_top, ['tzlookup_generator']) 203 204 zone_tab_file = '%s/zone.tab' % iana_data_dir 205 command = '%s/bin/tzlookup_generator' % android_host_out 206 subprocess.check_call([command, countryzones_source_file, zone_tab_file, tzlookup_dest_file, 207 tzids_dest_file]) 208 209 210def BuildTelephonylookup(): 211 telephonylookup_source_file = '%s/android/telephonylookup.txt' % timezone_input_data_dir 212 telephonylookup_dest_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir 213 214 print('Calling TelephonyLookupGenerator to create telephonylookup.xml...') 215 tzdatautil.InvokeSoong(android_build_top, ['telephonylookup_generator']) 216 217 command = '%s/bin/telephonylookup_generator' % android_host_out 218 subprocess.check_call([command, telephonylookup_source_file, telephonylookup_dest_file]) 219 220 221def CreateDistroFiles(iana_data_version, android_revision, 222 output_distro_dir, output_version_file): 223 create_distro_script = '%s/distro/tools/create-distro.py' % timezone_dir 224 225 tzdata_file = '%s/iana/tzdata' % timezone_output_data_dir 226 icu_dir = '%s/icu_overlay' % timezone_output_data_dir 227 tzlookup_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 228 telephonylookup_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir 229 230 distro_file_pattern = '%s/*.zip' % output_distro_dir 231 existing_files = glob.glob(distro_file_pattern) 232 233 print('Removing %s' % existing_files) 234 for existing_file in existing_files: 235 os.remove(existing_file) 236 237 subprocess.check_call([create_distro_script, 238 '-iana_version', iana_data_version, 239 '-revision', str(android_revision), 240 '-tzdata', tzdata_file, 241 '-icu_dir', icu_dir, 242 '-tzlookup', tzlookup_file, 243 '-telephonylookup', telephonylookup_file, 244 '-output_distro_dir', output_distro_dir, 245 '-output_version_file', output_version_file]) 246 247def UpdateTestFiles(): 248 testing_data_dir = '%s/testing/data' % timezone_dir 249 update_test_files_script = '%s/create-test-data.sh' % testing_data_dir 250 subprocess.check_call([update_test_files_script], cwd=testing_data_dir) 251 252 253# Run from any directory, with no special setup required. 254# In the rare case when tzdata has to be updated, but under the same version, 255# pass "-revision" argument. 256# See http://www.iana.org/time-zones/ for more about the source of this data. 257def main(): 258 parser = argparse.ArgumentParser() 259 parser.add_argument('-revision', type=int, default=1, 260 help='The distro revision for the IANA version, default = 1') 261 262 args = parser.parse_args() 263 android_revision = args.revision 264 265 print('Source data file structure: %s' % timezone_input_data_dir) 266 print('Source tools file structure: %s' % timezone_input_tools_dir) 267 print('Intermediate / working dir: %s' % tmp_dir) 268 print('Output data file structure: %s' % timezone_output_data_dir) 269 270 iana_input_data_dir = '%s/iana' % timezone_input_data_dir 271 iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata') 272 iana_data_version = GetIanaVersion(iana_data_tar_file) 273 print('IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file)) 274 275 icu_dir = icuutil.icuDir() 276 print('Found icu in %s ...' % icu_dir) 277 278 BuildIcuData(iana_data_tar_file) 279 280 iana_tools_dir = '%s/iana' % timezone_input_tools_dir 281 zic_binary_file = BuildZic(iana_tools_dir) 282 283 iana_data_dir = '%s/iana_data' % tmp_dir 284 ExtractTarFile(iana_data_tar_file, iana_data_dir) 285 BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) 286 287 BuildTzlookupAndTzIds(iana_data_dir) 288 289 BuildTelephonylookup() 290 291 # Create a distro file and version file from the output from prior stages. 292 output_distro_dir = '%s/distro' % timezone_output_data_dir 293 output_version_file = '%s/version/tz_version' % timezone_output_data_dir 294 CreateDistroFiles(iana_data_version, android_revision, 295 output_distro_dir, output_version_file) 296 297 # Update test versions of distro files too. 298 UpdateTestFiles() 299 300 print('Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir)) 301 sys.exit(0) 302 303 304if __name__ == '__main__': 305 main() 306