1#!/usr/bin/python3 -B 2 3# Copyright 2017 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Generates the timezone data files used by Android.""" 18 19import argparse 20import glob 21import os 22import re 23import subprocess 24import sys 25import tarfile 26import tempfile 27 28sys.path.append('%s/external/icu/tools' % os.environ.get('ANDROID_BUILD_TOP')) 29import i18nutil 30import icuutil 31import tzdatautil 32 33# Unix epoch timestamp in seconds up to which transitions in tzdata will be pregenerated. 34pregeneration_upper_bound = 4102444800 # 1 Jan 2100 35 36# Calculate the paths that are referred to by multiple functions. 37android_build_top = i18nutil.GetAndroidRootOrDie() 38timezone_dir = os.path.realpath('%s/system/timezone' % android_build_top) 39i18nutil.CheckDirExists(timezone_dir, 'system/timezone') 40 41android_host_out = i18nutil.GetAndroidHostOutOrDie() 42 43zone_compactor_dir = os.path.realpath('%s/system/timezone/input_tools/android' % android_build_top) 44i18nutil.CheckDirExists(zone_compactor_dir, 'system/timezone/input_tools/android') 45 46timezone_input_tools_dir = os.path.realpath('%s/input_tools' % timezone_dir) 47timezone_input_data_dir = os.path.realpath('%s/input_data' % timezone_dir) 48 49timezone_output_data_dir = '%s/output_data' % timezone_dir 50i18nutil.CheckDirExists(timezone_output_data_dir, 'output_data') 51 52tmp_dir = tempfile.mkdtemp('-tzdata') 53 54def GenerateZicInputFile(extracted_iana_data_dir): 55 # Android APIs assume DST means "summer time" so we follow the rearguard format 56 # introduced in 2018e. 57 zic_input_file_name = 'rearguard.zi' 58 59 # 'NDATA=' is used to remove unnecessary rules files. 60 subprocess.check_call(['make', '-C', extracted_iana_data_dir, 'NDATA=', zic_input_file_name]) 61 62 zic_input_file = '%s/%s' % (extracted_iana_data_dir, zic_input_file_name) 63 if not os.path.exists(zic_input_file): 64 print('Could not find %s' % zic_input_file) 65 sys.exit(1) 66 return zic_input_file 67 68 69def WriteSetupFile(zic_input_file): 70 """Writes the list of zones that ZoneCompactor should process.""" 71 links = [] 72 zones = [] 73 for line in open(zic_input_file): 74 fields = line.split() 75 if fields: 76 line_type = fields[0] 77 if line_type == 'Link': 78 # Each "Link" line requires the creation of a link from an old tz ID to 79 # a new tz ID, and implies the existence of a zone with the old tz ID. 80 # 81 # IANA terminology: 82 # TARGET = the new tz ID, LINK-NAME = the old tz ID 83 target = fields[1] 84 link_name = fields[2] 85 links.append('Link %s %s' % (target, link_name)) 86 zones.append('Zone %s' % link_name) 87 elif line_type == 'Zone': 88 # Each "Zone" line indicates the existence of a tz ID. 89 # 90 # IANA terminology: 91 # NAME is the tz ID, other fields like STDOFF, RULES, FORMAT,[UNTIL] are 92 # ignored. 93 name = fields[1] 94 zones.append('Zone %s' % name) 95 96 zone_compactor_setup_file = '%s/setup' % tmp_dir 97 setup = open(zone_compactor_setup_file, 'w') 98 99 # Ordering requirement from ZoneCompactor: Links must come first. 100 for link in sorted(set(links)): 101 setup.write('%s\n' % link) 102 for zone in sorted(set(zones)): 103 setup.write('%s\n' % zone) 104 setup.close() 105 return zone_compactor_setup_file 106 107 108def BuildIcuData(iana_data_tar_file): 109 icu_build_dir = '%s/icu' % tmp_dir 110 111 icuutil.PrepareIcuBuild(icu_build_dir) 112 icuutil.MakeTzDataFiles(icu_build_dir, iana_data_tar_file) 113 114 # Create ICU system image files. 115 icuutil.MakeAndCopyIcuDataFiles(icu_build_dir) 116 117 # Create the ICU overlay time zone file. 118 icu_overlay_dir = '%s/icu_overlay' % timezone_output_data_dir 119 icu_overlay_dat_file = '%s/icu_tzdata.dat' % icu_overlay_dir 120 icuutil.MakeAndCopyOverlayTzIcuData(icu_build_dir, icu_overlay_dat_file) 121 122 # There are files in ICU which generation depends on ICU itself, 123 # so multiple builds might be needed. 124 icuutil.GenerateIcuDataFiles() 125 126 # Copy ICU license file(s) 127 icuutil.CopyLicenseFiles(icu_overlay_dir) 128 129 130def GetIanaVersion(iana_tar_file): 131 iana_tar_filename = os.path.basename(iana_tar_file) 132 iana_version = re.search('tz(?:data|code)(.+)\\.tar\\.gz', iana_tar_filename).group(1) 133 return iana_version 134 135 136def ExtractTarFile(tar_file, dir): 137 print('Extracting %s...' % tar_file) 138 if not os.path.exists(dir): 139 os.mkdir(dir) 140 tar = tarfile.open(tar_file, 'r') 141 tar.extractall(dir) 142 143 144def BuildZic(iana_tools_dir): 145 iana_zic_code_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzcode') 146 iana_zic_code_version = GetIanaVersion(iana_zic_code_tar_file) 147 iana_zic_data_tar_file = tzdatautil.GetIanaTarFile(iana_tools_dir, 'tzdata') 148 iana_zic_data_version = GetIanaVersion(iana_zic_data_tar_file) 149 150 print('Found IANA zic release %s/%s in %s/%s ...' \ 151 % (iana_zic_code_version, iana_zic_data_version, iana_zic_code_tar_file, 152 iana_zic_data_tar_file)) 153 154 zic_build_dir = '%s/zic' % tmp_dir 155 ExtractTarFile(iana_zic_code_tar_file, zic_build_dir) 156 ExtractTarFile(iana_zic_data_tar_file, zic_build_dir) 157 158 # zic 159 print('Building zic...') 160 # VERSION_DEPS= is to stop the build process looking for files that might not 161 # be present across different versions. 162 subprocess.check_call(['make', '-C', zic_build_dir, 'zic']) 163 164 zic_binary_file = '%s/zic' % zic_build_dir 165 if not os.path.exists(zic_binary_file): 166 print('Could not find %s' % zic_binary_file) 167 sys.exit(1) 168 return zic_binary_file 169 170 171def BuildTzdata(zic_binary_file, extracted_iana_data_dir, iana_data_version): 172 print('Generating zic input file...') 173 zic_input_file = GenerateZicInputFile(extracted_iana_data_dir) 174 175 print('Calling zic...') 176 zic_output_dir = '%s/data' % tmp_dir 177 os.mkdir(zic_output_dir) 178 # -R specifies upper bound for generated transitions. 179 zic_cmd = [zic_binary_file, '-b', 'slim', '-R', f'@{pregeneration_upper_bound}', '-d', zic_output_dir, zic_input_file] 180 subprocess.check_call(zic_cmd) 181 182 # ZoneCompactor 183 zone_compactor_setup_file = WriteSetupFile(zic_input_file) 184 185 print('Calling ZoneCompactor to update tzdata to %s...' % iana_data_version) 186 187 tzdatautil.InvokeSoong(android_build_top, ['zone_compactor']) 188 189 # Create args for ZoneCompactor 190 header_string = 'tzdata%s' % iana_data_version 191 192 print('Executing ZoneCompactor...') 193 command = '%s/bin/zone_compactor' % android_host_out 194 iana_output_data_dir = '%s/iana' % timezone_output_data_dir 195 subprocess.check_call([command, zone_compactor_setup_file, zic_output_dir, iana_output_data_dir, 196 header_string]) 197 198 199def BuildTzlookupAndTzIds(iana_data_dir): 200 countryzones_source_file = '%s/android/countryzones.txt' % timezone_input_data_dir 201 tzlookup_dest_file = '%s/android/tzlookup.xml' % timezone_output_data_dir 202 tzids_dest_file = '%s/android/tzids.prototxt' % timezone_output_data_dir 203 204 print('Calling TzLookupGenerator to create tzlookup.xml / tzids.prototxt...') 205 tzdatautil.InvokeSoong(android_build_top, ['tzlookup_generator']) 206 207 zone_tab_file = '%s/zone.tab' % iana_data_dir 208 command = '%s/bin/tzlookup_generator' % android_host_out 209 subprocess.check_call([command, countryzones_source_file, zone_tab_file, tzlookup_dest_file, 210 tzids_dest_file]) 211 212 213def BuildTelephonylookup(): 214 telephonylookup_source_file = '%s/android/telephonylookup.txt' % timezone_input_data_dir 215 telephonylookup_dest_file = '%s/android/telephonylookup.xml' % timezone_output_data_dir 216 217 print('Calling TelephonyLookupGenerator to create telephonylookup.xml...') 218 tzdatautil.InvokeSoong(android_build_top, ['telephonylookup_generator']) 219 220 command = '%s/bin/telephonylookup_generator' % android_host_out 221 subprocess.check_call([command, telephonylookup_source_file, telephonylookup_dest_file]) 222 223 224def CreateTzVersion(iana_data_version, android_revision, output_version_file): 225 create_tz_version_script = '%s/input_tools/version/create-tz_version.py' % timezone_dir 226 227 subprocess.check_call([create_tz_version_script, 228 '-iana_version', iana_data_version, 229 '-revision', str(android_revision), 230 '-output_version_file', output_version_file]) 231 232def UpdateTestFiles(): 233 testing_data_dir = '%s/testing/data' % timezone_dir 234 update_test_files_script = '%s/create-test-data.sh' % testing_data_dir 235 subprocess.check_call([update_test_files_script], cwd=testing_data_dir) 236 237 238# Run from any directory, with no special setup required. 239# In the rare case when tzdata has to be updated, but under the same version, 240# pass "-revision" argument. 241# See http://www.iana.org/time-zones/ for more about the source of this data. 242def main(): 243 parser = argparse.ArgumentParser() 244 parser.add_argument('-revision', type=int, default=1, 245 help='Revision of current the IANA version, default = 1') 246 247 args = parser.parse_args() 248 android_revision = args.revision 249 250 print('Source data file structure: %s' % timezone_input_data_dir) 251 print('Source tools file structure: %s' % timezone_input_tools_dir) 252 print('Intermediate / working dir: %s' % tmp_dir) 253 print('Output data file structure: %s' % timezone_output_data_dir) 254 255 iana_input_data_dir = '%s/iana' % timezone_input_data_dir 256 iana_data_tar_file = tzdatautil.GetIanaTarFile(iana_input_data_dir, 'tzdata') 257 iana_data_version = GetIanaVersion(iana_data_tar_file) 258 print('IANA time zone data release %s in %s ...' % (iana_data_version, iana_data_tar_file)) 259 260 icu_dir = icuutil.icuDir() 261 print('Found icu in %s ...' % icu_dir) 262 263 BuildIcuData(iana_data_tar_file) 264 265 iana_tools_dir = '%s/iana' % timezone_input_tools_dir 266 zic_binary_file = BuildZic(iana_tools_dir) 267 268 iana_data_dir = '%s/iana_data' % tmp_dir 269 ExtractTarFile(iana_data_tar_file, iana_data_dir) 270 BuildTzdata(zic_binary_file, iana_data_dir, iana_data_version) 271 272 BuildTzlookupAndTzIds(iana_data_dir) 273 274 BuildTelephonylookup() 275 276 # Create a version file from the output from prior stages. 277 output_version_file = '%s/version/tz_version' % timezone_output_data_dir 278 CreateTzVersion(iana_data_version, android_revision, output_version_file) 279 280 # Update test versions of data files too. 281 UpdateTestFiles() 282 283 print('Look in %s and %s for new files' % (timezone_output_data_dir, icu_dir)) 284 sys.exit(0) 285 286 287if __name__ == '__main__': 288 main() 289