• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/python3 -B
2# Copyright 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Regenerates (just) ICU data source files used to build ICU data files."""
17
18from __future__ import print_function
19
20import os
21import shutil
22import subprocess
23import sys
24from pathlib import Path
25
26import i18nutil
27import icuutil
28
29
30# Run with no arguments from any directory, with no special setup required.
31# See icu4c/source/data/cldr-icu-readme.txt for the upstream ICU instructions.
32def main():
33  if subprocess.call(["which", "mvn"]) != 0 or subprocess.call(["which", "ant"]) != 0:
34    print("Can't find the required tools. Run `sudo apt-get install maven ant` to install")
35    exit(1)
36
37  if not os.path.exists(os.path.join(Path.home(), ".m2/settings.xml")):
38    print("Can\'t find `~/.m2/settings.xml`. Please follow the instructions at "
39          "http://cldr.unicode.org/development/maven to create one and the github token.")
40    exit(1)
41
42  cldr_dir = icuutil.cldrDir()
43  print('Found cldr in %s ...' % cldr_dir)
44  icu_dir = icuutil.icuDir()
45  print('Found icu in %s ...' % icu_dir)
46
47  # Ant doesn't have any mechanism for using a build directory separate from the
48  # source directory so this build script creates a temporary directory and then
49  # copies all necessary ICU4J and CLDR source code to here before building it:
50  i18nutil.SwitchToNewTemporaryDirectory()
51  build_dir = os.getcwd()
52  cldr_build_dir = os.path.join(build_dir, 'cldr')
53  icu4c_build_dir = os.path.join(build_dir, 'icu4c')
54  icu4j_build_dir = os.path.join(build_dir, 'icu4j')
55  icu_tools_build_dir = os.path.join(build_dir, 'icu_tools')
56
57  print('Copying CLDR source code ...')
58  shutil.copytree(cldr_dir, cldr_build_dir, symlinks=True)
59  print('Copying ICU4C source code ...')
60  shutil.copytree(os.path.join(icu_dir, 'icu4c'), icu4c_build_dir, symlinks=True)
61  print('Copying ICU4J source code ...')
62  shutil.copytree(os.path.join(icu_dir, 'icu4j'), icu4j_build_dir, symlinks=True)
63  print('Copying ICU tools source code ...')
64  shutil.copytree(os.path.join(icu_dir, 'tools'), icu_tools_build_dir, symlinks=True)
65
66  # Setup environment variables for all subshell
67  os.environ['ANT_OPTS'] = '-Xmx4096m'
68
69  # This is the location of the original CLDR source tree (not the temporary
70  # copy of the tools source code) from where the data files are to be read:
71  os.environ['CLDR_DIR'] = cldr_build_dir  # os.path.join(os.getcwd(), 'cldr')
72
73  os.environ['ICU4C_ROOT'] = icu4c_build_dir
74  os.environ['ICU4J_ROOT'] = icu4j_build_dir
75  os.environ['TOOLS_ROOT'] = icu_tools_build_dir
76  cldr_tmp_dir = os.path.join(build_dir, 'cldr-staging')
77  os.environ['CLDR_TMP_DIR'] = cldr_tmp_dir
78
79  icu_tools_cldr_dir = os.path.join(icu_tools_build_dir, 'cldr')
80  print('Installing CLDR tools ...')
81  os.chdir(icu_tools_cldr_dir)
82  subprocess.check_call(['ant', 'install-cldr-libs'])
83
84  print('Building ICU data...')
85  icu4c_data_build_dir = os.path.join(icu4c_build_dir, 'source/data')
86  os.chdir(icu4c_data_build_dir)
87  subprocess.check_call(['ant', 'cleanprod'])
88  subprocess.check_call(['ant', 'setup'])
89  subprocess.check_call(['ant', 'proddata'])
90
91  # Finally we "compile" CLDR-data to a "production" form and place it in ICU
92  os.chdir(os.path.join(icu_tools_build_dir, 'cldr', 'cldr-to-icu'))
93  cldr_production_tmp_dir = os.path.join(cldr_tmp_dir, 'production')
94  subprocess.check_call([
95    'ant',
96    '-f',
97    'build-icu-data.xml',
98    '-DcldrDataDir=' + cldr_production_tmp_dir,
99    '-DforceDelete=true',
100    '-DincludePseudoLocales=true'
101  ])
102
103  os.chdir(icu_tools_cldr_dir)
104  subprocess.check_call([
105    'ant',
106    'copy-cldr-testdata',
107  ])
108
109  # Copy the generated data files from the temporary directory into AOSP.
110  icu4c_data_source_dir = os.path.join(icu_dir, 'icu4c/source/data')
111  rmAndCopyTree(icu4c_data_build_dir, icu4c_data_source_dir)
112  rmAndCopyTree(
113    os.path.join(icu4c_build_dir, 'source/test/testdata/cldr'),
114    os.path.join(icu_dir, 'icu4c/source/test/testdata/cldr'))
115  rmAndCopyTree(
116    os.path.join(icu4j_build_dir, 'main/tests/core/src/com/ibm/icu/dev/data/cldr'),
117    os.path.join(icu_dir, 'icu4j/main/tests/core/src/com/ibm/icu/dev/data/cldr'))
118
119  localeCanonicalization_src = os.path.join(
120    cldr_production_tmp_dir, 'common/testData/localeIdentifiers/localeCanonicalization.txt')
121  shutil.copy(localeCanonicalization_src, os.path.join(
122    icu_dir, 'icu4c/source/test/testdata/localeCanonicalization.txt'))
123  shutil.copy(localeCanonicalization_src, os.path.join(
124    icu_dir, 'icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/localeCanonicalization.txt'))
125
126  # Apply a patch on icu4c/source/data/brkitr/ja.txt because we didn't
127  # cherry-pick the data to CLDR, but only to ICU.
128  # See https://r.android.com/1985568
129  # This patch will not needed when ICU is upgraded to version 71.
130  os.chdir(icu_dir)
131  subprocess.check_call(['patch', '-p0', '-s', '-i',
132                         'tools/data_patches/brkitr/ja.txt.patch'])
133
134  print('Look in %s for new data source files' % icu4c_data_source_dir)
135  sys.exit(0)
136
137
138def rmAndCopyTree(src, dst):
139  if os.path.exists(dst):
140    shutil.rmtree(dst)
141  shutil.copytree(src, dst)
142
143
144if __name__ == '__main__':
145  main()
146