• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2#
3# Copyright 2021 - The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the',  help='License');
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an',  help='AS IS' BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16import glob
17import json
18import logging
19import os
20from pathlib import Path
21import platform
22import shutil
23import socket
24import subprocess
25import sys
26from threading import currentThread
27
28from time_formatter import TimeFormatter
29
30if sys.version_info[0] == 3:
31  from queue import Queue
32else:
33  from Queue import Queue
34
35from threading import Thread, currentThread
36
37AOSP_ROOT = Path(__file__).absolute().parents[3]
38TOOLS = Path(AOSP_ROOT, "tools")
39EMULATOR_ARTIFACT_PATH = Path(AOSP_ROOT, "tools", "netsim", "emulator_tmp")
40PYTHON_EXE = sys.executable or "python3"
41TARGET_MAP = {
42    "windows": "windows_msvc-x86_64",
43    "windows_x64": "windows_msvc-x86_64",
44    "windows_x86_64": "windows_msvc-x86_64",
45    "linux": "linux-x86_64",
46    "linux_x64": "linux-x86_64",
47    "linux_x86_64": "linux-x86_64",
48    "linux_aarch64": "linux-aarch64",
49    "darwin": "darwin-x86_64",
50    "darwin_x64": "darwin-x86_64",
51    "darwin_x86_64": "darwin-x86_64",
52    "darwin_aarch64": "darwin-aarch64",
53}
54
55AVAILABLE = {
56    "windows_msvc-x86_64": "toolchain-windows_msvc-x86_64.cmake",
57    "linux-x86_64": "toolchain-linux-x86_64.cmake",
58    "darwin-x86_64": "toolchain-darwin-x86_64.cmake",
59    "linux-aarch64": "toolchain-linux-aarch64.cmake",
60    "darwin-aarch64": "toolchain-darwin-aarch64.cmake",
61}
62
63CMAKE = shutil.which(
64    "cmake",
65    path=str(
66        AOSP_ROOT
67        / "prebuilts"
68        / "cmake"
69        / f"{platform.system().lower()}-x86"
70        / "bin"
71    ),
72)
73
74
75def default_target() -> str:
76  """Returns default value for target"""
77  # If Mac M1, the default target should be 'darwin-aarch64'
78  if platform.system() == "Darwin" and platform.machine() == "arm64":
79    return "darwin-aarch64"
80  return platform.system()
81
82
83def create_emulator_artifact_path():
84  """Refresh or construct EMULATOR_ARTIFACT_PATH"""
85  if EMULATOR_ARTIFACT_PATH.exists():
86    shutil.rmtree(EMULATOR_ARTIFACT_PATH)
87  EMULATOR_ARTIFACT_PATH.mkdir(exist_ok=True, parents=True)
88
89
90def fetch_build_chaining_artifacts(out_dir, presubmit):
91  """Fetch the Emulator prebuilts for build_bots (go/build_chaining)"""
92  try:
93    out = Path(out_dir)
94    prebuilt_path = out / "prebuilt_cached" / "artifacts"
95    files = glob.glob(str(prebuilt_path / f"*.zip"))
96    for file in files:
97      shutil.copy2(prebuilt_path / file, EMULATOR_ARTIFACT_PATH)
98  except Exception as e:
99    if presubmit:
100      raise e
101    else:
102      logging.warn(
103          f"An error occurred during fetch_build_chaining_artifacts: {e}"
104      )
105
106
107def binary_extension(filename):
108  """Appends exe extension in case of Windows"""
109  if platform.system() == "Windows":
110    return filename + ".exe"
111  return filename
112
113
114def platform_to_cmake_target(target):
115  """Translates platform to cmake target"""
116  return TARGET_MAP[target.replace("-", "_")]
117
118
119def cmake_toolchain(target) -> str:
120  """Returns the path to the cmake toolchain file."""
121  return (
122      AOSP_ROOT
123      / "external"
124      / "qemu"
125      / "android"
126      / "build"
127      / "cmake"
128      / AVAILABLE[TARGET_MAP[target.replace("-", "_")]]
129  )
130
131
132def is_presubmit(build_id):
133  return build_id.startswith("P")
134
135
136def get_host_and_ip():
137  """Try to get my hostname and ip address."""
138  st = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
139  try:
140    st.connect(("10.255.255.255", 1))
141    my_ip = st.getsockname()[0]
142  except Exception:
143    my_ip = "127.0.0.1"
144  finally:
145    st.close()
146
147  try:
148    hostname = socket.gethostname()
149  except Exception:
150    hostname = "Unkwown"
151
152  return hostname, my_ip
153
154
155class LogBelowLevel(logging.Filter):
156
157  def __init__(self, exclusive_maximum, name=""):
158    super(LogBelowLevel, self).__init__(name)
159    self.max_level = exclusive_maximum
160
161  def filter(self, record):
162    return True if record.levelno < self.max_level else False
163
164
165def config_logging():
166  logging_handler_out = logging.StreamHandler(sys.stdout)
167  logging_handler_out.setFormatter(
168      TimeFormatter("%(asctime)s %(threadName)s | %(message)s")
169  )
170  logging_handler_out.setLevel(logging.DEBUG)
171  logging_handler_out.addFilter(LogBelowLevel(logging.WARNING))
172
173  logging_handler_err = logging.StreamHandler(sys.stderr)
174  logging_handler_err.setFormatter(
175      TimeFormatter("%(asctime)s %(threadName)s | %(message)s")
176  )
177  logging_handler_err.setLevel(logging.WARNING)
178
179  logging.root = logging.getLogger("build")
180  logging.root.setLevel(logging.INFO)
181  logging.root.addHandler(logging_handler_out)
182  logging.root.addHandler(logging_handler_err)
183
184  currentThread().setName("inf")
185
186
187def log_system_info():
188  """Log some useful system information."""
189  version = "{0[0]}.{0[1]}.{0[2]}".format(sys.version_info)
190  hostname, my_ip = get_host_and_ip()
191
192  logging.info(
193      "Hello from %s (%s). I'm a %s build bot",
194      hostname,
195      my_ip,
196      platform.system(),
197  )
198  logging.info("My uname is: %s", platform.uname())
199  logging.info(
200      "I'm happy to build the emulator using Python %s (%s)",
201      PYTHON_EXE,
202      version,
203  )
204
205
206def run(cmd, env, log_prefix, cwd=AOSP_ROOT, throw_on_failure=True):
207  currentThread().setName(log_prefix)
208  cmd_env = os.environ.copy()
209  cmd_env.update(env)
210  is_windows = platform.system() == "Windows"
211
212  cmd = [str(x) for x in cmd]
213  # logging.info("=" * 140)
214  # logging.info(json.dumps(cmd_env, sort_keys=True))
215  logging.info("%s $> %s", cwd, " ".join(cmd))
216  # logging.info("=" * 140)
217
218  proc = subprocess.Popen(
219      cmd,
220      stdout=subprocess.PIPE,
221      stderr=subprocess.PIPE,
222      shell=is_windows,  # Make sure windows propagates ENV vars properly.
223      cwd=cwd,
224      env=cmd_env,
225  )
226
227  _log_proc(proc, log_prefix)
228  proc.wait()
229  if proc.returncode != 0 and throw_on_failure:
230    raise Exception("Failed to run %s - %s" % (" ".join(cmd), proc.returncode))
231
232
233def log_to_queue(q, line):
234  """Logs the output of the given process."""
235  if q.full():
236    q.get()
237
238  strip = line.strip()
239  logging.info(strip)
240  q.put(strip)
241
242
243def _reader(pipe, logfn):
244  try:
245    with pipe:
246      for line in iter(pipe.readline, b""):
247        lg = line[:-1]
248        try:
249          lg = lg.decode("utf-8")
250        except Exception as e:
251          logfn("Failed to utf-8 decode line, {}".format(e))
252          lg = str(lg)
253        logfn(lg.strip())
254  finally:
255    pass
256
257
258def _log_proc(proc, log_prefix):
259  """Logs the output of the given process."""
260  q = Queue()
261  for args in [[proc.stdout, logging.info], [proc.stderr, logging.error]]:
262    t = Thread(target=_reader, args=args)
263    t.setName(log_prefix)
264    t.start()
265
266  return q
267