1#!/usr/bin/env python 2# 3# Copyright 2021 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the', help='License'); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an', help='AS IS' BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16import glob 17import json 18import logging 19import os 20from pathlib import Path 21import platform 22import shutil 23import socket 24import subprocess 25import sys 26from threading import currentThread 27 28from time_formatter import TimeFormatter 29 30if sys.version_info[0] == 3: 31 from queue import Queue 32else: 33 from Queue import Queue 34 35from threading import Thread, currentThread 36 37AOSP_ROOT = Path(__file__).absolute().parents[3] 38TOOLS = Path(AOSP_ROOT, "tools") 39EMULATOR_ARTIFACT_PATH = Path(AOSP_ROOT, "tools", "netsim", "emulator_tmp") 40PYTHON_EXE = sys.executable or "python3" 41TARGET_MAP = { 42 "windows": "windows_msvc-x86_64", 43 "windows_x64": "windows_msvc-x86_64", 44 "windows_x86_64": "windows_msvc-x86_64", 45 "linux": "linux-x86_64", 46 "linux_x64": "linux-x86_64", 47 "linux_x86_64": "linux-x86_64", 48 "linux_aarch64": "linux-aarch64", 49 "darwin": "darwin-x86_64", 50 "darwin_x64": "darwin-x86_64", 51 "darwin_x86_64": "darwin-x86_64", 52 "darwin_aarch64": "darwin-aarch64", 53} 54 55AVAILABLE = { 56 "windows_msvc-x86_64": "toolchain-windows_msvc-x86_64.cmake", 57 "linux-x86_64": "toolchain-linux-x86_64.cmake", 58 "darwin-x86_64": "toolchain-darwin-x86_64.cmake", 59 "linux-aarch64": "toolchain-linux-aarch64.cmake", 60 "darwin-aarch64": "toolchain-darwin-aarch64.cmake", 61} 62 63CMAKE = shutil.which( 64 "cmake", 65 path=str( 66 AOSP_ROOT 67 / "prebuilts" 68 / "cmake" 69 / f"{platform.system().lower()}-x86" 70 / "bin" 71 ), 72) 73 74 75def default_target() -> str: 76 """Returns default value for target""" 77 # If Mac M1, the default target should be 'darwin-aarch64' 78 if platform.system() == "Darwin" and platform.machine() == "arm64": 79 return "darwin-aarch64" 80 return platform.system() 81 82 83def create_emulator_artifact_path(): 84 """Refresh or construct EMULATOR_ARTIFACT_PATH""" 85 if EMULATOR_ARTIFACT_PATH.exists(): 86 shutil.rmtree(EMULATOR_ARTIFACT_PATH) 87 EMULATOR_ARTIFACT_PATH.mkdir(exist_ok=True, parents=True) 88 89 90def fetch_build_chaining_artifacts(out_dir, presubmit): 91 """Fetch the Emulator prebuilts for build_bots (go/build_chaining)""" 92 try: 93 out = Path(out_dir) 94 prebuilt_path = out / "prebuilt_cached" / "artifacts" 95 files = glob.glob(str(prebuilt_path / f"*.zip")) 96 for file in files: 97 shutil.copy2(prebuilt_path / file, EMULATOR_ARTIFACT_PATH) 98 except Exception as e: 99 if presubmit: 100 raise e 101 else: 102 logging.warn( 103 f"An error occurred during fetch_build_chaining_artifacts: {e}" 104 ) 105 106 107def binary_extension(filename): 108 """Appends exe extension in case of Windows""" 109 if platform.system() == "Windows": 110 return filename + ".exe" 111 return filename 112 113 114def platform_to_cmake_target(target): 115 """Translates platform to cmake target""" 116 return TARGET_MAP[target.replace("-", "_")] 117 118 119def cmake_toolchain(target) -> str: 120 """Returns the path to the cmake toolchain file.""" 121 return ( 122 AOSP_ROOT 123 / "external" 124 / "qemu" 125 / "android" 126 / "build" 127 / "cmake" 128 / AVAILABLE[TARGET_MAP[target.replace("-", "_")]] 129 ) 130 131 132def is_presubmit(build_id): 133 return build_id.startswith("P") 134 135 136def get_host_and_ip(): 137 """Try to get my hostname and ip address.""" 138 st = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 139 try: 140 st.connect(("10.255.255.255", 1)) 141 my_ip = st.getsockname()[0] 142 except Exception: 143 my_ip = "127.0.0.1" 144 finally: 145 st.close() 146 147 try: 148 hostname = socket.gethostname() 149 except Exception: 150 hostname = "Unkwown" 151 152 return hostname, my_ip 153 154 155class LogBelowLevel(logging.Filter): 156 157 def __init__(self, exclusive_maximum, name=""): 158 super(LogBelowLevel, self).__init__(name) 159 self.max_level = exclusive_maximum 160 161 def filter(self, record): 162 return True if record.levelno < self.max_level else False 163 164 165def config_logging(): 166 logging_handler_out = logging.StreamHandler(sys.stdout) 167 logging_handler_out.setFormatter( 168 TimeFormatter("%(asctime)s %(threadName)s | %(message)s") 169 ) 170 logging_handler_out.setLevel(logging.DEBUG) 171 logging_handler_out.addFilter(LogBelowLevel(logging.WARNING)) 172 173 logging_handler_err = logging.StreamHandler(sys.stderr) 174 logging_handler_err.setFormatter( 175 TimeFormatter("%(asctime)s %(threadName)s | %(message)s") 176 ) 177 logging_handler_err.setLevel(logging.WARNING) 178 179 logging.root = logging.getLogger("build") 180 logging.root.setLevel(logging.INFO) 181 logging.root.addHandler(logging_handler_out) 182 logging.root.addHandler(logging_handler_err) 183 184 currentThread().setName("inf") 185 186 187def log_system_info(): 188 """Log some useful system information.""" 189 version = "{0[0]}.{0[1]}.{0[2]}".format(sys.version_info) 190 hostname, my_ip = get_host_and_ip() 191 192 logging.info( 193 "Hello from %s (%s). I'm a %s build bot", 194 hostname, 195 my_ip, 196 platform.system(), 197 ) 198 logging.info("My uname is: %s", platform.uname()) 199 logging.info( 200 "I'm happy to build the emulator using Python %s (%s)", 201 PYTHON_EXE, 202 version, 203 ) 204 205 206def run(cmd, env, log_prefix, cwd=AOSP_ROOT, throw_on_failure=True): 207 currentThread().setName(log_prefix) 208 cmd_env = os.environ.copy() 209 cmd_env.update(env) 210 is_windows = platform.system() == "Windows" 211 212 cmd = [str(x) for x in cmd] 213 # logging.info("=" * 140) 214 # logging.info(json.dumps(cmd_env, sort_keys=True)) 215 logging.info("%s $> %s", cwd, " ".join(cmd)) 216 # logging.info("=" * 140) 217 218 proc = subprocess.Popen( 219 cmd, 220 stdout=subprocess.PIPE, 221 stderr=subprocess.PIPE, 222 shell=is_windows, # Make sure windows propagates ENV vars properly. 223 cwd=cwd, 224 env=cmd_env, 225 ) 226 227 _log_proc(proc, log_prefix) 228 proc.wait() 229 if proc.returncode != 0 and throw_on_failure: 230 raise Exception("Failed to run %s - %s" % (" ".join(cmd), proc.returncode)) 231 232 233def log_to_queue(q, line): 234 """Logs the output of the given process.""" 235 if q.full(): 236 q.get() 237 238 strip = line.strip() 239 logging.info(strip) 240 q.put(strip) 241 242 243def _reader(pipe, logfn): 244 try: 245 with pipe: 246 for line in iter(pipe.readline, b""): 247 lg = line[:-1] 248 try: 249 lg = lg.decode("utf-8") 250 except Exception as e: 251 logfn("Failed to utf-8 decode line, {}".format(e)) 252 lg = str(lg) 253 logfn(lg.strip()) 254 finally: 255 pass 256 257 258def _log_proc(proc, log_prefix): 259 """Logs the output of the given process.""" 260 q = Queue() 261 for args in [[proc.stdout, logging.info], [proc.stderr, logging.error]]: 262 t = Thread(target=_reader, args=args) 263 t.setName(log_prefix) 264 t.start() 265 266 return q 267