#!/usr/bin/env python # # Copyright 2021 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the', help='License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an', help='AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import glob import json import logging import os from pathlib import Path import platform import shutil import socket import subprocess import sys from threading import currentThread from time_formatter import TimeFormatter if sys.version_info[0] == 3: from queue import Queue else: from Queue import Queue from threading import Thread, currentThread AOSP_ROOT = Path(__file__).absolute().parents[3] TOOLS = Path(AOSP_ROOT, "tools") EMULATOR_ARTIFACT_PATH = Path(AOSP_ROOT, "tools", "netsim", "emulator_tmp") PYTHON_EXE = sys.executable or "python3" TARGET_MAP = { "windows": "windows_msvc-x86_64", "windows_x64": "windows_msvc-x86_64", "windows_x86_64": "windows_msvc-x86_64", "linux": "linux-x86_64", "linux_x64": "linux-x86_64", "linux_x86_64": "linux-x86_64", "linux_aarch64": "linux-aarch64", "darwin": "darwin-x86_64", "darwin_x64": "darwin-x86_64", "darwin_x86_64": "darwin-x86_64", "darwin_aarch64": "darwin-aarch64", } AVAILABLE = { "windows_msvc-x86_64": "toolchain-windows_msvc-x86_64.cmake", "linux-x86_64": "toolchain-linux-x86_64.cmake", "darwin-x86_64": "toolchain-darwin-x86_64.cmake", "linux-aarch64": "toolchain-linux-aarch64.cmake", "darwin-aarch64": "toolchain-darwin-aarch64.cmake", } CMAKE = shutil.which( "cmake", path=str( AOSP_ROOT / "prebuilts" / "cmake" / f"{platform.system().lower()}-x86" / "bin" ), ) def default_target() -> str: """Returns default value for target""" # If Mac M1, the default target should be 'darwin-aarch64' if platform.system() == "Darwin" and platform.machine() == "arm64": return "darwin-aarch64" return platform.system() def create_emulator_artifact_path(): """Refresh or construct EMULATOR_ARTIFACT_PATH""" if EMULATOR_ARTIFACT_PATH.exists(): shutil.rmtree(EMULATOR_ARTIFACT_PATH) EMULATOR_ARTIFACT_PATH.mkdir(exist_ok=True, parents=True) def fetch_build_chaining_artifacts(out_dir, presubmit): """Fetch the Emulator prebuilts for build_bots (go/build_chaining)""" try: out = Path(out_dir) prebuilt_path = out / "prebuilt_cached" / "artifacts" files = glob.glob(str(prebuilt_path / f"*.zip")) for file in files: shutil.copy2(prebuilt_path / file, EMULATOR_ARTIFACT_PATH) except Exception as e: if presubmit: raise e else: logging.warn( f"An error occurred during fetch_build_chaining_artifacts: {e}" ) def binary_extension(filename): """Appends exe extension in case of Windows""" if platform.system() == "Windows": return filename + ".exe" return filename def platform_to_cmake_target(target): """Translates platform to cmake target""" return TARGET_MAP[target.replace("-", "_")] def cmake_toolchain(target) -> str: """Returns the path to the cmake toolchain file.""" return ( AOSP_ROOT / "external" / "qemu" / "android" / "build" / "cmake" / AVAILABLE[TARGET_MAP[target.replace("-", "_")]] ) def is_presubmit(build_id): return build_id.startswith("P") def get_host_and_ip(): """Try to get my hostname and ip address.""" st = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: st.connect(("10.255.255.255", 1)) my_ip = st.getsockname()[0] except Exception: my_ip = "127.0.0.1" finally: st.close() try: hostname = socket.gethostname() except Exception: hostname = "Unkwown" return hostname, my_ip class LogBelowLevel(logging.Filter): def __init__(self, exclusive_maximum, name=""): super(LogBelowLevel, self).__init__(name) self.max_level = exclusive_maximum def filter(self, record): return True if record.levelno < self.max_level else False def config_logging(): logging_handler_out = logging.StreamHandler(sys.stdout) logging_handler_out.setFormatter( TimeFormatter("%(asctime)s %(threadName)s | %(message)s") ) logging_handler_out.setLevel(logging.DEBUG) logging_handler_out.addFilter(LogBelowLevel(logging.WARNING)) logging_handler_err = logging.StreamHandler(sys.stderr) logging_handler_err.setFormatter( TimeFormatter("%(asctime)s %(threadName)s | %(message)s") ) logging_handler_err.setLevel(logging.WARNING) logging.root = logging.getLogger("build") logging.root.setLevel(logging.INFO) logging.root.addHandler(logging_handler_out) logging.root.addHandler(logging_handler_err) currentThread().setName("inf") def log_system_info(): """Log some useful system information.""" version = "{0[0]}.{0[1]}.{0[2]}".format(sys.version_info) hostname, my_ip = get_host_and_ip() logging.info( "Hello from %s (%s). I'm a %s build bot", hostname, my_ip, platform.system(), ) logging.info("My uname is: %s", platform.uname()) logging.info( "I'm happy to build the emulator using Python %s (%s)", PYTHON_EXE, version, ) def run(cmd, env, log_prefix, cwd=AOSP_ROOT, throw_on_failure=True): currentThread().setName(log_prefix) cmd_env = os.environ.copy() cmd_env.update(env) is_windows = platform.system() == "Windows" cmd = [str(x) for x in cmd] # logging.info("=" * 140) # logging.info(json.dumps(cmd_env, sort_keys=True)) logging.info("%s $> %s", cwd, " ".join(cmd)) # logging.info("=" * 140) proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=is_windows, # Make sure windows propagates ENV vars properly. cwd=cwd, env=cmd_env, ) _log_proc(proc, log_prefix) proc.wait() if proc.returncode != 0 and throw_on_failure: raise Exception("Failed to run %s - %s" % (" ".join(cmd), proc.returncode)) def log_to_queue(q, line): """Logs the output of the given process.""" if q.full(): q.get() strip = line.strip() logging.info(strip) q.put(strip) def _reader(pipe, logfn): try: with pipe: for line in iter(pipe.readline, b""): lg = line[:-1] try: lg = lg.decode("utf-8") except Exception as e: logfn("Failed to utf-8 decode line, {}".format(e)) lg = str(lg) logfn(lg.strip()) finally: pass def _log_proc(proc, log_prefix): """Logs the output of the given process.""" q = Queue() for args in [[proc.stdout, logging.info], [proc.stderr, logging.error]]: t = Thread(target=_reader, args=args) t.setName(log_prefix) t.start() return q