xref: /tools/asuite/aidegen/lib/common_util.py

#!/usr/bin/env python3
#
# Copyright 2018 - The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""common_util

This module has a collection of functions that provide helper functions to
other modules.
"""

import fnmatch
import inspect
import json
import logging
import os
import re
import sys
import time
import xml.dom.minidom
import zipfile

from functools import partial
from functools import wraps
from xml.etree import ElementTree

from aidegen import constant
from aidegen.lib import errors
from atest import atest_utils
from atest import constants
from atest import module_info


COLORED_INFO = partial(atest_utils.colorize, color=constants.MAGENTA)
COLORED_PASS = partial(atest_utils.colorize, color=constants.GREEN)
COLORED_FAIL = partial(atest_utils.colorize, color=constants.RED)
FAKE_MODULE_ERROR = '{} is a fake module.'
OUTSIDE_ROOT_ERROR = '{} is outside android root.'
PATH_NOT_EXISTS_ERROR = 'The path {} doesn\'t exist.'
NO_MODULE_DEFINED_ERROR = 'No modules defined at {}.'
_REBUILD_MODULE_INFO = '%s We should rebuild module-info.json file for it.'
_ENVSETUP_NOT_RUN = ('Please run "source build/envsetup.sh" and "lunch" before '
                     'running aidegen.')
_LOG_FORMAT = '%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: %(message)s'
_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
_ARG_IS_NULL_ERROR = "{0}.{1}: argument '{2}' is null."
_ARG_TYPE_INCORRECT_ERROR = "{0}.{1}: argument '{2}': type is {3}, must be {4}."
_IDE_UNDEFINED = constant.IDE_DICT[constant.IDE_UNDEFINED]
_IDE_INTELLIJ = constant.IDE_DICT[constant.IDE_INTELLIJ]
_IDE_CLION = constant.IDE_DICT[constant.IDE_CLION]
_IDE_VSCODE = constant.IDE_DICT[constant.IDE_VSCODE]


def time_logged(func=None, *, message='', maximum=1):
    """Decorate a function to find out how much time it spends.

    Args:
        func: a function is to be calculated its spending time.
        message: the message the decorated function wants to show.
        maximum: an integer, minutes. If time exceeds the maximum time show
                 message, otherwise doesn't.

    Returns:
        The wrapper function.
    """
    if func is None:
        return partial(time_logged, message=message, maximum=maximum)

    @wraps(func)
    def wrapper(*args, **kwargs):
        """A wrapper function."""

        start = time.time()
        try:
            return func(*args, **kwargs)
        finally:
            timestamp = time.time() - start
            logging.debug('{}.{} takes: {:.2f}s'.format(
                func.__module__, func.__name__, timestamp))
            if message and timestamp > maximum * 60:
                print(message)

    return wrapper


def get_related_paths(atest_module_info, target=None):
    """Get the relative and absolute paths of target from module-info.

    Args:
        atest_module_info: A ModuleInfo instance.
        target: A string user input from command line. It could be several cases
                such as:
                1. Module name, e.g. Settings
                2. Module path, e.g. packages/apps/Settings
                3. Relative path, e.g. ../../packages/apps/Settings
                4. Current directory, e.g. '.' or no argument
                5. An empty string, which added by AIDEGen, used for generating
                   the iml files for the whole Android repo tree.
                   e.g.
                   1. ~/aosp$ aidegen
                   2. ~/aosp/frameworks/base$ aidegen -a
                6. An absolute path, e.g. /usr/local/home/test/aosp

    Return:
        rel_path: The relative path of a module, return None if no matching
                  module found.
        abs_path: The absolute path of a module, return None if no matching
                  module found.
    """
    rel_path = None
    abs_path = None
    # take the command 'aidegen .' as 'aidegen'.
    if target == '.':
        target = None
    if target:
        # For the case of whole Android repo tree.
        if target == constant.WHOLE_ANDROID_TREE_TARGET:
            rel_path = ''
            abs_path = get_android_root_dir()
        # User inputs an absolute path.
        elif os.path.isabs(target):
            abs_path = target
            rel_path = os.path.relpath(abs_path, get_android_root_dir())
        # User inputs a module name.
        elif atest_module_info.is_module(target):
            paths = atest_module_info.get_paths(target)
            if paths:
                rel_path = paths[0].strip(os.sep)
                abs_path = os.path.join(get_android_root_dir(), rel_path)
        # User inputs a module path or a relative path of android root folder.
        elif (atest_module_info.get_module_names(target)
              or os.path.isdir(os.path.join(get_android_root_dir(), target))):
            rel_path = target.strip(os.sep)
            abs_path = os.path.join(get_android_root_dir(), rel_path)
        # User inputs a relative path of current directory.
        else:
            abs_path = os.path.abspath(os.path.join(os.getcwd(), target))
            rel_path = os.path.relpath(abs_path, get_android_root_dir())
    else:
        # User doesn't input.
        abs_path = os.getcwd()
        if is_android_root(abs_path):
            rel_path = ''
        else:
            rel_path = os.path.relpath(abs_path, get_android_root_dir())
    return rel_path, abs_path


def is_target_android_root(atest_module_info, targets):
    """Check if any target is the Android root path.

    Args:
        atest_module_info: A ModuleInfo instance contains data of
                           module-info.json.
        targets: A list of target modules or project paths from user input.

    Returns:
        True if target is Android root, otherwise False.
    """
    for target in targets:
        _, abs_path = get_related_paths(atest_module_info, target)
        if is_android_root(abs_path):
            return True
    return False


def is_android_root(abs_path):
    """Check if an absolute path is the Android root path.

    Args:
        abs_path: The absolute path of a module.

    Returns:
        True if abs_path is Android root, otherwise False.
    """
    return abs_path == get_android_root_dir()


def has_build_target(atest_module_info, rel_path):
    """Determine if a relative path contains buildable module.

    Args:
        atest_module_info: A ModuleInfo instance contains data of
                           module-info.json.
        rel_path: The module path relative to android root.

    Returns:
        True if the relative path contains a build target, otherwise false.
    """
    return any(
        is_source_under_relative_path(mod_path, rel_path)
        for mod_path in atest_module_info.path_to_module_info)


def _check_modules(atest_module_info, targets, raise_on_lost_module=True):
    """Check if all targets are valid build targets.

    Args:
        atest_module_info: A ModuleInfo instance contains data of
                           module-info.json.
        targets: A list of target modules or project paths from user input.
                When locating the path of the target, given a matched module
                name has priority over path. Below is the priority of locating a
                target:
                1. Module name, e.g. Settings
                2. Module path, e.g. packages/apps/Settings
                3. Relative path, e.g. ../../packages/apps/Settings
                4. Current directory, e.g. '.' or no argument
        raise_on_lost_module: A boolean, pass to _check_module to determine if
                ProjectPathNotExistError or NoModuleDefinedInModuleInfoError
                should be raised.

    Returns:
        True if any _check_module return flip the True/False.
    """
    for target in targets:
        if not check_module(atest_module_info, target, raise_on_lost_module):
            return False
    return True


def check_module(atest_module_info, target, raise_on_lost_module=True):
    """Check if a target is valid or it's a path containing build target.

    Args:
        atest_module_info: A ModuleInfo instance contains the data of
                module-info.json.
        target: A target module or project path from user input.
                When locating the path of the target, given a matched module
                name has priority over path. Below is the priority of locating a
                target:
                1. Module name, e.g. Settings
                2. Module path, e.g. packages/apps/Settings
                3. Relative path, e.g. ../../packages/apps/Settings
                4. Current directory, e.g. . or no argument
                5. An absolute path, e.g. /usr/local/home/test/aosp
        raise_on_lost_module: A boolean, handles if ProjectPathNotExistError or
                NoModuleDefinedInModuleInfoError should be raised.

    Returns:
        1. If there is no error _check_module always return True.
        2. If there is an error,
            a. When raise_on_lost_module is False, _check_module will raise the
               error.
            b. When raise_on_lost_module is True, _check_module will return
               False if module's error is ProjectPathNotExistError or
               NoModuleDefinedInModuleInfoError else raise the error.

    Raises:
        Raise ProjectPathNotExistError and NoModuleDefinedInModuleInfoError only
        when raise_on_lost_module is True, others don't subject to the limit.
        The rules for raising exceptions:
        1. Absolute path of a module is None -> FakeModuleError
        2. Module doesn't exist in repo root -> ProjectOutsideAndroidRootError
        3. The given absolute path is not a dir -> ProjectPathNotExistError
        4. If the given abs path doesn't contain any target and not repo root
           -> NoModuleDefinedInModuleInfoError
    """
    rel_path, abs_path = get_related_paths(atest_module_info, target)
    if not abs_path:
        err = FAKE_MODULE_ERROR.format(target)
        logging.error(err)
        raise errors.FakeModuleError(err)
    if not is_source_under_relative_path(abs_path, get_android_root_dir()):
        err = OUTSIDE_ROOT_ERROR.format(abs_path)
        logging.error(err)
        raise errors.ProjectOutsideAndroidRootError(err)
    if not os.path.isdir(abs_path):
        err = PATH_NOT_EXISTS_ERROR.format(rel_path)
        if raise_on_lost_module:
            logging.error(err)
            raise errors.ProjectPathNotExistError(err)
        logging.debug(_REBUILD_MODULE_INFO, err)
        return False
    if (not has_build_target(atest_module_info, rel_path)
            and not is_android_root(abs_path)):
        err = NO_MODULE_DEFINED_ERROR.format(rel_path)
        if raise_on_lost_module:
            logging.error(err)
            raise errors.NoModuleDefinedInModuleInfoError(err)
        logging.debug(_REBUILD_MODULE_INFO, err)
        return False
    return True


def get_abs_path(rel_path):
    """Get absolute path from a relative path.

    Args:
        rel_path: A string, a relative path to get_android_root_dir().

    Returns:
        abs_path: A string, an absolute path starts with
                  get_android_root_dir().
    """
    if not rel_path:
        return get_android_root_dir()
    if is_source_under_relative_path(rel_path, get_android_root_dir()):
        return rel_path
    return os.path.join(get_android_root_dir(), rel_path)


def is_target(src_file, src_file_extensions):
    """Check if src_file is a type of src_file_extensions.

    Args:
        src_file: A string of the file path to be checked.
        src_file_extensions: A list of file types to be checked

    Returns:
        True if src_file is one of the types of src_file_extensions, otherwise
        False.
    """
    return any(src_file.endswith(x) for x in src_file_extensions)


def get_atest_module_info(targets=None):
    """Get the right version of atest ModuleInfo instance.

    The rules:
        1. If targets is None:
           We just want to get an atest ModuleInfo instance.
        2. If targets isn't None:
           Check if the targets don't exist in ModuleInfo, we'll regain a new
           atest ModuleInfo instance by setting force_build=True and call
           _check_modules again. If targets still don't exist, raise exceptions.

    Args:
        targets: A list of targets to be built.

    Returns:
        An atest ModuleInfo instance.
    """
    amodule_info = module_info.ModuleInfo()
    if targets and not _check_modules(
            amodule_info, targets, raise_on_lost_module=False):
        amodule_info = module_info.ModuleInfo(force_build=True)
        _check_modules(amodule_info, targets)
    return amodule_info


def get_blueprint_json_path(file_name):
    """Assemble the path of blueprint json file.

    Args:
        file_name: A string of blueprint json file name.

    Returns:
        The path of json file.
    """
    return os.path.join(get_soong_out_path(), file_name)


def back_to_cwd(func):
    """Decorate a function change directory back to its original one.

    Args:
        func: a function is to be changed back to its original directory.

    Returns:
        The wrapper function.
    """

    @wraps(func)
    def wrapper(*args, **kwargs):
        """A wrapper function."""
        cwd = os.getcwd()
        try:
            return func(*args, **kwargs)
        finally:
            os.chdir(cwd)

    return wrapper


def get_android_out_dir():
    """Get out directory in different conditions.

    Returns:
        Android out directory path.
    """
    android_root_path = get_android_root_dir()
    android_out_dir = os.getenv(constants.ANDROID_OUT_DIR)
    out_dir_common_base = os.getenv(constant.OUT_DIR_COMMON_BASE_ENV_VAR)
    android_out_dir_common_base = (os.path.join(
        out_dir_common_base, os.path.basename(android_root_path))
                                   if out_dir_common_base else None)
    return (android_out_dir or android_out_dir_common_base
            or constant.ANDROID_DEFAULT_OUT)


def get_android_root_dir():
    """Get Android root directory.

    If the path doesn't exist show message to ask users to run envsetup.sh and
    lunch first.

    Returns:
        Android root directory path.
    """
    android_root_path = os.environ.get(constants.ANDROID_BUILD_TOP)
    if not android_root_path:
        _show_env_setup_msg_and_exit()
    return android_root_path


def get_aidegen_root_dir():
    """Get AIDEGen root directory.

    Returns:
        AIDEGen root directory path.
    """
    return os.path.join(get_android_root_dir(), constant.AIDEGEN_ROOT_PATH)


def _show_env_setup_msg_and_exit():
    """Show message if users didn't run envsetup.sh and lunch."""
    print(_ENVSETUP_NOT_RUN)
    sys.exit(0)


def get_soong_out_path():
    """Assemble out directory's soong path.

    Returns:
        Out directory's soong path.
    """
    return os.path.join(get_android_root_dir(), get_android_out_dir(), 'soong')


def configure_logging(verbose):
    """Configure the logger.

    Args:
        verbose: A boolean. If true, display DEBUG level logs.
    """
    log_format = _LOG_FORMAT
    datefmt = _DATE_FORMAT
    level = logging.DEBUG if verbose else logging.INFO
    # Clear all handlers to prevent setting level not working.
    logging.getLogger().handlers = []
    logging.basicConfig(level=level, format=log_format, datefmt=datefmt)


def exist_android_bp(abs_path):
    """Check if the Android.bp exists under specific folder.

    Args:
        abs_path: An absolute path string.

    Returns: A boolean, true if the Android.bp exists under the folder,
             otherwise false.
    """
    return os.path.isfile(os.path.join(abs_path, constant.ANDROID_BP))


def exist_android_mk(abs_path):
    """Check if the Android.mk exists under specific folder.

    Args:
        abs_path: An absolute path string.

    Returns: A boolean, true if the Android.mk exists under the folder,
             otherwise false.
    """
    return os.path.isfile(os.path.join(abs_path, constant.ANDROID_MK))


def is_source_under_relative_path(source, relative_path):
    """Check if a source file is a project relative path file.

    Args:
        source: Android source file path.
        relative_path: Relative path of the module.

    Returns:
        True if source file is a project relative path file, otherwise False.
    """
    return re.search(
        constant.RE_INSIDE_PATH_CHECK.format(relative_path), source)


def remove_user_home_path(data):
    """Replace the user home path string with a constant string.

    Args:
        data: A string of xml content or an attributeError of error message.

    Returns:
        A string which replaced the user home path to $USER_HOME$.
    """
    return str(data).replace(os.path.expanduser('~'), constant.USER_HOME)


def io_error_handle(func):
    """Decorates a function of handling io error and raising exception.

    Args:
        func: A function is to be raised exception if writing file failed.

    Returns:
        The wrapper function.
    """

    @wraps(func)
    def wrapper(*args, **kwargs):
        """A wrapper function."""
        try:
            return func(*args, **kwargs)
        except (OSError, IOError) as err:
            print('{0}.{1} I/O error: {2}'.format(
                func.__module__, func.__name__, err))
            raise
    return wrapper


def check_args(**decls):
    """Decorates a function to check its argument types.

    Usage:
        @check_args(name=str, text=str)
        def parse_rule(name, text):
            ...

    Args:
        decls: A dictionary with keys as arguments' names and values as
             arguments' types.

    Returns:
        The wrapper function.
    """

    def decorator(func):
        """A wrapper function."""
        fmodule = func.__module__
        fname = func.__name__
        fparams = inspect.signature(func).parameters

        @wraps(func)
        def decorated(*args, **kwargs):
            """A wrapper function."""
            params = dict(zip(fparams, args))
            for arg_name, arg_type in decls.items():
                try:
                    arg_val = params[arg_name]
                except KeyError:
                    # If arg_name can't be found in function's signature, it
                    # might be a case of a partial function or default
                    # parameters, we'll neglect it.
                    if arg_name not in kwargs:
                        continue
                    arg_val = kwargs.get(arg_name)
                if arg_val is None:
                    raise TypeError(_ARG_IS_NULL_ERROR.format(
                        fmodule, fname, arg_name))
                if not isinstance(arg_val, arg_type):
                    raise TypeError(_ARG_TYPE_INCORRECT_ERROR.format(
                        fmodule, fname, arg_name, type(arg_val), arg_type))
            return func(*args, **kwargs)
        return decorated

    return decorator


@io_error_handle
def dump_json_dict(json_path, data):
    """Dumps a dictionary of data into a json file.

    Args:
        json_path: An absolute json file path string.
        data: A dictionary of data to be written into a json file.
    """
    with open(json_path, 'w', encoding='utf-8') as json_file:
        json.dump(data, json_file, indent=4)


@io_error_handle
def get_json_dict(json_path):
    """Loads a json file from path and convert it into a json dictionary.

    Args:
        json_path: An absolute json file path string.

    Returns:
        A dictionary loaded from the json_path.
    """
    with open(json_path, 'r', encoding='utf-8') as jfile:
        return json.load(jfile)


@io_error_handle
def read_file_line_to_list(file_path):
    """Read a file line by line and write them into a list.

    Args:
        file_path: A string of a file's path.

    Returns:
        A list of the file's content by line.
    """
    files = []
    with open(file_path, 'r', encoding='utf8') as infile:
        for line in infile:
            files.append(line.strip())
    return files


@io_error_handle
def read_file_content(path, encode_type='utf8'):
    """Read file's content.

    Args:
        path: Path of input file.
        encode_type: A string of encoding name, default to UTF-8.

    Returns:
        String: Content of the file.
    """
    with open(path, 'r', encoding=encode_type) as template:
        return template.read()


@io_error_handle
def file_generate(path, content):
    """Generate file from content.

    Args:
        path: Path of target file.
        content: String content of file.
    """
    if not os.path.exists(os.path.dirname(path)):
        os.makedirs(os.path.dirname(path))
    with open(path, 'w', encoding='utf-8') as target:
        target.write(content)


def get_lunch_target():
    """Gets the Android lunch target in current console.

    Returns:
        A json format string of lunch target in current console.
    """
    product = os.environ.get(constant.TARGET_PRODUCT)
    build_variant = os.environ.get(constant.TARGET_BUILD_VARIANT)
    if product and build_variant:
        return json.dumps(
            {constant.LUNCH_TARGET: "-".join([product, build_variant])})
    return None


def get_blueprint_json_files_relative_dict():
    """Gets a dictionary with key: environment variable, value: absolute path.

    Returns:
        A dictionary  with key: environment variable and value: absolute path of
        the file generated by the environment variable.
    """
    data = {}
    root_dir = get_android_root_dir()
    bp_java_path = os.path.join(
        root_dir, get_blueprint_json_path(
            constant.BLUEPRINT_JAVA_JSONFILE_NAME))
    data[constant.GEN_JAVA_DEPS] = bp_java_path
    bp_cc_path = os.path.join(
        root_dir, get_blueprint_json_path(constant.BLUEPRINT_CC_JSONFILE_NAME))
    data[constant.GEN_CC_DEPS] = bp_cc_path
    data[constant.GEN_COMPDB] = os.path.join(get_soong_out_path(),
                                             constant.RELATIVE_COMPDB_PATH,
                                             constant.COMPDB_JSONFILE_NAME)
    data[constant.GEN_RUST] = os.path.join(
        root_dir, get_blueprint_json_path(constant.RUST_PROJECT_JSON))
    return data


def to_pretty_xml(root, indent="  "):
    """Gets pretty xml from an xml.etree.ElementTree root.

    Args:
        root: An element tree root.
        indent: The indent of XML.
    Returns:
        A string of pretty xml.
    """
    xml_string = xml.dom.minidom.parseString(
        ElementTree.tostring(root)).toprettyxml(indent)
    # Remove the xml declaration since IntelliJ doesn't use it.
    xml_string = xml_string.split("\n", 1)[1]
    # Remove the weird newline issue from toprettyxml.
    return os.linesep.join([s for s in xml_string.splitlines() if s.strip()])


def to_boolean(str_bool):
    """Converts a string to a boolean.

    Args:
        str_bool: A string in the expression of boolean type.

    Returns:
        A boolean True if the string is one of ('True', 'true', 'T', 't', '1')
        else False.
    """
    return str_bool and str_bool.lower() in ('true', 't', '1')


def find_git_root(relpath):
    """Finds the parent directory which has a .git folder from the relpath.

    Args:
        relpath: A string of relative path.

    Returns:
        A string of the absolute path which contains a .git, otherwise, none.
    """
    dir_list = relpath.split(os.sep)
    for i in range(len(dir_list), 0, -1):
        real_path = os.path.join(get_android_root_dir(),
                                 os.sep.join(dir_list[:i]),
                                 constant.GIT_FOLDER_NAME)
        if os.path.exists(real_path):
            return os.path.dirname(real_path)
    logging.warning('%s can\'t find its .git folder.', relpath)
    return None


def determine_language_ide(lang, ide, jlist=None, clist=None, rlist=None):
    """Determines the language and IDE by the input language and IDE arguments.

    If IDE and language are undefined, the priority of the language is:
      1. Java
      2. C/C++
      3. Rust

    Args:
        lang: A character represents the input language.
        ide: A character represents the input IDE.
        jlist: A list of Android Java projects, the default value is None.
        clist: A list of Android C/C++ projects, the default value is None.
        rlist: A list of Android Rust projects, the default value is None.

    Returns:
        A tuple of the determined language and IDE name strings.
    """
    if ide == _IDE_UNDEFINED and lang == constant.LANG_UNDEFINED:
        if jlist:
            lang = constant.LANG_JAVA
        elif clist:
            lang = constant.LANG_CC
        elif rlist:
            lang = constant.LANG_RUST
    if lang in (constant.LANG_UNDEFINED, constant.LANG_JAVA):
        if ide == _IDE_UNDEFINED:
            ide = _IDE_INTELLIJ
        lang = constant.LANG_JAVA
        if constant.IDE_NAME_DICT[ide] == constant.IDE_CLION:
            lang = constant.LANG_CC
    elif lang == constant.LANG_CC:
        if ide == _IDE_UNDEFINED:
            ide = _IDE_CLION
        if constant.IDE_NAME_DICT[ide] == constant.IDE_INTELLIJ:
            lang = constant.LANG_JAVA
    elif lang == constant.LANG_RUST:
        ide = _IDE_VSCODE
    return constant.LANGUAGE_NAME_DICT[lang], constant.IDE_NAME_DICT[ide]


def check_java_or_kotlin_file_exists(abs_path):
    """Checks if any Java or Kotlin files exist in an abs_path directory.

    Args:
        abs_path: A string of absolute path of a directory to be checked.

    Returns:
        True if any Java or Kotlin files exist otherwise False.
    """
    for _, _, filenames in os.walk(abs_path):
        for extension in (constant.JAVA_FILES, constant.KOTLIN_FILES):
            if fnmatch.filter(filenames, extension):
                return True
    return False


@io_error_handle
def unzip_file(src, dest):
    """Unzips the source zip file and extract it to the destination directory.

    Args:
        src: A string of the file to be unzipped.
        dest: A string of the destination directory to be extracted to.
    """
    with zipfile.ZipFile(src, 'r') as zip_ref:
        zip_ref.extractall(dest)