• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright 2017, The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""
16Utility functions for atest.
17"""
18
19from __future__ import print_function
20
21import itertools
22import logging
23import os
24import re
25import subprocess
26import sys
27try:
28    # If PYTHON2
29    from urllib2 import urlopen
30except ImportError:
31    from urllib.request import urlopen
32
33import constants
34
35_MAKE_CMD = '%s/build/soong/soong_ui.bash' % os.environ.get(
36    constants.ANDROID_BUILD_TOP)
37BUILD_CMD = [_MAKE_CMD, '--make-mode']
38_BASH_RESET_CODE = '\033[0m\n'
39# Arbitrary number to limit stdout for failed runs in _run_limited_output.
40# Reason for its use is that the make command itself has its own carriage
41# return output mechanism that when collected line by line causes the streaming
42# full_output list to be extremely large.
43_FAILED_OUTPUT_LINE_LIMIT = 100
44# Regular expression to match the start of a ninja compile:
45# ex: [ 99% 39710/39711]
46_BUILD_COMPILE_STATUS = re.compile(r'\[\s*(\d{1,3}%\s+)?\d+/\d+\]')
47_BUILD_FAILURE = 'FAILED: '
48
49
50def _capture_fail_section(full_log):
51    """Return the error message from the build output.
52
53    Args:
54        full_log: List of strings representing full output of build.
55
56    Returns:
57        capture_output: List of strings that are build errors.
58    """
59    am_capturing = False
60    capture_output = []
61    for line in full_log:
62        if am_capturing and _BUILD_COMPILE_STATUS.match(line):
63            break
64        if am_capturing or line.startswith(_BUILD_FAILURE):
65            capture_output.append(line)
66            am_capturing = True
67            continue
68    return capture_output
69
70
71def _run_limited_output(cmd, env_vars=None):
72    """Runs a given command and streams the output on a single line in stdout.
73
74    Args:
75        cmd: A list of strings representing the command to run.
76        env_vars: Optional arg. Dict of env vars to set during build.
77
78    Raises:
79        subprocess.CalledProcessError: When the command exits with a non-0
80            exitcode.
81    """
82    # Send stderr to stdout so we only have to deal with a single pipe.
83    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
84                            stderr=subprocess.STDOUT, env=env_vars)
85    sys.stdout.write('\n')
86    # Determine the width of the terminal. We'll need to clear this many
87    # characters when carriage returning.
88    _, term_width = os.popen('stty size', 'r').read().split()
89    term_width = int(term_width)
90    white_space = " " * int(term_width)
91    full_output = []
92    while proc.poll() is None:
93        line = proc.stdout.readline()
94        # Readline will often return empty strings.
95        if not line:
96            continue
97        full_output.append(line.decode('utf-8'))
98        # Trim the line to the width of the terminal.
99        # Note: Does not handle terminal resizing, which is probably not worth
100        #       checking the width every loop.
101        if len(line) >= term_width:
102            line = line[:term_width - 1]
103        # Clear the last line we outputted.
104        sys.stdout.write('\r%s\r' % white_space)
105        sys.stdout.write('%s' % line.strip())
106        sys.stdout.flush()
107    # Reset stdout (on bash) to remove any custom formatting and newline.
108    sys.stdout.write(_BASH_RESET_CODE)
109    sys.stdout.flush()
110    # Wait for the Popen to finish completely before checking the returncode.
111    proc.wait()
112    if proc.returncode != 0:
113        # Parse out the build error to output.
114        output = _capture_fail_section(full_output)
115        if not output:
116            output = full_output
117        if len(output) >= _FAILED_OUTPUT_LINE_LIMIT:
118            output = output[-_FAILED_OUTPUT_LINE_LIMIT:]
119        output = 'Output (may be trimmed):\n%s' % ''.join(output)
120        raise subprocess.CalledProcessError(proc.returncode, cmd, output)
121
122
123def build(build_targets, verbose=False, env_vars=None):
124    """Shell out and make build_targets.
125
126    Args:
127        build_targets: A set of strings of build targets to make.
128        verbose: Optional arg. If True output is streamed to the console.
129                 If False, only the last line of the build output is outputted.
130        env_vars: Optional arg. Dict of env vars to set during build.
131
132    Returns:
133        Boolean of whether build command was successful, True if nothing to
134        build.
135    """
136    if not build_targets:
137        logging.debug('No build targets, skipping build.')
138        return True
139    full_env_vars = os.environ.copy()
140    if env_vars:
141        full_env_vars.update(env_vars)
142    print('\n%s\n%s' % (colorize("Building Dependencies...", constants.CYAN),
143                        ', '.join(build_targets)))
144    logging.debug('Building Dependencies: %s', ' '.join(build_targets))
145    cmd = BUILD_CMD + list(build_targets)
146    logging.debug('Executing command: %s', cmd)
147    try:
148        if verbose:
149            subprocess.check_call(cmd, stderr=subprocess.STDOUT,
150                                  env=full_env_vars)
151        else:
152            # TODO: Save output to a log file.
153            _run_limited_output(cmd, env_vars=full_env_vars)
154        logging.info('Build successful')
155        return True
156    except subprocess.CalledProcessError as err:
157        logging.error('Error building: %s', build_targets)
158        if err.output:
159            logging.error(err.output)
160        return False
161
162
163def _can_upload_to_result_server():
164    """Return True if we can talk to result server."""
165    # TODO: Also check if we have a slow connection to result server.
166    if constants.RESULT_SERVER:
167        try:
168            urlopen(constants.RESULT_SERVER,
169                    timeout=constants.RESULT_SERVER_TIMEOUT).close()
170            return True
171        # pylint: disable=broad-except
172        except Exception as err:
173            logging.debug('Talking to result server raised exception: %s', err)
174    return False
175
176
177def get_result_server_args():
178    """Return list of args for communication with result server."""
179    if _can_upload_to_result_server():
180        return constants.RESULT_SERVER_ARGS
181    return []
182
183
184def sort_and_group(iterable, key):
185    """Sort and group helper function."""
186    return itertools.groupby(sorted(iterable, key=key), key=key)
187
188
189def is_test_mapping(args):
190    """Check if the atest command intends to run tests in test mapping.
191
192    When atest runs tests in test mapping, it must have at most one test
193    specified. If a test is specified, it must be started with  `:`,
194    which means the test value is a test group name in TEST_MAPPING file, e.g.,
195    `:postsubmit`.
196
197    If any test mapping options is specified, the atest command must also be
198    set to run tests in test mapping files.
199
200    Args:
201        args: arg parsed object.
202
203    Returns:
204        True if the args indicates atest shall run tests in test mapping. False
205        otherwise.
206    """
207    return (
208        args.test_mapping or
209        args.include_subdirs or
210        not args.tests or
211        (len(args.tests) == 1 and args.tests[0][0] == ':'))
212
213
214def _has_colors(stream):
215    """Check the the output stream is colorful.
216
217    Args:
218        stream: The standard file stream.
219
220    Returns:
221        True if the file stream can interpreter the ANSI color code.
222    """
223    # Following from Python cookbook, #475186
224    if not hasattr(stream, "isatty"):
225        return False
226    if not stream.isatty():
227        # Auto color only on TTYs
228        return False
229    try:
230        import curses
231        curses.setupterm()
232        return curses.tigetnum("colors") > 2
233    # pylint: disable=broad-except
234    except Exception as err:
235        logging.debug('Checking colorful raised exception: %s', err)
236        return False
237
238
239def colorize(text, color, highlight=False):
240    """ Convert to colorful string with ANSI escape code.
241
242    Args:
243        text: A string to print.
244        color: ANSI code shift for colorful print. They are defined
245               in constants_default.py.
246        highlight: True to print with highlight.
247
248    Returns:
249        Colorful string with ANSI escape code.
250    """
251    clr_pref = '\033[1;'
252    clr_suff = '\033[0m'
253    has_colors = _has_colors(sys.stdout)
254    if has_colors:
255        if highlight:
256            ansi_shift = 40 + color
257        else:
258            ansi_shift = 30 + color
259        clr_str = "%s%dm%s%s" % (clr_pref, ansi_shift, text, clr_suff)
260    else:
261        clr_str = text
262    return clr_str
263
264
265def colorful_print(text, color, highlight=False, auto_wrap=True):
266    """Print out the text with color.
267
268    Args:
269        text: A string to print.
270        color: ANSI code shift for colorful print. They are defined
271               in constants_default.py.
272        highlight: True to print with highlight.
273        auto_wrap: If True, Text wraps while print.
274    """
275    output = colorize(text, color, highlight)
276    if auto_wrap:
277        print(output)
278    else:
279        print(output, end="")
280
281
282def is_external_run():
283    """Check is external run or not.
284
285    Returns:
286        True if this is an external run, False otherwise.
287    """
288    try:
289        output = subprocess.check_output(['git', 'config', '--get', 'user.email'],
290                                         universal_newlines=True)
291        if output and output.strip().endswith(constants.INTERNAL_EMAIL):
292            return False
293    except OSError:
294        # OSError can be raised when running atest_unittests on a host
295        # without git being set up.
296        # This happens before atest._configure_logging is called to set up
297        # logging. Therefore, use print to log the error message, instead of
298        # logging.debug.
299        print('Unable to determine if this is an external run, git is not found.')
300    except subprocess.CalledProcessError:
301        print('Unable to determine if this is an external run, email is not '
302              'found in git config.')
303    return True
304
305
306def print_data_collection_notice():
307    """Print the data collection notice."""
308    anonymous = ''
309    user_type = 'INTERNAL'
310    if is_external_run():
311        anonymous = ' anonymous'
312        user_type = 'EXTERNAL'
313    notice = ('  We collect%s usage statistics in accordance with our Content '
314              'Licenses (%s), Contributor License Agreement (%s), Privacy '
315              'Policy (%s) and Terms of Service (%s).'
316             ) % (anonymous,
317                  constants.CONTENT_LICENSES_URL,
318                  constants.CONTRIBUTOR_AGREEMENT_URL[user_type],
319                  constants.PRIVACY_POLICY_URL,
320                  constants.TERMS_SERVICE_URL
321                 )
322    print('\n==================')
323    colorful_print("Notice:", constants.RED)
324    colorful_print("%s" % notice, constants.GREEN)
325    print('==================\n')
326