system/test/gen_coverage.py

#!/usr/bin/env python
#
# Copyright 2018, The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import logging
import json
import os
import shutil
import subprocess
import sys
import webbrowser

from run_host_unit_tests import *
"""
This script is used to generate code coverage results host supported libraries.
The script by default will generate an html report that summarizes the coverage
results of the specified tests. The results can also be browsed to provide a
report of which lines have been traveled upon execution of the binary.

NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as
having been executed 0 times, thus reducing overall coverage.

The steps in order to add coverage support to a new library and its
corrisponding host test are as follows.

1. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the
   source library(s) you want statistics for.
   NOTE: Forgoing this default will cause no coverage data to be generated for
         the source files in the library.

2. Add "clang_coverage_bin" as a default to the host supported test binary that
   excercises the libraries that you covered in step 1.
   NOTE: Forgetting to add this will cause there to be *NO* coverage data
         generated when the binary is run.

3. Add the host test binary name and the files/directories you want coverage
   statistics for to the COVERAGE_TESTS variable defined below. You may add
   individual filenames or a directory to be tested.
   NOTE: Avoid using a / at the beginning of a covered_files entry as this
         breaks how the coverage generator resolves filenames.

TODO: Support generating XML data and printing results to standard out.
"""

COVERAGE_TESTS = [
    {
        "test_name": "net_test_avrcp",
        "covered_files": [
            "packages/modules/Bluetooth/system/profile/avrcp",
        ],
    },
    {
        "test_name": "bluetooth_test_sdp",
        "covered_files": [
            "packages/modules/Bluetooth/system/profile/sdp",
        ],
    },
    {
        "test_name": "rootcanal_test_host",
        "covered_files": [
            "packages/modules/Bluetooth/tools/rootcanal/include",
            "packages/modules/Bluetooth/tools/rootcanal/src",
        ],
    },
    {
        "test_name": "rootcanal-packets_test_host",
        "covered_files": [
            "packages/modules/Bluetooth/tools/rootcanal/packets",
        ],
    },
    {
        "test_name": "bluetooth_test_common",
        "covered_files": [
            "packages/modules/Bluetooth/system/common",
        ],
    },
]

WORKING_DIR = '/tmp/coverage'
SOONG_UI_BASH = 'build/soong/soong_ui.bash'
LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r353983b/bin'
LLVM_MERGE = LLVM_DIR + '/llvm-profdata'
LLVM_COV = LLVM_DIR + '/llvm-cov'


def write_root_html_head(f):
    # Write the header part of the root html file. This was pulled from the
    # page source of one of the generated html files.
    f.write("<!doctype html><html><head>" \
      "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \
      "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \
      "s'></head><body><h2>Coverage Report</h2><h4>Created: " +
      str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) +
      "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \
      "ge.html#interpreting-reports'>here</a> for information about interpreti" \
      "ng this report.</p><div class='centered'><table><tr><td class='column-e" \
      "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \
      "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \
      "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \
      "gion Coverage</td></tr>"
    )


def write_root_html_column(f, covered, count):
    percent = covered * 100.0 / count
    value = "%.2f%% (%d/%d) " % (percent, covered, count)
    color = 'column-entry-yellow'
    if percent == 100:
        color = 'column-entry-green'
    if percent < 80.0:
        color = 'column-entry-red'
    f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>")


def write_root_html_rows(f, tests):
    totals = {
        "functions": {
            "covered": 0,
            "count": 0
        },
        "instantiations": {
            "covered": 0,
            "count": 0
        },
        "lines": {
            "covered": 0,
            "count": 0
        },
        "regions": {
            "covered": 0,
            "count": 0
        }
    }

    # Write the tests with their coverage summaries.
    for test in tests:
        test_name = test['test_name']
        covered_files = test['covered_files']
        json_results = generate_coverage_json(test)
        test_totals = json_results['data'][0]['totals']

        f.write("<tr class='light-row'><td><pre><a href=\'" + os.path.join(test_name, "index.html") + "\'>" +
                test_name + "</a></pre></td>")
        for field_name in ['functions', 'instantiations', 'lines', 'regions']:
            field = test_totals[field_name]
            totals[field_name]['covered'] += field['covered']
            totals[field_name]['count'] += field['count']
            write_root_html_column(f, field['covered'], field['count'])
        f.write("</tr>")

    #Write the totals row.
    f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>")
    for field_name in ['functions', 'instantiations', 'lines', 'regions']:
        field = totals[field_name]
        write_root_html_column(f, field['covered'], field['count'])
    f.write("</tr>")


def write_root_html_tail(f):
    # Pulled from the generated html coverage report.
    f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \
      "/h5></body></html>")


def generate_root_html(tests):
    # Copy the css file from one of the coverage reports.
    source_file = os.path.join(os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css")
    dest_file = os.path.join(WORKING_DIR, "style.css")
    shutil.copy2(source_file, dest_file)

    # Write the root index.html file that sumarizes all the tests.
    f = open(os.path.join(WORKING_DIR, "index.html"), "w")
    write_root_html_head(f)
    write_root_html_rows(f, tests)
    write_root_html_tail(f)


def get_profraw_for_test(test_name):
    test_root = get_native_test_root_or_die()
    test_cmd = os.path.join(os.path.join(test_root, test_name), test_name)
    if not os.path.isfile(test_cmd):
        logging.error('The test ' + test_name + ' does not exist, please compile first')
        sys.exit(1)

    profraw_file_name = test_name + ".profraw"
    profraw_path = os.path.join(WORKING_DIR, os.path.join(test_name, profraw_file_name))
    llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\""

    test_cmd = llvm_env_var + " " + test_cmd
    logging.info('Generating profraw data for ' + test_name)
    logging.debug('cmd: ' + test_cmd)
    if subprocess.call(test_cmd, shell=True) != 0:
        logging.error('Test ' + test_name + ' failed. Please fix the test before generating coverage.')
        sys.exit(1)

    if not os.path.isfile(profraw_path):
        logging.error(
            'Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?')
        sys.exit(1)

    return profraw_file_name


def merge_profraw_data(test_name):
    cmd = []
    cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge "))

    test_working_dir = os.path.join(WORKING_DIR, test_name)
    cmd.append(os.path.join(test_working_dir, test_name + ".profraw"))
    profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

    cmd.append('-o ' + profdata_file)
    logging.info('Combining profraw files into profdata for ' + test_name)
    logging.debug('cmd: ' + " ".join(cmd))
    if subprocess.call(" ".join(cmd), shell=True) != 0:
        logging.error('Failed to merge profraw files for ' + test_name)
        sys.exit(1)


def generate_coverage_html(test):
    COVERAGE_ROOT = '/proc/self/cwd'

    test_name = test['test_name']
    file_list = test['covered_files']

    test_working_dir = os.path.join(WORKING_DIR, test_name)
    test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

    cmd = [
        os.path.join(get_android_root_or_die(), LLVM_COV), "show", "-format=html", "-summary-only",
        "-show-line-counts-or-regions", "-show-instantiation-summary", "-instr-profile=" + test_profdata_file,
        "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"",
        "-output-dir=" + test_working_dir
    ]

    # We have to have one object file not as an argument otherwise we can't specify source files.
    test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
    cmd.append(test_cmd)

    # Filter out the specific files we want coverage for
    for filename in file_list:
        cmd.append(os.path.join(get_android_root_or_die(), filename))

    logging.info('Generating coverage report for ' + test['test_name'])
    logging.debug('cmd: ' + " ".join(cmd))
    if subprocess.call(" ".join(cmd), shell=True) != 0:
        logging.error('Failed to generate coverage for ' + test['test_name'])
        sys.exit(1)


def generate_coverage_json(test):
    COVERAGE_ROOT = '/proc/self/cwd'
    test_name = test['test_name']
    file_list = test['covered_files']

    test_working_dir = os.path.join(WORKING_DIR, test_name)
    test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")

    cmd = [
        os.path.join(get_android_root_or_die(), LLVM_COV),
        "export",
        "-summary-only",
        "-show-region-summary",
        "-instr-profile=" + test_profdata_file,
        "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"",
    ]

    test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
    cmd.append(test_cmd)

    # Filter out the specific files we want coverage for
    for filename in file_list:
        cmd.append(os.path.join(get_android_root_or_die(), filename))

    logging.info('Generating coverage json for ' + test['test_name'])
    logging.debug('cmd: ' + " ".join(cmd))

    json_str = subprocess.check_output(" ".join(cmd), shell=True)
    return json.loads(json_str)


def write_json_summary(test):
    test_name = test['test_name']
    test_working_dir = os.path.join(WORKING_DIR, test_name)
    test_json_summary_file = os.path.join(test_working_dir, test_name + '.json')
    logging.debug('Writing json summary file: ' + test_json_summary_file)
    json_file = open(test_json_summary_file, 'w')
    json.dump(generate_coverage_json(test), json_file)
    json_file.close()


def list_tests():
    for test in COVERAGE_TESTS:
        print "Test Name: " + test['test_name']
        print "Covered Files: "
        for covered_file in test['covered_files']:
            print "  " + covered_file
        print


def main():
    parser = argparse.ArgumentParser(description='Generate code coverage for enabled tests.')
    parser.add_argument(
        '-l',
        '--list-tests',
        action='store_true',
        dest='list_tests',
        help='List all the available tests to be run as well as covered files.')
    parser.add_argument(
      '-a', '--all',
      action='store_true',
      help='Runs all available tests and prints their outputs. If no tests ' \
           'are specified via the -t option all tests will be run.')
    parser.add_argument(
      '-t', '--test',
      dest='tests',
      action='append',
      type=str,
      metavar='TESTNAME',
      default=[],
      help='Specifies a test to be run. Multiple tests can be specified by ' \
           'using this option multiple times. ' \
           'Example: \"gen_coverage.py -t test1 -t test2\"')
    parser.add_argument(
      '-o', '--output',
      type=str,
      metavar='DIRECTORY',
      default='/tmp/coverage',
      help='Specifies the directory to store all files. The directory will be ' \
           'created if it does not exist. Default is \"/tmp/coverage\"')
    parser.add_argument(
        '-s',
        '--skip-html',
        dest='skip_html',
        action='store_true',
        help='Skip opening up the results of the coverage report in a browser.')
    parser.add_argument(
        '-j',
        '--json-file',
        dest='json_file',
        action='store_true',
        help='Write out summary results to json file in test directory.')

    logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(levelname)s %(message)s')
    logging.addLevelName(logging.DEBUG, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG))
    logging.addLevelName(logging.INFO, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO))
    logging.addLevelName(logging.WARNING, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING))
    logging.addLevelName(logging.ERROR, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR))

    args = parser.parse_args()
    logging.debug("Args: " + str(args))

    # Set the working directory
    global WORKING_DIR
    WORKING_DIR = os.path.abspath(args.output)
    logging.debug("Working Dir: " + WORKING_DIR)

    # Print out the list of tests then exit
    if args.list_tests:
        list_tests()
        sys.exit(0)

    # Check to see if a test was specified and if so only generate coverage for
    # that test.
    if len(args.tests) == 0:
        args.all = True

    tests_to_run = []
    for test in COVERAGE_TESTS:
        if args.all or test['test_name'] in args.tests:
            tests_to_run.append(test)
        if test['test_name'] in args.tests:
            args.tests.remove(test['test_name'])

    # Error if a test was specified but doesn't exist.
    if len(args.tests) != 0:
        for test_name in args.tests:
            logging.error('\"' + test_name + '\" was not found in the list of available tests.')
        sys.exit(1)

    # Generate the info for the tests
    for test in tests_to_run:
        logging.info('Getting coverage for ' + test['test_name'])
        get_profraw_for_test(test['test_name'])
        merge_profraw_data(test['test_name'])
        if args.json_file:
            write_json_summary(test)
        generate_coverage_html(test)

    # Generate the root index.html page that sumarizes all of the coverage reports.
    generate_root_html(tests_to_run)

    # Open the results in a browser.
    if not args.skip_html:
        webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html'))


if __name__ == '__main__':
    main()