#!/usr/bin/python # # Copyright 2017 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Generates a report on CKI syscall coverage in VTS LTP. This module generates a report on the syscalls in the Android CKI and their coverage in VTS LTP. The coverage report provides, for each syscall in the CKI, the number of enabled and disabled LTP tests for the syscall in VTS. If VTS test output is supplied, the report instead provides the number of disabled, skipped, failing, and passing tests for each syscall. Assumptions are made about the structure of files in LTP source and the naming convention. """ import argparse import os.path import re import sys import xml.etree.ElementTree as ET sys.path.append(os.path.join(os.environ["ANDROID_BUILD_TOP"], "bionic/libc/tools")) import gensyscalls sys.path.append(os.path.join(os.environ["ANDROID_BUILD_TOP"], "test/vts-testcase/kernel/ltp/configs")) import disabled_tests as vts_disabled import stable_tests as vts_stable bionic_libc_root = os.path.join(os.environ["ANDROID_BUILD_TOP"], "bionic/libc") class CKI_Coverage(object): """Determines current test coverage of CKI system calls in LTP. Many of the system calls in the CKI are tested by LTP. For a given system call an LTP test may or may not exist, that LTP test may or may not be currently compiling properly for Android, the test may not be stable, the test may not be running due to environment issues or passing. This class looks at various sources of information to determine the current test coverage of system calls in the CKI from LTP. Note that due to some deviations in LTP of tests from the common naming convention there there may be tests that are flagged here as not having coverage when in fact they do. """ LTP_SYSCALL_ROOT = os.path.join(os.environ["ANDROID_BUILD_TOP"], "external/ltp/testcases/kernel/syscalls") DISABLED_IN_LTP_PATH = os.path.join(os.environ["ANDROID_BUILD_TOP"], "external/ltp/android/tools/disabled_tests.txt") ltp_full_set = [] cki_syscalls = [] disabled_in_ltp = [] disabled_in_vts_ltp = vts_disabled.DISABLED_TESTS stable_in_vts_ltp = vts_stable.STABLE_TESTS syscall_tests = {} disabled_tests = {} failing_tests = {} skipped_tests = {} passing_tests = {} test_results = {} def __init__(self, arch): self.arch = arch def load_ltp_tests(self): """Load the list of LTP syscall tests. Load the list of all syscall tests existing in LTP. """ for path, dirs, files in os.walk(self.LTP_SYSCALL_ROOT): for filename in files: basename, ext = os.path.splitext(filename) if ext != ".c": continue self.ltp_full_set.append(basename) def load_ltp_disabled_tests(self): """Load the list of LTP tests not being compiled. The LTP repository in Android contains a list of tests which are not compiled due to incompatibilities with Android. """ with open(self.DISABLED_IN_LTP_PATH) as fp: for line in fp: line = line.strip() if not line: continue test_re = re.compile(r"^(\w+)") test_match = re.match(test_re, line) if not test_match: continue self.disabled_in_ltp.append(test_match.group(1)) def parse_test_results(self, results): """Parse xml from VTS output to collect LTP results. Parse xml to collect pass/fail results for each LTP test. A failure occurs if a test has a result other than pass or fail. Args: results: Path to VTS output XML file. """ tree = ET.parse(results) root = tree.getroot() # find LTP module for module in root.findall("Module"): if module.attrib["name"] != "VtsKernelLtp": continue # find LTP testcase for testcase in module.findall("TestCase"): if testcase.attrib["name"] != "KernelLtpTest": continue # iterate over each LTP test for test in testcase.findall("Test"): test_re = re.compile(r"^syscalls.(\w+)_((32bit)|(64bit))$") test_match = re.match(test_re, test.attrib["name"]) if not test_match: continue test_name = test_match.group(1) if test.attrib["result"] == "pass": self.test_results[test_name] = "pass" elif test.attrib["result"] == "fail": self.test_results[test_name] = "fail" else: print ("Unknown VTS LTP test result for %s is %s" % (test_name, test.attrib["result"])) sys.exit(-1) def ltp_test_special_cases(self, syscall, test): """Detect special cases in syscall to LTP mapping. Most syscall tests in LTP follow a predictable naming convention, but some do not. Detect known special cases. Args: syscall: The name of a syscall. test: The name of a testcase. Returns: A boolean indicating whether the given syscall is tested by the given testcase. """ if syscall == "clock_nanosleep" and test == "clock_nanosleep2_01": return True if syscall == "fadvise" and test.startswith("posix_fadvise"): return True if syscall == "futex" and test.startswith("futex_"): return True if syscall == "inotify_add_watch" or syscall == "inotify_rm_watch": test_re = re.compile(r"^inotify\d+$") if re.match(test_re, test): return True if syscall == "newfstatat": test_re = re.compile(r"^fstatat\d+$") if re.match(test_re, test): return True return False def match_syscalls_to_tests(self, syscalls): """Match syscalls with tests in LTP. Create a mapping from CKI syscalls and tests in LTP. This mapping can largely be determined using a common naming convention in the LTP file hierarchy but there are special cases that have to be taken care of. Args: syscalls: List of syscall structures containing all syscalls in the CKI. """ for syscall in syscalls: if self.arch not in syscall: continue self.cki_syscalls.append(syscall["name"]) self.syscall_tests[syscall["name"]] = [] # LTP does not use the 64 at the end of syscall names for testcases. ltp_syscall_name = syscall["name"] if ltp_syscall_name.endswith("64"): ltp_syscall_name = ltp_syscall_name[0:-2] # Most LTP syscalls have source files for the tests that follow # a naming convention in the regexp below. Exceptions exist though. # For now those are checked for specifically. test_re = re.compile(r"^%s_?0?\d\d?$" % ltp_syscall_name) for test in self.ltp_full_set: if (re.match(test_re, test) or self.ltp_test_special_cases(ltp_syscall_name, test)): # The filenames of the ioctl tests in LTP do not match the name # of the testcase defined in that source, which is what shows # up in VTS. if ltp_syscall_name == "ioctl": test = "ioctl01_02" self.syscall_tests[syscall["name"]].append(test) self.cki_syscalls.sort() def update_test_status(self): """Populate test configuration and output for all CKI syscalls. Go through VTS test configuration and test results (if provided) to populate data for all CKI syscalls. """ for syscall in self.cki_syscalls: self.disabled_tests[syscall] = [] self.skipped_tests[syscall] = [] self.failing_tests[syscall] = [] self.passing_tests[syscall] = [] if not self.syscall_tests[syscall]: continue for test in self.syscall_tests[syscall]: if (test in self.disabled_in_ltp or "syscalls.%s" % test in self.disabled_in_vts_ltp or ("syscalls.%s_32bit" % test not in self.stable_in_vts_ltp and "syscalls.%s_64bit" % test not in self.stable_in_vts_ltp)): self.disabled_tests[syscall].append(test) continue if not self.test_results: continue if test not in self.test_results: self.skipped_tests[syscall].append(test) elif self.test_results[test] == "fail": self.failing_tests[syscall].append(test) elif self.test_results[test] == "pass": self.passing_tests[syscall].append(test) else: print ("Warning - could not resolve test %s status for syscall %s" % (test, syscall)) def output_results(self): """Pretty print the CKI syscall LTP coverage results. Pretty prints a table of the CKI syscall LTP coverage, pointing out syscalls which have no passing tests in VTS LTP. """ if not self.test_results: self.output_limited_results() return count = 0 uncovered = 0 for syscall in self.cki_syscalls: if not count % 20: print ("%25s Disabled Skipped Failing Passing -------------" % "-------------") sys.stdout.write("%25s %s %s %s %s" % (syscall, len(self.disabled_tests[syscall]), len(self.skipped_tests[syscall]), len(self.failing_tests[syscall]), len(self.passing_tests[syscall]))) if not self.passing_tests[syscall]: print " <-- uncovered" uncovered += 1 else: print "" count += 1 print "" print ("Total uncovered syscalls: %s out of %s" % (uncovered, len(self.cki_syscalls))) def output_limited_results(self): """Pretty print the CKI syscall LTP coverage without VTS test results. When no VTS test results are supplied then only the count of enabled and disabled LTP tests may be shown. """ count = 0 uncovered = 0 for syscall in self.cki_syscalls: if not count % 20: print ("%25s Disabled Enabled -------------" % "-------------") sys.stdout.write("%25s %s %s" % (syscall, len(self.disabled_tests[syscall]), len(self.syscall_tests[syscall]) - len(self.disabled_tests[syscall]))) if (len(self.syscall_tests[syscall]) - len(self.disabled_tests[syscall]) <= 0): print " <-- uncovered" uncovered += 1 else: print "" count += 1 print "" print ("Total uncovered syscalls: %s out of %s" % (uncovered, len(self.cki_syscalls))) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Output list of system calls " "in the Common Kernel Interface and their VTS LTP coverage. If VTS " "test output is supplied, output includes system calls which have " "tests in VTS LTP, but the tests are skipped or are failing.") parser.add_argument("arch", help="architecture of Android platform") parser.add_argument("-l", action="store_true", help="list CKI syscalls only, without coverage") parser.add_argument("-r", "--results", help="path to VTS test_result.xml") args = parser.parse_args() cki = gensyscalls.SysCallsTxtParser() cki.parse_file(os.path.join(bionic_libc_root, "SYSCALLS.TXT")) cki.parse_file(os.path.join(bionic_libc_root, "SECCOMP_WHITELIST.TXT")) if args.l: for syscall in cki.syscalls: if args.arch in syscall: print syscall["name"] exit(0) cki_cov = CKI_Coverage(args.arch) cki_cov.load_ltp_tests() cki_cov.load_ltp_disabled_tests() if args.results: cki_cov.parse_test_results(args.results) cki_cov.match_syscalls_to_tests(cki.syscalls) cki_cov.update_test_status() beta_string = ("*** WARNING: This script is still in development and may\n" "*** report both false positives and negatives.") print beta_string cki_cov.output_results() print beta_string