1#! /usr/bin/env python 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18# Parses the output of parse_ltp_{make,make_install} and generates a 19# corresponding Android.mk. 20# 21# This process is split into two steps so this second step can later be replaced 22# with an Android.bp generator. 23 24from __future__ import print_function 25 26"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites""" 27 28import os 29import argparse 30import os.path 31import sys 32 33def scan_tests(ltp_root, suite): 34 ''' Find all tests that are run as part of given test suite in LTP. 35 36 Args: 37 ltp_root: Path ot the LTP project. 38 suite: Name of testsuite. 39 40 Returns: 41 List of tests names that are run as part of the given test suite. 42 ''' 43 44 tests = [] 45 if not suite: 46 return tests 47 48 runtest_dir = ltp_root + os.path.sep + 'runtest' 49 test_suiteFile = runtest_dir + os.path.sep + suite 50 if not os.path.isfile(test_suiteFile): 51 print ('No tests defined for suite {}',format(suite)) 52 return tests 53 54 with open(test_suiteFile) as f: 55 for line in f: 56 line = line.strip() 57 if line and not line.startswith('#'): 58 tests.append(line.split()[0]) 59 f.close() 60 return tests 61 62def scan_test_suites(ltp_root, scenario): 63 ''' Find all testuites and tests run as part of given LTP scenario 64 65 Args: 66 ltp_root: Path to the LTP project. 67 scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts" 68 69 Returns: 70 List of testsuite names that are run as part of given scenario (e.g. vts). 71 If scenario is not specified, return all testsuites. 72 ''' 73 74 runtest_dir = ltp_root + os.path.sep + 'runtest' 75 if not os.path.isdir(runtest_dir): 76 print ('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root)) 77 sys.exit(2) 78 79 test_suites = [] 80 if scenario: 81 scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario 82 if not os.path.isfile(scenarioFile): 83 return test_suites 84 with open(scenarioFile) as f: 85 for line in f: 86 line = line.strip() 87 if not line.startswith('#'): 88 test_suites.append(line) 89 return test_suites 90 91 runtest_dir = ltp_root + os.path.sep + 'runtest' 92 test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)] 93 94 return test_suites 95 96def scan_ltp(ltp_root, scenario): 97 ''' scan LTP project and return all tests and testuites present. 98 99 Args: 100 ltp_root: Path to the LTP project. 101 scenario: specific scenario we want to scan. e.g. vts 102 103 Returns: 104 Dictionary of all LTP test names keyed by testsuite they are run as part of. 105 E.g. 106 { 107 'mem': ['oom01', 'mem02',...], 108 'syscalls': ['open01', 'read02',...], 109 ... 110 } 111 ''' 112 113 if not os.path.isdir(ltp_root): 114 print ('ltp_root {} does not exist'.format(ltp_root)) 115 sys.exit(1) 116 117 test_suites = scan_test_suites(ltp_root, scenario) 118 if not test_suites: 119 print ('No Testsuites found for scenario {}'.format(scenario)) 120 sys.exit(3) 121 122 ltp_tests = {} 123 for suite in test_suites: 124 ltp_tests[suite] = scan_tests(ltp_root, suite) 125 return ltp_tests 126 127def show_diff(ltp_tests_1, ltp_tests_2): 128 ''' Find and print diff between testcases in 2 LTP project checkouts. 129 130 Args: 131 ltp_tests_1: dictionary of tests keyed by test suite names 132 ltp_tests_2: dictionary of tests keyed by test suite names 133 ''' 134 DEFAULT_WIDTH = 8 135 test_suites1 = set(sorted(ltp_tests_1.keys())) 136 test_suites2 = set(sorted(ltp_tests_2.keys())) 137 138 # Generate lists of deleted, added and common test suites between 139 # LTP1 & LTP2 140 deleted_test_suites = sorted(test_suites1.difference(test_suites2)) 141 added_test_suites = sorted(test_suites2.difference(test_suites1)) 142 common_test_suites = sorted(test_suites1.intersection(test_suites2)) 143 144 deleted_tests = [] 145 added_tests = [] 146 for suite in common_test_suites: 147 tests1 = set(sorted(ltp_tests_1[suite])) 148 tests2 = set(sorted(ltp_tests_2[suite])) 149 150 exclusive_test1 = tests1.difference(tests2) 151 exclusive_test2 = tests2.difference(tests1) 152 for e in exclusive_test1: 153 deleted_tests.append(suite + '.' + e) 154 for e in exclusive_test2: 155 added_tests.append(suite + '.' + e) 156 157 # find the maximum width of a test suite name or test name 158 # we have to print to decide the alignment. 159 if not deleted_test_suites: 160 width_suites = DEFAULT_WIDTH 161 else: 162 width_suites = max([len(x) for x in deleted_test_suites]) 163 164 if not deleted_tests: 165 width_tests = DEFAULT_WIDTH 166 else: 167 width_tests = max([len(x) for x in deleted_tests]) 168 width = max(width_suites, width_tests); 169 170 # total rows we have to print 171 total_rows = max(len(deleted_test_suites), len(added_test_suites)) 172 if total_rows > 0: 173 print ('{:*^{len}}'.format(' Tests Suites ', len=width*2+2)) 174 print ('{:>{len}} {:>{len}}'.format('Deleted', 'Added', len=width)) 175 for i in range(total_rows): 176 print ('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), 177 '' if i >= len(added_test_suites) else str(added_test_suites[i]), len=width)) 178 179 print('') 180 # total rows we have to print 181 total_rows = max(len(deleted_tests), len(added_tests)) 182 if total_rows: 183 print ('{:*^{len}}'.format(' Tests ', len=width*2+2)) 184 print ('{:>{len}} {:>{len}}'.format('Deleted', 'Added', len=width)) 185 for i in range(total_rows): 186 print ('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_tests) else str(deleted_tests[i]), 187 '' if i >= len(added_tests) else str(added_tests[i]), len=width)) 188def main(): 189 arg_parser = argparse.ArgumentParser( 190 description='Diff 2 LTP projects for supported test cases') 191 arg_parser.add_argument('--ltp-root1', 192 dest='ltp_root1', 193 required=True, 194 help="LTP Root Directory before merge") 195 arg_parser.add_argument('--ltp-root2', 196 dest='ltp_root2', 197 required=True, 198 help="LTP Root Directory after merge") 199 arg_parser.add_argument('--scenario', default=None, 200 dest='scenario', 201 help="LTP scenario to list tests for") 202 args = arg_parser.parse_args() 203 204 ltp_tests1 = scan_ltp(args.ltp_root1, args.scenario) 205 ltp_tests2 = scan_ltp(args.ltp_root2, args.scenario) 206 show_diff(ltp_tests1, ltp_tests2) 207 208if __name__ == '__main__': 209 main() 210