1#! /usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites. 20 21The tool can be run in two separate steps by dumping the list of tests as a json 22file, i.e.: 23 24$ git checkout master 25$ compare_ltp_projects.py --ltp-old . --ltp-old-json ./ltp_old.json 26$ git checkout mymergebranch 27$ compare_ltp_projects.py --ltp-old-json ./ltp_old.json --ltp-new . 28""" 29 30import os 31import argparse 32import os.path 33import sys 34import json 35 36def scan_tests(ltp_root, suite): 37 ''' Find all tests that are run as part of given test suite in LTP. 38 39 Args: 40 ltp_root: Path ot the LTP project. 41 suite: Name of testsuite. 42 43 Returns: 44 List of tests names that are run as part of the given test suite. 45 ''' 46 47 tests = [] 48 if not suite: 49 return tests 50 51 runtest_dir = ltp_root + os.path.sep + 'runtest' 52 test_suiteFile = runtest_dir + os.path.sep + suite 53 if not os.path.isfile(test_suiteFile): 54 print('No tests defined for suite {}',format(suite)) 55 return tests 56 57 with open(test_suiteFile) as f: 58 for line in f: 59 line = line.strip() 60 if line and not line.startswith('#'): 61 tests.append(line.split()[0]) 62 f.close() 63 return tests 64 65def scan_test_suites(ltp_root, scenario): 66 ''' Find all testuites and tests run as part of given LTP scenario 67 68 Args: 69 ltp_root: Path to the LTP project. 70 scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts" 71 72 Returns: 73 List of testsuite names that are run as part of given scenario (e.g. vts). 74 If scenario is not specified, return all testsuites. 75 ''' 76 77 runtest_dir = ltp_root + os.path.sep + 'runtest' 78 if not os.path.isdir(runtest_dir): 79 print('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root)) 80 sys.exit(2) 81 82 test_suites = [] 83 if scenario: 84 scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario 85 if not os.path.isfile(scenarioFile): 86 return test_suites 87 with open(scenarioFile) as f: 88 for line in f: 89 line = line.strip() 90 if not line.startswith('#'): 91 test_suites.append(line) 92 return test_suites 93 94 runtest_dir = ltp_root + os.path.sep + 'runtest' 95 test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)] 96 97 return test_suites 98 99def scan_ltp(ltp_root, scenario): 100 ''' scan LTP project and return all tests and testuites present. 101 102 Args: 103 ltp_root: Path to the LTP project. 104 scenario: specific scenario we want to scan. e.g. vts 105 106 Returns: 107 Dictionary of all LTP test names keyed by testsuite they are run as part of. 108 E.g. 109 { 110 'mem': ['oom01', 'mem02',...], 111 'syscalls': ['open01', 'read02',...], 112 ... 113 } 114 ''' 115 116 if not os.path.isdir(ltp_root): 117 print('ltp_root {} does not exist'.format(ltp_root)) 118 sys.exit(1) 119 120 test_suites = scan_test_suites(ltp_root, scenario) 121 if not test_suites: 122 print('No Testsuites found for scenario {}'.format(scenario)) 123 sys.exit(3) 124 125 ltp_tests = {} 126 for suite in test_suites: 127 ltp_tests[suite] = scan_tests(ltp_root, suite) 128 return ltp_tests 129 130def show_diff(ltp_tests_1, ltp_tests_2): 131 ''' Find and print diff between testcases in 2 LTP project checkouts. 132 133 Args: 134 ltp_tests_1: dictionary of tests keyed by test suite names 135 ltp_tests_2: dictionary of tests keyed by test suite names 136 ''' 137 test_suites1 = set(ltp_tests_1.keys()) 138 test_suites2 = set(ltp_tests_2.keys()) 139 140 # Generate lists of deleted, added and common test suites between 141 # LTP1 & LTP2 142 deleted_test_suites = sorted(test_suites1.difference(test_suites2)) 143 added_test_suites = sorted(test_suites2.difference(test_suites1)) 144 common_test_suites = test_suites1.intersection(test_suites2) 145 146 deleted_tests = [] 147 added_tests = [] 148 for suite in common_test_suites: 149 tests1 = set(ltp_tests_1[suite]) 150 tests2 = set(ltp_tests_2[suite]) 151 152 exclusive_test1 = tests1.difference(tests2) 153 exclusive_test2 = tests2.difference(tests1) 154 for e in exclusive_test1: 155 deleted_tests.append(suite + '.' + e) 156 for e in exclusive_test2: 157 added_tests.append(suite + '.' + e) 158 deleted_tests = sorted(deleted_tests) 159 added_tests = sorted(added_tests) 160 print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests) 161 162 163def print_columns(added_test_suites, deleted_test_suites, added_tests, deleted_tests): 164 DEFAULT_WIDTH = 8 165 # find the maximum width of a test suite name or test name 166 # we have to print to decide the alignment. 167 if not deleted_test_suites: 168 width_suites = DEFAULT_WIDTH 169 else: 170 width_suites = max(len(x) for x in deleted_test_suites) 171 172 if added_test_suites: 173 width_suites = max(width_suites, max(len(x) for x in added_test_suites)) 174 175 if not deleted_tests: 176 width_tests = DEFAULT_WIDTH 177 else: 178 width_tests = max(len(x) for x in deleted_tests) 179 180 if added_tests: 181 width_tests = max(width_tests, max(len(x) for x in added_tests)) 182 183 width = max(width_suites, width_tests); 184 185 # total rows we have to print 186 total_rows = max(len(deleted_test_suites), len(added_test_suites)) 187 added_text = 'Added ({})'.format(len(added_test_suites)) 188 deleted_text = 'Deleted ({})'.format(len(deleted_test_suites)) 189 if total_rows > 0: 190 print('{:*^{len}}'.format(' Tests Suites ', len=width*2+2)) 191 print('{:<{len}} {:<{len}}'.format(added_text, deleted_text, len=width)) 192 for i in range(total_rows): 193 print('{:<{len}} {:<{len}}'.format('' if i >= len(added_test_suites) else str(added_test_suites[i]), 194 '' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), len=width)) 195 196 print('') 197 # total rows we have to print 198 total_rows = max(len(deleted_tests), len(added_tests)) 199 added_text = 'Added ({})'.format(len(added_tests)) 200 deleted_text = 'Deleted ({})'.format(len(deleted_tests)) 201 if total_rows: 202 print('{:*^{len}}'.format(' Tests ', len=width*2+2)) 203 print('{:^{len}} {:^{len}}'.format(added_text, deleted_text, len=width)) 204 for i in range(total_rows): 205 print('{:<{len}} {:<{len}}'.format('' if i >= len(added_tests) else str(added_tests[i]), 206 '' if i >= len(deleted_tests) else str(deleted_tests[i]), len=width)) 207def main(): 208 arg_parser = argparse.ArgumentParser( 209 description='Diff 2 LTP projects for supported test cases') 210 arg_parser.add_argument('-o', '--ltp-old', 211 dest='ltp_old', 212 help="LTP Root Directory before merge") 213 arg_parser.add_argument('-oj', '--ltp-old-json', 214 dest='ltp_old_json', 215 default='ltp_old.json', 216 help="Old LTP parsed directory in json format") 217 arg_parser.add_argument('-n', '--ltp-new', 218 dest='ltp_new', 219 help="LTP Root Directory after merge") 220 arg_parser.add_argument('-nj', '--ltp-new-json', 221 dest='ltp_new_json', 222 default='ltp_new.json', 223 help="New LTP parsed directory in json format") 224 arg_parser.add_argument('--scenario', default=None, 225 dest='scenario', 226 help="LTP scenario to list tests for") 227 args = arg_parser.parse_args() 228 229 # Find tests in the "old" directory or read the json output from a prior run 230 if args.ltp_old: 231 ltp_tests1 = scan_ltp(args.ltp_old, args.scenario) 232 elif args.ltp_old_json and os.path.isfile(args.ltp_old_json): 233 with open(args.ltp_old_json) as f: 234 ltp_tests1 = json.load(f) 235 else: 236 ltp_tests1 = None 237 238 # Do the same for the "new" directory 239 if args.ltp_new: 240 ltp_tests2 = scan_ltp(args.ltp_new, args.scenario) 241 elif args.ltp_new_json and os.path.isfile(args.ltp_new_json): 242 with open(args.ltp_new_json) as f: 243 ltp_tests2 = json.load(f) 244 else: 245 ltp_tests2 = None 246 247 if ltp_tests1 and ltp_tests2: 248 # Show the difference of the two directories if both are present 249 show_diff(ltp_tests1, ltp_tests2) 250 elif ltp_tests1: 251 # If just the old directory was read, dump it as json 252 with open(args.ltp_old_json, 'w') as f: 253 json.dump(ltp_tests1, f) 254 elif ltp_tests2: 255 # If just the new directory was read, dump it as json 256 with open(args.ltp_new_json, 'w') as f: 257 json.dump(ltp_tests2, f) 258 259if __name__ == '__main__': 260 main() 261