1#! /usr/bin/env python3 2# 3# Copyright 2018 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17from __future__ import print_function 18 19"""Tool for comparing 2 LTP projects to find added / removed tests & testsuites. 20 21The tool can be run in two separate steps by dumping the list of tests as a json 22file, i.e.: 23 24$ git checkout master 25$ compare_ltp_projects.py --ltp-old . --ltp-old-json ./ltp_old.json 26$ git checkout mymergebranch 27$ compare_ltp_projects.py --ltp-old-json ./ltp_old.json --ltp-new . 28""" 29 30import os 31import argparse 32import os.path 33import sys 34import json 35 36def scan_tests(ltp_root, suite): 37 ''' Find all tests that are run as part of given test suite in LTP. 38 39 Args: 40 ltp_root: Path ot the LTP project. 41 suite: Name of testsuite. 42 43 Returns: 44 List of tests names that are run as part of the given test suite. 45 ''' 46 47 tests = [] 48 if not suite: 49 return tests 50 51 runtest_dir = ltp_root + os.path.sep + 'runtest' 52 test_suiteFile = runtest_dir + os.path.sep + suite 53 if not os.path.isfile(test_suiteFile): 54 print('No tests defined for suite {}',format(suite)) 55 return tests 56 57 with open(test_suiteFile) as f: 58 for line in f: 59 line = line.strip() 60 if line and not line.startswith('#'): 61 tests.append(line.split()[0]) 62 f.close() 63 return tests 64 65def scan_test_suites(ltp_root, scenario): 66 ''' Find all testuites and tests run as part of given LTP scenario 67 68 Args: 69 ltp_root: Path to the LTP project. 70 scenario: name of the scenario (found in ltp_root/scenario_groups). E.g. "vts" 71 72 Returns: 73 List of testsuite names that are run as part of given scenario (e.g. vts). 74 If scenario is not specified, return all testsuites. 75 ''' 76 77 runtest_dir = ltp_root + os.path.sep + 'runtest' 78 if not os.path.isdir(runtest_dir): 79 print('Invalid ltp_root {}, runtest directory doesnt exist'.format(ltp_root)) 80 sys.exit(2) 81 82 test_suites = [] 83 if scenario: 84 scenarioFile = ltp_root + os.path.sep + 'scenario_groups' + os.path.sep + scenario 85 if not os.path.isfile(scenarioFile): 86 return test_suites 87 with open(scenarioFile) as f: 88 for line in f: 89 line = line.strip() 90 if not line.startswith('#'): 91 test_suites.append(line) 92 return test_suites 93 94 runtest_dir = ltp_root + os.path.sep + 'runtest' 95 test_suites = [f for f in os.listdir(runtest_dir) if os.path.isfile(runtest_dir + os.path.sep + f)] 96 97 return test_suites 98 99def scan_ltp(ltp_root, scenario): 100 ''' scan LTP project and return all tests and testuites present. 101 102 Args: 103 ltp_root: Path to the LTP project. 104 scenario: specific scenario we want to scan. e.g. vts 105 106 Returns: 107 Dictionary of all LTP test names keyed by testsuite they are run as part of. 108 E.g. 109 { 110 'mem': ['oom01', 'mem02',...], 111 'syscalls': ['open01', 'read02',...], 112 ... 113 } 114 ''' 115 116 if not os.path.isdir(ltp_root): 117 print('ltp_root {} does not exist'.format(ltp_root)) 118 sys.exit(1) 119 120 test_suites = scan_test_suites(ltp_root, scenario) 121 if not test_suites: 122 print('No Testsuites found for scenario {}'.format(scenario)) 123 sys.exit(3) 124 125 ltp_tests = {} 126 for suite in test_suites: 127 ltp_tests[suite] = scan_tests(ltp_root, suite) 128 return ltp_tests 129 130def show_diff(ltp_tests_1, ltp_tests_2): 131 ''' Find and print diff between testcases in 2 LTP project checkouts. 132 133 Args: 134 ltp_tests_1: dictionary of tests keyed by test suite names 135 ltp_tests_2: dictionary of tests keyed by test suite names 136 ''' 137 DEFAULT_WIDTH = 8 138 test_suites1 = set(sorted(ltp_tests_1.keys())) 139 test_suites2 = set(sorted(ltp_tests_2.keys())) 140 141 # Generate lists of deleted, added and common test suites between 142 # LTP1 & LTP2 143 deleted_test_suites = sorted(test_suites1.difference(test_suites2)) 144 added_test_suites = sorted(test_suites2.difference(test_suites1)) 145 common_test_suites = sorted(test_suites1.intersection(test_suites2)) 146 147 deleted_tests = [] 148 added_tests = [] 149 for suite in common_test_suites: 150 tests1 = set(sorted(ltp_tests_1[suite])) 151 tests2 = set(sorted(ltp_tests_2[suite])) 152 153 exclusive_test1 = tests1.difference(tests2) 154 exclusive_test2 = tests2.difference(tests1) 155 for e in exclusive_test1: 156 deleted_tests.append(suite + '.' + e) 157 for e in exclusive_test2: 158 added_tests.append(suite + '.' + e) 159 160 # find the maximum width of a test suite name or test name 161 # we have to print to decide the alignment. 162 if not deleted_test_suites: 163 width_suites = DEFAULT_WIDTH 164 else: 165 width_suites = max([len(x) for x in deleted_test_suites]) 166 167 if not deleted_tests: 168 width_tests = DEFAULT_WIDTH 169 else: 170 width_tests = max([len(x) for x in deleted_tests]) 171 width = max(width_suites, width_tests); 172 173 # total rows we have to print 174 total_rows = max(len(deleted_test_suites), len(added_test_suites)) 175 deleted_text = 'Deleted ({})'.format(len(deleted_test_suites)) 176 added_text = 'Added ({})'.format(len(added_test_suites)) 177 if total_rows > 0: 178 print('{:*^{len}}'.format(' Tests Suites ', len=width*2+2)) 179 print('{:>{len}} {:>{len}}'.format(deleted_text, added_text, len=width)) 180 for i in range(total_rows): 181 print('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_test_suites) else str(deleted_test_suites[i]), 182 '' if i >= len(added_test_suites) else str(added_test_suites[i]), len=width)) 183 184 print('') 185 # total rows we have to print 186 total_rows = max(len(deleted_tests), len(added_tests)) 187 deleted_text = 'Deleted ({})'.format(len(deleted_tests)) 188 added_text = 'Added ({})'.format(len(added_tests)) 189 if total_rows: 190 print('{:*^{len}}'.format(' Tests ', len=width*2+2)) 191 print('{:>{len}} {:>{len}}'.format(deleted_text, added_text, len=width)) 192 for i in range(total_rows): 193 print('{:>{len}} {:>{len}}'.format('' if i >= len(deleted_tests) else str(deleted_tests[i]), 194 '' if i >= len(added_tests) else str(added_tests[i]), len=width)) 195def main(): 196 arg_parser = argparse.ArgumentParser( 197 description='Diff 2 LTP projects for supported test cases') 198 arg_parser.add_argument('-o', '--ltp-old', 199 dest='ltp_old', 200 help="LTP Root Directory before merge") 201 arg_parser.add_argument('-oj', '--ltp-old-json', 202 dest='ltp_old_json', 203 default='ltp_old.json', 204 help="Old LTP parsed directory in json format") 205 arg_parser.add_argument('-n', '--ltp-new', 206 dest='ltp_new', 207 help="LTP Root Directory after merge") 208 arg_parser.add_argument('-nj', '--ltp-new-json', 209 dest='ltp_new_json', 210 default='ltp_new.json', 211 help="New LTP parsed directory in json format") 212 arg_parser.add_argument('--scenario', default=None, 213 dest='scenario', 214 help="LTP scenario to list tests for") 215 args = arg_parser.parse_args() 216 217 # Find tests in the "old" directory or read the json output from a prior run 218 if args.ltp_old: 219 ltp_tests1 = scan_ltp(args.ltp_old, args.scenario) 220 elif args.ltp_old_json and os.path.isfile(args.ltp_old_json): 221 with open(args.ltp_old_json) as f: 222 ltp_tests1 = json.load(f) 223 else: 224 ltp_tests1 = None 225 226 # Do the same for the "new" directory 227 if args.ltp_new: 228 ltp_tests2 = scan_ltp(args.ltp_new, args.scenario) 229 elif args.ltp_new_json and os.path.isfile(args.ltp_new_json): 230 with open(args.ltp_new_json) as f: 231 ltp_tests2 = json.load(f) 232 else: 233 ltp_tests2 = None 234 235 if ltp_tests1 and ltp_tests2: 236 # Show the difference of the two directories if both are present 237 show_diff(ltp_tests1, ltp_tests2) 238 elif ltp_tests1: 239 # If just the old directory was read, dump it as json 240 with open(args.ltp_old_json, 'w') as f: 241 json.dump(ltp_tests1, f) 242 elif ltp_tests2: 243 # If just the new directory was read, dump it as json 244 with open(args.ltp_new_json, 'w') as f: 245 json.dump(ltp_tests2, f) 246 247if __name__ == '__main__': 248 main() 249