1#!/usr/bin/env python 2# coding=utf-8 3############################################## 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16############################################## 17import json 18import os.path 19import random 20import sys 21import time 22import ast 23from subprocess import Popen 24from subprocess import PIPE 25 26 27def print_help(): 28 content = "\n" \ 29 "Usage: signtool.jar -scope <simple|all|`component`> -n <round, default 1> <--random>\n" \ 30 " signtool.jar : Main progress jar file\n" \ 31 " component: \n" \ 32 " --random: random test, default false" \ 33 "\n" \ 34 "Example: \n" \ 35 " signtool.jar \n" \ 36 " signtool.jar -runtest\n" \ 37 " signtool.jar -scope all -n 1000\n" \ 38 " signtool.jar -scope generate-profile-cert\n" \ 39 " signtool.jar -n 50 --random\n" \ 40 "\n" 41 42 print(content) 43 pass 44 45 46def random_pwd(): 47 min_pwd = 100000 48 max_pwd = 999999 49 return random.randint(min_pwd, max_pwd), random.randint(min_pwd, max_pwd) 50 51 52keystorePwd, keyPwd = random_pwd() 53 54 55random_scope = { 56 'generate-keypair': { 57 'required': { 58 'keyAlias': 'oh-app1-key-v1', 59 'keyAlg': ["RSA", "ECC"], 60 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 61 'keystoreFile': ['ohtest.jks', 'ohtest.p12'] 62 }, 63 'others': { 64 'keyPwd': '123456', 65 'keystorePwd': '123456' 66 } 67 }, 68 'generate-csr': { 69 'required': { 70 'keyAlias': 'oh-app1-key-v1', 71 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 72 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 73 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 74 'outFile': 'oh-app1-key-v1.csr' 75 }, 76 'others': { 77 'keyPwd': '123456', 78 'keystorePwd': '132456' 79 } 80 }, 81 'generate-ca': { 82 'required': { 83 'keyAlias': ['oh-ca-key-v1', "oh-app-sign-srv-ca-key-v1"], 84 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 85 'keyAlg': ['RSA', 'ECC'], 86 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 87 'subject': ["C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 88 "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA"], 89 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 90 'outFile': 'app-sign-srv-ca.cer' 91 }, 92 'others': { 93 'keyPwd': '123456', 94 'keystorePwd': '132456', 95 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 96 'issuerKeyAlias': 'oh-app-sign-srv-ca-key-v1', 97 'issuerKeyPwd': '123456', 98 'validity': '365', 99 'basicConstraintsPathLen': '2' 100 } 101 }, 102 'generate-cert': { 103 'required': { 104 'keyAlias': ['oh-sub-key-v1', 'oh-ca-key-v1'], 105 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 106 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 107 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 108 'issuerKeyAlias': 'oh-ca-key-v1', 109 'extKeyUsage': 'codeSignature', 110 'keyUsage': ['digitalSignature,nonRepudiation,keyEncipherment', 111 'dataEncipherment,keyAgreement, certificateSignature, crlSignature', 112 'encipherOnly, encipherOnly'], 113 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 114 'outFile': 'app1.cer' 115 }, 116 'others': { 117 'extKeyUsage': ['serverAuthentication', 'clientAuthentication', 'emailProtection'], 118 'extKeyUsageCritical': ['false', 'true'], 119 'keyUsageCritical': ['false', 'true'], 120 'issuerKeyPwd': '123456', 121 'keyPwd': '123456', 122 'validity': '365', 123 'keystorePwd': '123456' 124 } 125 } 126} 127 128simple_scope = { 129 'generate-keypair': [ 130 'generate-keypair -keyAlias "oh-app1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-256 ' 131 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 132 133 'generate-keypair -keyAlias "oh-profile1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-384 ' 134 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 135 136 'generate-keypair -keyAlias "oh-app2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 2048 ' 137 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456', 138 139 'generate-keypair -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 4096 ' 140 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456' 141 ], 142 'generate-csr': [ 143 'generate-csr -keyAlias "oh-app1-key-v1" -keyPwd 123456 -subject ' 144 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withECDSA ' 145 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/oh-app1-key-v1.csr"', 146 147 'generate-csr -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -subject ' 148 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withRSA ' 149 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/oh-profile2-key-v1.csr"' 150 ], 151 'generate-ca': [ 152 # Root CA in ohtest.jks 153 'generate-ca -keyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 154 '-validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 155 '-outFile "./test1/root-ca1.cer" -keyAlg ECC -keySize NIST-P-256', 156 # Sub app cert in ohtest.jks 157 'generate-ca -keyAlias "oh-app-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 158 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 159 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withECDSA ' 160 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-sign-srv-ca1.cer" ' 161 '-keyAlg ECC -keySize NIST-P-256', 162 # Sub profile cert in ohtest.jks 163 'generate-ca -keyAlias "oh-profile-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 164 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 165 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" ' 166 '-keystorePwd 123456 -outFile "./test1/profile-sign-srv-ca1.cer" -keyAlg ECC -keySize NIST-P-384', 167 168 # Root CA in ohtest.p12 169 'generate-ca -keyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 170 '-validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 ' 171 '-outFile "./test2/root-ca2.cer" -keyAlg RSA -keySize 2048', 172 # Sub app cert in ohtest.p12 173 'generate-ca -keyAlias "oh-app-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 174 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 175 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withRSA ' 176 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-sign-srv-ca2.cer" ' 177 '-keyAlg RSA -keySize 2048', 178 # Sub profile cert in ohtest.p12 179 'generate-ca -keyAlias "oh-profile-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 180 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 181 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" ' 182 '-keystorePwd 123456 -outFile "./test2/profile-sign-srv-ca2.cer" -keyAlg RSA -keySize 2048' 183 ], 184 'generate-cert': [ 185 # Self-Definition cert - Root CA 186 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 187 '-issuerKeyAlias "oh-app1-key-v1" -issuerKeyPwd 123456 ' 188 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" -validity 365 ' 189 '-keyUsage "certificateSignature, crlSignature" -signAlg SHA256withECDSA ' 190 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-root.cer" -keyPwd 123456', 191 # Self-definition sign cert - app cert 192 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 193 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" ' 194 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 ' 195 '-keyUsage digitalSignature -extKeyUsage codeSignature -signAlg SHA256withECDSA ' 196 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-app1.cer" -keyPwd 123456' 197 ], 198 'generate-app-cert': [ 199 # App sign cert via ohtest.jks 200 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 201 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 202 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 203 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app1.cer" -keyPwd 123456 ' 204 '-outForm cert ', 205 # App sign cert chain via ohtest.jks 206 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 207 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 208 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 209 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-release1.pem" ' 210 '-subCaCertFile ./test1/app-sign-srv-ca1.cer -outForm certChain ' 211 '-rootCaCertFile ./test1/root-ca1.cer -keyPwd 123456', 212 # App sign cert via ohtest.p12 213 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 214 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 215 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 216 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app2.cer" -keyPwd 123456 ' 217 '-outForm cert ', 218 # App sign cert chain via ohtest.p12 219 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 220 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 221 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 222 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-release2.pem" ' 223 '-subCaCertFile ./test2/app-sign-srv-ca2.cer -outForm certChain ' 224 '-rootCaCertFile ./test2/root-ca2.cer -keyPwd 123456' 225 ], 226 'generate-profile-cert': [ 227 # Profile sign cert via ohtest.jks 228 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 229 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 230 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" ' 231 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 232 '-keystorePwd 123456 -outFile "./test1/profile1.cer" -keyPwd 123456 -outForm cert ', 233 # Profile sign cert chain via ohtest.jks 234 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 235 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 236 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" -validity 365 ' 237 '-signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 238 '-keystorePwd 123456 -outFile "./test1/profile-release1.pem" ' 239 '-subCaCertFile "./test1/profile-sign-srv-ca1.cer" -outForm certChain ' 240 '-rootCaCertFile "./test1/root-ca1.cer" -keyPwd 123456', 241 # Profile sign cert via ohtest.p12 242 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 243 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 244 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" ' 245 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 246 '-keystorePwd 123456 -outFile "./test2/profile2.cer" -keyPwd 123456 -outForm cert ', 247 # Profile sign cert chain via ohtest.p12 248 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 249 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 250 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" -validity 365 ' 251 '-signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 252 '-keystorePwd 123456 -outFile "./test2/profile-release2.pem" -subCaCertFile "./test2/profile-sign-srv-ca2.cer" ' 253 '-outForm certChain -rootCaCertFile "./test2/root-ca2.cer" -keyPwd 123456' 254 ], 255 'sign-profile': [ 256 'sign-profile -mode localSign -keyAlias "oh-profile1-key-v1" -profileCertFile "./test1/profile-release1.pem" ' 257 '-inFile "profile.json" -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 258 '-outFile "./test1/app1-profile1.p7b" -keyPwd 123456' 259 ], 260 'verify-profile': [ 261 'verify-profile -inFile "./test1/app1-profile1.p7b"' 262 ] 263} 264 265def get_test_scope_from_file(): 266 with open('commands.config', 'r', encoding='utf-8') as f: 267 content = f.read() 268 return ast.literal_eval(content) 269 270test_scope = get_test_scope_from_file() 271 272test_result = {} 273 274 275def run_target(case, cmd): 276 if not test_result.get(case, None): 277 test_result[case] = {'times': 0, 'total_cost': 0, 'success': 0, 'fail': 0} 278 279 case_result = test_result.get(case) 280 case_result['times'] = case_result['times'] + 1 281 start = time.time() 282 283 command = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=True) 284 285 286 287 288 out = command.stdout.readlines() 289 with open("log.txt", mode='a+', encoding='utf-8') as f: 290 if len(out) > 0: 291 f.writelines(cmd + "\r\n") 292 for line in out: 293 f.writelines(str(line.strip()) + "\r\n") 294 295 success = True 296 error = command.stderr.readlines() 297 with open("error.txt", mode='a+', encoding='utf-8') as f: 298 if len(error) > 0: 299 f.writelines(cmd + "\r\n") 300 for line in error: 301 success = False 302 f.writelines(str(line.strip()) + "\r\n") 303 304 command.wait() 305 end = time.time() 306 case_result['total_cost'] = case_result['total_cost'] + (end - start) 307 308 if success: 309 case_result['success'] = case_result['success'] + 1 310 else: 311 case_result['fail'] = case_result['fail'] + 1 312 return success 313 314 315def run_simple_case(case, jar_file): 316 test_case = simple_scope.get(case, None) 317 if not test_case: 318 print("Not found test case: {}".format(case)) 319 exit(0) 320 321 for k in test_case: 322 cmd = 'java -jar {} {}'.format(jar_file, k) 323 print("== Run command: {}".format(cmd)) 324 result = run_target(case, cmd) 325 print("== Done command: {}".format(result)) 326 327def run_test_case(case, jar_file): 328 test_case = test_scope.get(case, None) 329 if not test_case: 330 print("Not found test case: {}".format(case)) 331 exit(0) 332 333 334 335 for k in test_case: 336 cmd = 'java -jar {} {}'.format(jar_file, k) 337 print("== Run command: {}".format(cmd)) 338 result = run_target(case, cmd) 339 340 with open('test_result.log', 'r', encoding='utf-8') as f: 341 content = f.read() 342 test_result_dict = ast.literal_eval(content) 343 344 345 if case == 'case-assert-true': 346 if result: 347 print("== Done command: Expected True and tested True") 348 else: 349 test_result_dict['commands_expected_True_but_tested_False'].append(cmd) 350 print("== Done command: Expected True but tested False") 351 else: 352 if result: 353 test_result_dict['commands_expected_False_but_tested_True'].append(cmd) 354 print("== Done command: Expected False but tested True") 355 else: 356 print("== Done command: Expected False and tested False") 357 358 359 with open("test_result.log", mode='w', encoding='utf-8') as tr: 360 tr.write(json.dumps(test_result_dict, indent=4)) 361 362 363def random_str(): 364 strs = "abcdefghjiklmnopqstuvwxyzABCDEFGHIJKLMNOPQRS TUVWXYZ1234567890~!@#ls%^&*()_+,./<>?;':" 365 result = '' 366 for i in range(random.randint(1, 30)): 367 result = result + random.choice(strs) 368 return result 369 370 371def run_random_case(case, jar_file): 372 test_case = random_scope.get(case, None) 373 if not test_case: 374 print("Not found test case: {}".format(case)) 375 exit(0) 376 377 cmd = 'java -jar {} {}'.format(jar_file, case) 378 for k, v in test_case.get('required').items(): 379 r = random.choice(['none', 'choice', 'choice', 'random']) 380 if r == 'choice': 381 cmd = cmd + ' -{} "{}" '.format(k, random.choice(v) if isinstance(v, list) else v) 382 elif r == 'random': 383 cmd = cmd + ' -{} "{}" '.format(k, random_str()) 384 385 for k, v in test_case.get('others').items(): 386 r = random.choice(['none', 'choice', 'choice', 'random']) 387 if r == 'choice': 388 cmd = cmd + ' -{} "{}" '.format(k, random.choice(v) if isinstance(v, list) else v) 389 elif r == 'random': 390 cmd = cmd + ' -{} "{}" '.format(k, random_str()) 391 392 print("== Run command: {}".format(cmd)) 393 result = run_target(case, cmd) 394 print("== Done command: {}".format(result)) 395 396 397def run_all_case(case, jar_file): 398 test_case = random_scope.get(case, None) 399 if not test_case: 400 print("Not found test case: {}".format(case)) 401 exit(0) 402 403 cmd = 'java -jar {} {}'.format(jar_file, case) 404 for ak, av in test_case.get('required').items(): 405 cmd = cmd + ' -{} "{}" '.format(ak, random.choice(av) if isinstance(av, list) else av) 406 407 print("== Run command: {}".format(cmd)) 408 result = run_target(case, cmd) 409 print("== Done command: {}".format(result)) 410 411 412def prepare_env(): 413 test_dirs = ['test1', 'test2'] 414 for test_dir in test_dirs: 415 if not os.path.exists(test_dir): 416 os.mkdir(test_dir) 417 418 for key_file in ['ohtest.jks', 'ohtest.p12']: 419 target_file = os.path.join(test_dir, key_file) 420 if os.path.exists(target_file): 421 os.remove(target_file) 422 if os.path.exists(key_file): 423 os.remove(key_file) 424 425 426def process_cmd(args): 427 run_round = 1 428 run_scope = 'simple' 429 is_random = False 430 431 if len(args) <= 1 or ('.jar' not in args[1]) or '--help' == args[1] or '-h' == args[1]: 432 print_help() 433 exit(0) 434 435 jar_file = args[1] 436 if not os.path.exists(jar_file): 437 print("Jar file '{}' not found".format(jar_file)) 438 exit(0) 439 440 if len(args) >= 3: 441 try: 442 for i in range(2, len(args), 1): 443 if args[i] == '-n': 444 run_round = int(args[i + 1]) 445 elif args[i] == '-scope': 446 run_scope = args[i + 1] 447 elif args[i] == '--random': 448 is_random = True 449 elif args[i] == '-runtest': 450 run_scope = 'runtest' 451 except IndexError: 452 print_help() 453 exit(0) 454 455 print('=== Start testing ===') 456 print('Scope: {}. Round: {}. Random: {}'.format(run_scope, run_round, is_random)) 457 458 if os.path.exists('log.txt'): 459 os.remove('log.txt') 460 if os.path.exists('error.txt'): 461 os.remove('error.txt') 462 463 for i in range(run_round): 464 if run_scope == 'all': 465 for r_scope, _ in random_scope.items(): 466 run_all_case(r_scope, jar_file) 467 elif is_random: 468 for r_scope, _ in random_scope.items(): 469 run_random_case(r_scope, jar_file) 470 elif run_scope == 'simple': 471 prepare_env() 472 for s_scope, _ in simple_scope.items(): 473 run_simple_case(s_scope, jar_file) 474 elif run_scope == 'runtest': 475 prepare_env() 476 with open("test_result.log", mode='w', encoding='utf-8') as file_result: 477 result_dict = { 478 'commands_expected_True_but_tested_False': [], 479 'commands_expected_False_but_tested_True': [] 480 } 481 file_result.write(str(result_dict)) 482 for t_scope, _ in test_scope.items(): 483 run_test_case(t_scope, jar_file) 484 else: 485 run_simple_case(run_scope, jar_file) 486 487 488if __name__ == '__main__': 489 process_cmd(sys.argv) 490 print("All test done") 491 print("========================") 492 for rk, rv in test_result.items(): 493 print("Case {}, run times: {}, avg cost: {}s, total success: {}, total fail: {}".format(rk, rv['times'], round( 494 rv['total_cost'] / rv['times'], 2), rv['success'], rv['fail'])) 495 print("========================") 496 print("See log.txt / error.txt") 497