1#!/usr/bin/env python 2# coding=utf-8 3############################################## 4# Copyright (c) 2021-2022 Huawei Device Co., Ltd. 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16############################################## 17import json 18import os.path 19import random 20import sys 21import time 22import ast 23from subprocess import Popen 24from subprocess import PIPE 25 26 27def print_help(): 28 content = "\n" \ 29 "Usage: signtool.jar -scope <simple|all|`component`> -n <round, default 1> <--random>\n" \ 30 " signtool.jar : Main progress jar file\n" \ 31 " component: \n" \ 32 " --random: random test, default false" \ 33 "\n" \ 34 "Example: \n" \ 35 " signtool.jar \n" \ 36 " signtool.jar -runtest\n" \ 37 " signtool.jar -scope all -n 1000\n" \ 38 " signtool.jar -scope generate-profile-cert\n" \ 39 " signtool.jar -n 50 --random\n" \ 40 "\n" 41 42 print(content) 43 pass 44 45random_scope = { 46 'generate-keypair': { 47 'required': { 48 'keyAlias': 'oh-app1-key-v1', 49 'keyAlg': ["RSA", "ECC"], 50 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 51 'keystoreFile': ['ohtest.jks', 'ohtest.p12'] 52 }, 53 'others': { 54 'keyPwd': '123456', 55 'keystorePwd': '123456' 56 } 57 }, 58 'generate-csr': { 59 'required': { 60 'keyAlias': 'oh-app1-key-v1', 61 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 62 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 63 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 64 'outFile': 'oh-app1-key-v1.csr' 65 }, 66 'others': { 67 'keyPwd': '123456', 68 'keystorePwd': '132456' 69 } 70 }, 71 'generate-ca': { 72 'required': { 73 'keyAlias': ['oh-ca-key-v1', "oh-app-sign-srv-ca-key-v1"], 74 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 75 'keyAlg': ['RSA', 'ECC'], 76 'keySize': ["2048", "3072", "4096", "NIST-P-256", "NIST-P-384"], 77 'subject': ["C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 78 "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA"], 79 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 80 'outFile': 'app-sign-srv-ca.cer' 81 }, 82 'others': { 83 'keyPwd': '123456', 84 'keystorePwd': '132456', 85 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 86 'issuerKeyAlias': 'oh-app-sign-srv-ca-key-v1', 87 'issuerKeyPwd': '123456', 88 'validity': '365', 89 'basicConstraintsPathLen': '2' 90 } 91 }, 92 'generate-cert': { 93 'required': { 94 'keyAlias': ['oh-sub-key-v1', 'oh-ca-key-v1'], 95 'signAlg': ["SHA256withRSA", "SHA384withRSA", "SHA256withECDSA", "SHA384withECDSA"], 96 'subject': "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release", 97 'issuer': 'C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Application Signature Service CA', 98 'issuerKeyAlias': 'oh-ca-key-v1', 99 'extKeyUsage': 'codeSignature', 100 'keyUsage': ['digitalSignature,nonRepudiation,keyEncipherment', 101 'dataEncipherment,keyAgreement, certificateSignature, crlSignature', 102 'encipherOnly, encipherOnly'], 103 'keystoreFile': ['ohtest.jks', 'ohtest.p12'], 104 'outFile': 'app1.cer' 105 }, 106 'others': { 107 'extKeyUsage': ['serverAuthentication', 'clientAuthentication', 'emailProtection'], 108 'extKeyUsageCritical': ['false', 'true'], 109 'keyUsageCritical': ['false', 'true'], 110 'issuerKeyPwd': '123456', 111 'keyPwd': '123456', 112 'validity': '365', 113 'keystorePwd': '123456' 114 } 115 } 116} 117 118simple_scope = { 119 'generate-keypair': [ 120 'generate-keypair -keyAlias "oh-app1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-256 ' 121 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 122 123 'generate-keypair -keyAlias "oh-profile1-key-v1" -keyPwd 123456 -keyAlg ECC -keySize NIST-P-384 ' 124 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456', 125 126 'generate-keypair -keyAlias "oh-app2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 2048 ' 127 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456', 128 129 'generate-keypair -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -keyAlg RSA -keySize 4096 ' 130 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456' 131 ], 132 'generate-csr': [ 133 'generate-csr -keyAlias "oh-app1-key-v1" -keyPwd 123456 -subject ' 134 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withECDSA ' 135 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/oh-app1-key-v1.csr"', 136 137 'generate-csr -keyAlias "oh-profile2-key-v1" -keyPwd 123456 -subject ' 138 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -signAlg SHA256withRSA ' 139 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/oh-profile2-key-v1.csr"' 140 ], 141 'generate-ca': [ 142 # Root CA in ohtest.jks 143 'generate-ca -keyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 144 '-validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 145 '-outFile "./test1/root-ca1.cer" -keyAlg ECC -keySize NIST-P-256', 146 # Sub app cert in ohtest.jks 147 'generate-ca -keyAlias "oh-app-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 148 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 149 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withECDSA ' 150 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-sign-srv-ca1.cer" ' 151 '-keyAlg ECC -keySize NIST-P-256', 152 # Sub profile cert in ohtest.jks 153 'generate-ca -keyAlias "oh-profile-sign-srv-ca-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 154 'CN=Root CA" -issuerKeyAlias "oh-root-ca-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 155 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withECDSA -keystoreFile "./test1/ohtest.jks" ' 156 '-keystorePwd 123456 -outFile "./test1/profile-sign-srv-ca1.cer" -keyAlg ECC -keySize NIST-P-384', 157 158 # Root CA in ohtest.p12 159 'generate-ca -keyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 160 '-validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 ' 161 '-outFile "./test2/root-ca2.cer" -keyAlg RSA -keySize 2048', 162 # Sub app cert in ohtest.p12 163 'generate-ca -keyAlias "oh-app-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 164 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 165 'CN= Application Signature Service CA" -validity 365 -signAlg SHA384withRSA ' 166 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-sign-srv-ca2.cer" ' 167 '-keyAlg RSA -keySize 2048', 168 # Sub profile cert in ohtest.p12 169 'generate-ca -keyAlias "oh-profile-sign-srv-ca2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 170 'CN=Root CA" -issuerKeyAlias "oh-root-ca2-key-v1" -subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 171 'CN= Profile Signature Service CA" -validity 365 -signAlg SHA384withRSA -keystoreFile "./test2/ohtest.p12" ' 172 '-keystorePwd 123456 -outFile "./test2/profile-sign-srv-ca2.cer" -keyAlg RSA -keySize 2048' 173 ], 174 'generate-cert': [ 175 # Self-Definition cert - Root CA 176 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" ' 177 '-issuerKeyAlias "oh-app1-key-v1" -issuerKeyPwd 123456 ' 178 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Root CA" -validity 365 ' 179 '-keyUsage "certificateSignature, crlSignature" -signAlg SHA256withECDSA ' 180 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-root.cer" -keyPwd 123456', 181 # Self-definition sign cert - app cert 182 'generate-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 183 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" ' 184 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 ' 185 '-keyUsage digitalSignature -extKeyUsage codeSignature -signAlg SHA256withECDSA ' 186 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/single-app1.cer" -keyPwd 123456' 187 ], 188 'generate-app-cert': [ 189 # App sign cert via ohtest.jks 190 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 191 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 192 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 193 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app1.cer" -keyPwd 123456 ' 194 '-outForm cert ', 195 # App sign cert chain via ohtest.jks 196 'generate-app-cert -keyAlias "oh-app1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 197 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca-key-v1" -subject ' 198 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 199 '-keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 -outFile "./test1/app-release1.pem" ' 200 '-subCaCertFile ./test1/app-sign-srv-ca1.cer -outForm certChain ' 201 '-rootCaCertFile ./test1/root-ca1.cer -keyPwd 123456', 202 # App sign cert via ohtest.p12 203 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 204 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 205 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 206 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app2.cer" -keyPwd 123456 ' 207 '-outForm cert ', 208 # App sign cert chain via ohtest.p12 209 'generate-app-cert -keyAlias "oh-app2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 210 'CN=Application Signature Service CA" -issuerKeyAlias "oh-app-sign-srv-ca2-key-v1" -subject ' 211 '"C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=App1 Release" -validity 365 -signAlg SHA256withECDSA ' 212 '-keystoreFile "./test2/ohtest.p12" -keystorePwd 123456 -outFile "./test2/app-release2.pem" ' 213 '-subCaCertFile ./test2/app-sign-srv-ca2.cer -outForm certChain ' 214 '-rootCaCertFile ./test2/root-ca2.cer -keyPwd 123456' 215 ], 216 'generate-profile-cert': [ 217 # Profile sign cert via ohtest.jks 218 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 219 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 220 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" ' 221 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 222 '-keystorePwd 123456 -outFile "./test1/profile1.cer" -keyPwd 123456 -outForm cert ', 223 # Profile sign cert chain via ohtest.jks 224 'generate-profile-cert -keyAlias "oh-profile1-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 225 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca-key-v1" ' 226 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile1 Release" -validity 365 ' 227 '-signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" ' 228 '-keystorePwd 123456 -outFile "./test1/profile-release1.pem" ' 229 '-subCaCertFile "./test1/profile-sign-srv-ca1.cer" -outForm certChain ' 230 '-rootCaCertFile "./test1/root-ca1.cer" -keyPwd 123456', 231 # Profile sign cert via ohtest.p12 232 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 233 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 234 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" ' 235 '-validity 365 -signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 236 '-keystorePwd 123456 -outFile "./test2/profile2.cer" -keyPwd 123456 -outForm cert ', 237 # Profile sign cert chain via ohtest.p12 238 'generate-profile-cert -keyAlias "oh-profile2-key-v1" -issuer "C=CN,O=OpenHarmony,OU=OpenHarmony Community,' 239 'CN=Profile Signature Service CA" -issuerKeyAlias "oh-profile-sign-srv-ca2-key-v1" ' 240 '-subject "C=CN,O=OpenHarmony,OU=OpenHarmony Community,CN=Profile2 Release" -validity 365 ' 241 '-signAlg SHA256withECDSA -keystoreFile "./test2/ohtest.p12" ' 242 '-keystorePwd 123456 -outFile "./test2/profile-release2.pem" -subCaCertFile "./test2/profile-sign-srv-ca2.cer" ' 243 '-outForm certChain -rootCaCertFile "./test2/root-ca2.cer" -keyPwd 123456' 244 ], 245 'sign-profile': [ 246 'sign-profile -mode localSign -keyAlias "oh-profile1-key-v1" -profileCertFile "./test1/profile-release1.pem" ' 247 '-inFile "profile.json" -signAlg SHA256withECDSA -keystoreFile "./test1/ohtest.jks" -keystorePwd 123456 ' 248 '-outFile "./test1/app1-profile1.p7b" -keyPwd 123456' 249 ], 250 'verify-profile': [ 251 'verify-profile -inFile "./test1/app1-profile1.p7b"' 252 ] 253} 254 255 256def get_test_scope_from_file(): 257 with open('commands.config', 'r', encoding='utf-8') as f: 258 content = f.read() 259 return ast.literal_eval(content) 260 261test_scope = get_test_scope_from_file() 262 263test_result = {} 264 265 266def run_target(case, cmd): 267 if not test_result.get(case, None): 268 test_result[case] = {'times': 0, 'total_cost': 0, 'success': 0, 'fail': 0} 269 270 case_result = test_result.get(case) 271 case_result['times'] = case_result['times'] + 1 272 start = time.time() 273 274 command = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, shell=False) 275 276 277 278 279 out = command.stdout.readlines() 280 with open("log.txt", mode='a+', encoding='utf-8') as f: 281 if len(out) > 0: 282 f.writelines(cmd + "\r\n") 283 for line in out: 284 f.writelines(str(line.strip()) + "\r\n") 285 286 success = True 287 error = command.stderr.readlines() 288 with open("error.txt", mode='a+', encoding='utf-8') as f: 289 if len(error) > 0: 290 f.writelines(cmd + "\r\n") 291 for line in error: 292 f.writelines(str(line.strip()) + "\r\n") 293 294 code = command.wait() 295 if code != 0: 296 success = False 297 end = time.time() 298 case_result['total_cost'] = case_result['total_cost'] + (end - start) 299 300 if success: 301 case_result['success'] = case_result['success'] + 1 302 else: 303 case_result['fail'] = case_result['fail'] + 1 304 return success 305 306 307def run_simple_case(case, jar_file): 308 test_case = simple_scope.get(case, None) 309 if not test_case: 310 print("Not found test case: {}".format(case)) 311 exit(0) 312 313 for k in test_case: 314 cmd = 'java -jar {} {}'.format(jar_file, k) 315 print("== Run command: {}".format(cmd)) 316 result = run_target(case, cmd) 317 print("== Done command: {}".format(result)) 318 319 320def run_test_case(case, jar_file): 321 test_case = test_scope.get(case, None) 322 if not test_case: 323 print("Not found test case: {}".format(case)) 324 exit(0) 325 326 327 328 for k in test_case: 329 cmd = 'java -jar {} {}'.format(jar_file, k) 330 print("== Run command: {}".format(cmd)) 331 result = run_target(case, cmd) 332 333 with open('test_result.log', 'r', encoding='utf-8') as f: 334 content = f.read() 335 test_result_dict = ast.literal_eval(content) 336 337 338 if case == 'case-assert-true': 339 if result: 340 print("== Done command: Expected True and tested True") 341 else: 342 test_result_dict['commands_expected_True_but_tested_False'].append(cmd) 343 print("== Done command: Expected True but tested False") 344 else: 345 if result: 346 test_result_dict['commands_expected_False_but_tested_True'].append(cmd) 347 print("== Done command: Expected False but tested True") 348 else: 349 print("== Done command: Expected False and tested False") 350 351 352 with open("test_result.log", mode='w', encoding='utf-8') as tr: 353 tr.write(json.dumps(test_result_dict, indent=4)) 354 355 356def random_str(): 357 strs = "abcdefghjiklmnopqstuvwxyzABCDEFGHIJKLMNOPQRS TUVWXYZ1234567890~!@#ls%^&*()_+,./<>?;':" 358 result = '' 359 for i in range(random.randint(1, 30)): 360 result = result + random.choice(strs) 361 return result 362 363 364def run_random_case(case, jar_file): 365 test_case = random_scope.get(case, None) 366 if not test_case: 367 print("Not found test case: {}".format(case)) 368 exit(0) 369 370 cmd = 'java -jar {} {}'.format(jar_file, case) 371 for k, v in test_case.get('required').items(): 372 r = random.choice(['none', 'choice', 'choice', 'random']) 373 if r == 'choice': 374 cmd = cmd + ' -{} "{}" '.format(k, random.choice(v) if isinstance(v, list) else v) 375 elif r == 'random': 376 cmd = cmd + ' -{} "{}" '.format(k, random_str()) 377 378 for k, v in test_case.get('others').items(): 379 r = random.choice(['none', 'choice', 'choice', 'random']) 380 if r == 'choice': 381 cmd = cmd + ' -{} "{}" '.format(k, random.choice(v) if isinstance(v, list) else v) 382 elif r == 'random': 383 cmd = cmd + ' -{} "{}" '.format(k, random_str()) 384 385 print("== Run command: {}".format(cmd)) 386 result = run_target(case, cmd) 387 print("== Done command: {}".format(result)) 388 389 390def run_all_case(case, jar_file): 391 test_case = random_scope.get(case, None) 392 if not test_case: 393 print("Not found test case: {}".format(case)) 394 exit(0) 395 396 cmd = 'java -jar {} {}'.format(jar_file, case) 397 for ak, av in test_case.get('required').items(): 398 cmd = cmd + ' -{} "{}" '.format(ak, random.choice(av) if isinstance(av, list) else av) 399 400 print("== Run command: {}".format(cmd)) 401 result = run_target(case, cmd) 402 print("== Done command: {}".format(result)) 403 404 405def prepare_env(): 406 test_dirs = ['test1', 'test2'] 407 for test_dir in test_dirs: 408 if not os.path.exists(test_dir): 409 os.mkdir(test_dir) 410 411 for key_file in ['ohtest.jks', 'ohtest.p12']: 412 target_file = os.path.join(test_dir, key_file) 413 if os.path.exists(target_file): 414 os.remove(target_file) 415 if os.path.exists(key_file): 416 os.remove(key_file) 417 418 419def process_cmd(args): 420 run_round: int = 1 421 run_scope: str = 'simple' 422 is_random: bool = False 423 424 if len(args) <= 1 or ('.jar' not in args[1]) or '--help' == args[1] or '-h' == args[1]: 425 print_help() 426 exit(0) 427 428 jar_file = args[1] 429 if not os.path.exists(jar_file): 430 print("Jar file '{}' not found".format(jar_file)) 431 exit(0) 432 433 if len(args) >= 3: 434 temp_round, temp_scope, temp_random = get_run_format(args) 435 run_round: int = temp_round 436 run_scope: str = temp_scope 437 is_random: bool = temp_random 438 439 print('=== Start testing ===') 440 print('Scope: {}. Round: {}. Random: {}'.format(run_scope, run_round, is_random)) 441 442 if os.path.exists('log.txt'): 443 os.remove('log.txt') 444 if os.path.exists('error.txt'): 445 os.remove('error.txt') 446 447 for i in range(run_round): 448 if run_scope == 'all': 449 for r_scope, _ in random_scope.items(): 450 run_all_case(r_scope, jar_file) 451 elif is_random: 452 for r_scope, _ in random_scope.items(): 453 run_random_case(r_scope, jar_file) 454 elif run_scope == 'simple': 455 prepare_env() 456 for s_scope, _ in simple_scope.items(): 457 run_simple_case(s_scope, jar_file) 458 elif run_scope == 'runtest': 459 prepare_env() 460 with open("test_result.log", mode='w', encoding='utf-8') as file_result: 461 result_dict = { 462 'commands_expected_True_but_tested_False': [], 463 'commands_expected_False_but_tested_True': [] 464 } 465 file_result.write(str(result_dict)) 466 for t_scope, _ in test_scope.items(): 467 run_test_case(t_scope, jar_file) 468 else: 469 run_simple_case(run_scope, jar_file) 470 471 472def get_run_format(args): 473 run_round: int = 1 474 run_scope: str = 'simple' 475 is_random: bool = False 476 try: 477 for i in range(2, len(args), 1): 478 if args[i] == '-n': 479 run_round = int(args[i + 1]) 480 elif args[i] == '-scope': 481 run_scope = args[i + 1] 482 elif args[i] == '--random': 483 is_random = True 484 elif args[i] == '-runtest': 485 run_scope = 'runtest' 486 except IndexError: 487 print_help() 488 exit(0) 489 return run_round, run_scope, is_random 490if __name__ == '__main__': 491 process_cmd(sys.argv) 492 print("All test done") 493 print("========================") 494 for rk, rv in test_result.items(): 495 times = rv['times'] 496 if times != 0: 497 print("Case {}, run times: {}, avg cost: {}s, total success: {}, total fail: {}".format(rk, times, round( 498 rv['total_cost'] / times, 2), rv['success'], rv['fail'])) 499 print("========================") 500 print("See log.txt / error.txt") 501