1#!/usr/bin/env python3 2# Copyright (C) 2018 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15 16from __future__ import absolute_import 17from __future__ import division 18from __future__ import print_function 19 20import argparse 21import datetime 22import json 23import os 24import re 25import signal 26import sys 27 28ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 29sys.path.append(os.path.join(ROOT_DIR)) 30 31from python.generators.diff_tests.testing import TestType 32from python.generators.diff_tests.utils import ctrl_c_handler 33from python.generators.diff_tests.runner import DiffTestsRunner 34 35 36def main(): 37 signal.signal(signal.SIGINT, ctrl_c_handler) 38 parser = argparse.ArgumentParser() 39 parser.add_argument('--test-type', type=str, default='all') 40 parser.add_argument('--trace-descriptor', type=str) 41 parser.add_argument('--metrics-descriptor', type=str) 42 parser.add_argument('--perf-file', type=str) 43 parser.add_argument( 44 '--name-filter', 45 default='.*', 46 type=str, 47 help='Filter the name of the tests to run (regex syntax)') 48 parser.add_argument( 49 '--keep-input', 50 action='store_true', 51 help='Save the (generated) input pb file for debugging') 52 parser.add_argument( 53 '--rebase', 54 action='store_true', 55 help='Update the expected output file with the actual result') 56 parser.add_argument( 57 '--no-colors', action='store_true', help='Print without coloring') 58 parser.add_argument( 59 'trace_processor', type=str, help='location of trace processor binary') 60 args = parser.parse_args() 61 62 test_runner = DiffTestsRunner(args.name_filter, args.trace_processor, 63 args.trace_descriptor, args.no_colors) 64 sys.stderr.write(f"[==========] Running {len(test_runner.tests)} tests.\n") 65 66 results = test_runner.run_all_tests(args.metrics_descriptor, args.keep_input, 67 args.rebase) 68 sys.stderr.write(results.str(args.no_colors, len(test_runner.tests))) 69 70 if args.rebase: 71 sys.stderr.write(results.rebase_str()) 72 73 if len(results.test_failures) > 0: 74 return 1 75 76 if args.perf_file: 77 test_dir = os.path.join(ROOT_DIR, 'test') 78 trace_processor_dir = os.path.join(test_dir, 'trace_processor') 79 80 metrics = [] 81 sorted_data = sorted( 82 results.perf_data, key=lambda x: (x.test.type.name, x.test.name)) 83 for perf_args in sorted_data: 84 metrics.append({ 85 'metric': 'tp_perf_test_ingest_time', 86 'value': float(perf_args.ingest_time_ns) / 1.0e9, 87 'unit': 's', 88 'tags': { 89 'test_name': perf_args.test.name, 90 'test_type': perf_args.test.type.name, 91 }, 92 'labels': {}, 93 }) 94 metrics.append({ 95 'metric': 'perf_test_real_time', 96 'value': float(perf_args.real_time_ns) / 1.0e9, 97 'unit': 's', 98 'tags': { 99 'test_name': perf_args.test.name, 100 'test_type': perf_args.test.type.name, 101 }, 102 'labels': {}, 103 }) 104 105 output_data = {'metrics': metrics} 106 with open(args.perf_file, 'w+') as perf_file: 107 perf_file.write(json.dumps(output_data, indent=2)) 108 return 0 109 110 111if __name__ == '__main__': 112 sys.exit(main()) 113