1#!/usr/bin/env python 2# 3# Copyright (C) 2016 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16# 17 18import logging 19 20from vts.proto import VtsReportMessage_pb2 as ReportMsg 21from vts.runners.host import asserts 22from vts.runners.host import base_test 23from vts.runners.host import const 24from vts.runners.host import test_runner 25from vts.utils.python.controllers import android_device 26from vts.utils.python.cpu import cpu_frequency_scaling 27 28# number of threads to use when running the throughput tests on target. 29_THREAD_LIST = [2, 3, 4, 5, 7, 10, 30, 50, 70, 100, 200] 30 31_ITERATIONS_PER_SECOND = "iterations_per_second" 32_TIME_AVERAGE = "time_average" 33_TIME_WORST = "time_worst" 34_TIME_BEST = "time_best" 35_TIME_PERCENTILE = "time_percentile" 36 37 38class BinderThroughputBenchmark(base_test.BaseTestClass): 39 """A test case for the binder throughput benchmarking.""" 40 41 def setUpClass(self): 42 self.dut = self.android_devices[0] 43 self._cpu_freq = cpu_frequency_scaling.CpuFrequencyScalingController(self.dut) 44 self._cpu_freq.DisableCpuScaling() 45 46 def setUp(self): 47 self._cpu_freq.SkipIfThermalThrottling(retry_delay_secs=30) 48 49 def tearDown(self): 50 self._cpu_freq.SkipIfThermalThrottling() 51 52 def tearDownClass(self): 53 self._cpu_freq.EnableCpuScaling() 54 55 def testRunBenchmark32Bit(self): 56 """A test case which runs the 32-bit benchmark.""" 57 self.RunBenchmarkAndReportResult(32) 58 59 def testRunBenchmark64Bit(self): 60 """A test case which runs the 64-bit benchmark.""" 61 self.RunBenchmarkAndReportResult(64) 62 63 def RunBenchmarkAndReportResult(self, bits): 64 """Runs the native binary and stores its result to the web DB. 65 66 Args: 67 bits: integer (32 or 64), the number of bits in a word chosen 68 at the compile time (e.g., 32- vs. 64-bit library). 69 """ 70 labels = [] 71 iterations_per_second = [] 72 time_average = [] 73 time_best = [] 74 time_worst = [] 75 time_percentile_50 = [] 76 time_percentile_90 = [] 77 time_percentile_95 = [] 78 time_percentile_99 = [] 79 80 for thread in _THREAD_LIST: 81 result = self.RunBenchmark(bits, thread) 82 labels.append("%s_thread" % thread) 83 iterations_per_second.append(result["iterations_per_second"]) 84 time_average.append(result["time_average"]) 85 time_best.append(result["time_best"]) 86 time_worst.append(result["time_worst"]) 87 time_percentile_50.append(result["time_percentile"][50]) 88 time_percentile_90.append(result["time_percentile"][90]) 89 time_percentile_95.append(result["time_percentile"][95]) 90 time_percentile_99.append(result["time_percentile"][99]) 91 92 # To upload to the web DB. 93 self.web.AddProfilingDataLabeledVector( 94 "binder_throughput_iterations_per_second_%sbits" % bits, 95 labels, iterations_per_second, x_axis_label="Number of Threads", 96 y_axis_label="Binder RPC Iterations Per Second", 97 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 98 99 self.web.AddProfilingDataLabeledVector( 100 "binder_throughput_time_average_ns_%sbits" % bits, 101 labels, time_average, x_axis_label="Number of Threads", 102 y_axis_label="Binder RPC Time - Average (nanoseconds)", 103 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 104 self.web.AddProfilingDataLabeledVector( 105 "binder_throughput_time_best_ns_%sbits" % bits, 106 labels, time_best, x_axis_label="Number of Threads", 107 y_axis_label="Binder RPC Time - Best Case (nanoseconds)") 108 self.web.AddProfilingDataLabeledVector( 109 "binder_throughput_time_worst_ns_%sbits" % bits, 110 labels, time_worst, x_axis_label="Number of Threads", 111 y_axis_label="Binder RPC Time - Worst Case (nanoseconds)", 112 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 113 114 self.web.AddProfilingDataLabeledVector( 115 "binder_throughput_time_50percentile_ns_%sbits" % bits, 116 labels, time_percentile_50, x_axis_label="Number of Threads", 117 y_axis_label="Binder RPC Time - 50 Percentile (nanoseconds)", 118 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 119 self.web.AddProfilingDataLabeledVector( 120 "binder_throughput_time_90percentile_ns_%sbits" % bits, 121 labels, time_percentile_90, x_axis_label="Number of Threads", 122 y_axis_label="Binder RPC Time - 90 Percentile (nanoseconds)", 123 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 124 self.web.AddProfilingDataLabeledVector( 125 "binder_throughput_time_95percentile_ns_%sbits" % bits, 126 labels, time_percentile_95, x_axis_label="Number of Threads", 127 y_axis_label="Binder RPC Time - 95 Percentile (nanoseconds)", 128 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 129 self.web.AddProfilingDataLabeledVector( 130 "binder_throughput_time_99percentile_ns_%sbits" % bits, 131 labels, time_percentile_99, x_axis_label="Number of Threads", 132 y_axis_label="Binder RPC Time - 99 Percentile (nanoseconds)", 133 regression_mode=ReportMsg.VTS_REGRESSION_MODE_DISABLED) 134 135 def RunBenchmark(self, bits, threads): 136 """Runs the native binary and parses its result. 137 138 Args: 139 bits: integer (32 or 64), the number of bits in a word chosen 140 at the compile time (e.g., 32- vs. 64-bit library). 141 threads: positive integer, the number of threads to use. 142 143 Returns: 144 a dict which contains the benchmarking result where the keys are: 145 'iterations_per_second', 'time_average', 'time_worst', 146 'time_best', 'time_percentile'. 147 """ 148 # Runs the benchmark. 149 logging.info("Start to run the benchmark (%s bit mode)", bits) 150 binary = "/data/local/tmp/%s/binderThroughputTest%s" % (bits, bits) 151 152 results = self.dut.shell.Execute( 153 ["chmod 755 %s" % binary, 154 "LD_LIBRARY_PATH=/data/local/tmp/%s/hw:" 155 "/data/local/tmp/%s:" 156 "$LD_LIBRARY_PATH %s -w %s" % (bits, bits, binary, threads)]) 157 158 # Parses the result. 159 asserts.assertEqual(len(results[const.STDOUT]), 2) 160 logging.info("stderr: %s", results[const.STDERR][1]) 161 stdout_lines = results[const.STDOUT][1].split("\n") 162 logging.info("stdout: %s", stdout_lines) 163 164 asserts.assertFalse( 165 any(results[const.EXIT_CODE]), 166 "testRunBenchmark%sBit(%s thread) failed." % (bits, threads)) 167 168 # To upload to the web DB. 169 summary = {} 170 index = next(i for i, string in enumerate(stdout_lines) 171 if "iterations per sec:" in string) 172 summary[_ITERATIONS_PER_SECOND] = int(float( 173 stdout_lines[index].replace("iterations per sec: ", ""))) 174 # an example is 'iterations per sec: 34868.7' 175 176 index = next(i for i, string in enumerate(stdout_lines) 177 if "average:" in string) 178 stats_string = stdout_lines[index].split() 179 # an example is 'average:0.0542985ms worst:0.314584ms best:0.02651ms' 180 summary[_TIME_AVERAGE] = int(float( 181 stats_string[0].replace( 182 "average:", "").replace("ms", "")) * 1000000) 183 summary[_TIME_WORST] = int(float( 184 stats_string[1].replace("worst:", "").replace("ms", "")) * 1000000) 185 summary[_TIME_BEST] = int(float( 186 stats_string[2].replace("best:", "").replace("ms", "")) * 1000000) 187 188 index = next(i for i, string in enumerate(stdout_lines) 189 if "50%: " in string) 190 percentiles_string = stdout_lines[index].split() 191 summary[_TIME_PERCENTILE] = {} 192 summary[_TIME_PERCENTILE][50] = int(float(percentiles_string[1]) 193 * 1000000) 194 summary[_TIME_PERCENTILE][90] = int(float(percentiles_string[3]) 195 * 1000000) 196 summary[_TIME_PERCENTILE][95] = int(float(percentiles_string[5]) 197 * 1000000) 198 summary[_TIME_PERCENTILE][99] = int(float(percentiles_string[7]) 199 * 1000000) 200 return summary 201 202if __name__ == "__main__": 203 test_runner.main() 204