1// Protocol messages for describing the results of benchmarks and unit tests. 2syntax = "proto3"; 3 4import "google/protobuf/any.proto"; 5 6option cc_enable_arenas = true; 7option java_outer_classname = "TestLogProtos"; 8option java_multiple_files = true; 9option java_package = "org.tensorflow.util.testlog"; 10 11package tensorflow; 12 13message EntryValue { 14 oneof kind { 15 double double_value = 1; 16 string string_value = 2; 17 } 18}; 19 20// Each unit test or benchmark in a test or benchmark run provides 21// some set of information. Here we provide some reasonable keys 22// one would expect to see, with optional key/value pairs for things 23// we haven't considered. 24// 25// This BenchmarkEntry should be emitted by each unit test or benchmark 26// reporter. 27message BenchmarkEntry { 28 // The name of the specific benchmark or test 29 // (e.g. BM_AdjustContrast_gpu_B_W_H) 30 string name = 1; 31 32 // If a benchmark, how many iterations it was run for 33 int64 iters = 2; 34 35 // Total cpu time used for all iterations (in seconds) 36 double cpu_time = 3; 37 38 // Total wall time used for all iterations (in seconds) 39 double wall_time = 4; 40 41 // Throughput (in MB/s) 42 double throughput = 5; 43 44 // Generic map from result key to value. 45 map<string, EntryValue> extras = 6; 46}; 47 48message BenchmarkEntries { 49 repeated BenchmarkEntry entry = 1; 50} 51 52message BuildConfiguration { 53 string mode = 1; // opt, dbg, etc 54 repeated string cc_flags = 2; // CC compiler flags, if known 55 repeated string opts = 3; // Bazel compilation options, if known 56}; 57 58message CommitId { 59 oneof kind { 60 // Submitted changelist. 61 int64 changelist = 1; 62 string hash = 2; 63 } 64 // Hash of intermediate change between hash/changelist and what was tested. 65 // Not used if the build is from a commit without modifications. 66 string snapshot = 3; 67 // Changelist tested if the change list is not already submitted. 68 int64 pending_changelist = 4; 69}; 70 71message CPUInfo { 72 int64 num_cores = 1; 73 74 int64 num_cores_allowed = 2; 75 76 // How fast are these cpus? 77 double mhz_per_cpu = 3; 78 79 // Additional cpu information. For example, 80 // Intel Ivybridge with HyperThreading (24 cores) dL1:32KB dL2:256KB dL3:30MB 81 string cpu_info = 4; 82 83 // What kind of cpu scaling is enabled on the host. 84 // Examples include "performance", "ondemand", "conservative", "mixed". 85 string cpu_governor = 5; 86 87 // Cache sizes (in bytes), e.g. "L2": 262144 (for 256KB) 88 map<string, int64> cache_size = 6; 89}; 90 91message MemoryInfo { 92 int64 total = 1; // Total virtual memory in bytes 93 int64 available = 2; // Immediately available memory in bytes 94} 95 96message GPUInfo { 97 string model = 1; // e.g. "Tesla K40c" 98 string uuid = 2; // Final entry in output of "nvidia-smi -L" 99 string bus_id = 3; // e.g. "0000:04:00.0" 100}; 101 102message PlatformInfo { 103 string bits = 1; // e.g. '64bit' 104 string linkage = 2; // e.g. 'ELF' 105 string machine = 3; // e.g. 'i386' 106 string release = 4; // e.g. '3.13.0-76-generic' 107 string system = 5; // e.g. 'Linux' 108 string version = 6; // e.g. '#120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016' 109}; 110 111message AvailableDeviceInfo { // Matches DeviceAttributes 112 string name = 1; // Device name. 113 string type = 2; // Device type, e.g. 'CPU' or 'GPU'. 114 int64 memory_limit = 3; // Memory capacity in bytes. 115 string physical_description = 4; // The physical description of this device. 116}; 117 118message MachineConfiguration { 119 // Host name of machine that ran the benchmark. 120 string hostname = 1; 121 122 // Unique serial number of the machine. 123 string serial_identifier = 7; 124 125 // Additional platform information. 126 PlatformInfo platform_info = 2; 127 128 // CPU Information. 129 CPUInfo cpu_info = 3; 130 131 // Other devices that are attached and relevant (e.g. GPUInfo). 132 repeated google.protobuf.Any device_info = 4; 133 134 // Devices accessible to the test (e.g. as given by list_local_devices). 135 repeated AvailableDeviceInfo available_device_info = 5; 136 137 MemoryInfo memory_info = 6; 138}; 139 140// Run-specific items such as arguments to the test / benchmark. 141message RunConfiguration { 142 repeated string argument = 1; 143} 144 145// The output of one benchmark / test run. Each run contains a list of 146// tests or benchmarks, stored as BenchmarkEntry messages. 147// 148// This message should be emitted by the reporter (which runs the 149// test / BM in a subprocess and then reads the emitted BenchmarkEntry messages; 150// usually from a serialized json file, finally collecting them along 151// with additional information about the test run. 152message TestResults { 153 // The target of the run, e.g.: 154 // //tensorflow/core:kernels_adjust_contrast_op_benchmark_test 155 string target = 1; 156 157 // The list of tests or benchmarks in this run. 158 BenchmarkEntries entries = 2; 159 160 // The configuration of the build (compiled opt? with cuda? any copts?) 161 BuildConfiguration build_configuration = 3; 162 163 // The commit id (git hash or changelist) 164 CommitId commit_id = 4; 165 166 // The time the run started (in seconds of UTC time since Unix epoch) 167 int64 start_time = 5; 168 169 // The amount of time the total run took (wall time in seconds) 170 double run_time = 6; 171 172 // Machine-specific parameters (Platform and CPU info) 173 MachineConfiguration machine_configuration = 7; 174 175 // Run-specific parameters (arguments, etc) 176 RunConfiguration run_configuration = 8; 177 178 // Benchmark target identifier. 179 string name = 9; 180 181 // The type of benchmark. 182 enum BenchmarkType { 183 UNKNOWN = 0; // Fallback for protos written before Type was introduced. 184 CPP_MICROBENCHMARK = 1; 185 PYTHON_BENCHMARK = 2; 186 ANDROID_BENCHMARK = 3; 187 } 188 BenchmarkType benchmark_type = 10; 189 190 // Used for differentiating between continuous and debug builds. 191 // Must be one of: 192 // * cbuild: results from continuous build. 193 // * presubmit: results from oneshot requests. 194 // * culprit: results from culprit finder rerun. 195 string run_mode = 11; 196}; 197