• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Protocol messages for describing the results of benchmarks and unit tests.
2syntax = "proto3";
3
4import "google/protobuf/any.proto";
5
6option cc_enable_arenas = true;
7option java_outer_classname = "TestLogProtos";
8option java_multiple_files = true;
9option java_package = "org.tensorflow.util.testlog";
10
11package tensorflow;
12
13message EntryValue {
14  oneof kind {
15    double double_value = 1;
16    string string_value = 2;
17  }
18};
19
20// Each unit test or benchmark in a test or benchmark run provides
21// some set of information.  Here we provide some reasonable keys
22// one would expect to see, with optional key/value pairs for things
23// we haven't considered.
24//
25// This BenchmarkEntry should be emitted by each unit test or benchmark
26// reporter.
27message BenchmarkEntry {
28  // The name of the specific benchmark or test
29  // (e.g. BM_AdjustContrast_gpu_B_W_H)
30  string name = 1;
31
32  // If a benchmark, how many iterations it was run for
33  int64 iters = 2;
34
35  // Total cpu time used for all iterations (in seconds)
36  double cpu_time = 3;
37
38  // Total wall time used for all iterations (in seconds)
39  double wall_time = 4;
40
41  // Throughput (in MB/s)
42  double throughput = 5;
43
44  // Generic map from result key to value.
45  map<string, EntryValue> extras = 6;
46};
47
48message BenchmarkEntries {
49  repeated BenchmarkEntry entry = 1;
50}
51
52message BuildConfiguration {
53  string mode = 1;               // opt, dbg, etc
54  repeated string cc_flags = 2;  // CC compiler flags, if known
55  repeated string opts = 3;      // Bazel compilation options, if known
56};
57
58message CommitId {
59  oneof kind {
60    // Submitted changelist.
61    int64 changelist = 1;
62    string hash = 2;
63  }
64  // Hash of intermediate change between hash/changelist and what was tested.
65  // Not used if the build is from a commit without modifications.
66  string snapshot = 3;
67  // Changelist tested if the change list is not already submitted.
68  int64 pending_changelist = 4;
69};
70
71message CPUInfo {
72  int64 num_cores = 1;
73
74  int64 num_cores_allowed = 2;
75
76  // How fast are these cpus?
77  double mhz_per_cpu = 3;
78
79  // Additional cpu information. For example,
80  // Intel Ivybridge with HyperThreading (24 cores) dL1:32KB dL2:256KB dL3:30MB
81  string cpu_info = 4;
82
83  // What kind of cpu scaling is enabled on the host.
84  // Examples include "performance", "ondemand", "conservative", "mixed".
85  string cpu_governor = 5;
86
87  // Cache sizes (in bytes), e.g. "L2": 262144 (for 256KB)
88  map<string, int64> cache_size = 6;
89};
90
91message MemoryInfo {
92  int64 total = 1;      // Total virtual memory in bytes
93  int64 available = 2;  // Immediately available memory in bytes
94}
95
96message GPUInfo {
97  string model = 1;  // e.g. "Tesla K40c"
98  string uuid = 2;   // Final entry in output of "nvidia-smi -L"
99  string bus_id = 3;  // e.g. "0000:04:00.0"
100};
101
102message PlatformInfo {
103  string bits = 1;       // e.g. '64bit'
104  string linkage = 2;    // e.g. 'ELF'
105  string machine = 3;    // e.g. 'i386'
106  string release = 4;    // e.g. '3.13.0-76-generic'
107  string system = 5;     // e.g. 'Linux'
108  string version = 6;    // e.g. '#120-Ubuntu SMP Mon Jan 18 15:59:10 UTC 2016'
109};
110
111message AvailableDeviceInfo {       // Matches DeviceAttributes
112  string name = 1;                  // Device name.
113  string type = 2;                  // Device type, e.g. 'CPU' or 'GPU'.
114  int64 memory_limit = 3;           // Memory capacity in bytes.
115  string physical_description = 4;  // The physical description of this device.
116};
117
118message MachineConfiguration {
119  // Host name of machine that ran the benchmark.
120  string hostname = 1;
121
122  // Unique serial number of the machine.
123  string serial_identifier = 7;
124
125  // Additional platform information.
126  PlatformInfo platform_info = 2;
127
128  // CPU Information.
129  CPUInfo cpu_info = 3;
130
131  // Other devices that are attached and relevant (e.g. GPUInfo).
132  repeated google.protobuf.Any device_info = 4;
133
134  // Devices accessible to the test (e.g. as given by list_local_devices).
135  repeated AvailableDeviceInfo available_device_info = 5;
136
137  MemoryInfo memory_info = 6;
138};
139
140// Run-specific items such as arguments to the test / benchmark.
141message RunConfiguration {
142  repeated string argument = 1;
143}
144
145// The output of one benchmark / test run.  Each run contains a list of
146// tests or benchmarks, stored as BenchmarkEntry messages.
147//
148// This message should be emitted by the reporter (which runs the
149// test / BM in a subprocess and then reads the emitted BenchmarkEntry messages;
150// usually from a serialized json file, finally collecting them along
151// with additional information about the test run.
152message TestResults {
153  // The target of the run, e.g.:
154  //  //tensorflow/core:kernels_adjust_contrast_op_benchmark_test
155  string target = 1;
156
157  // The list of tests or benchmarks in this run.
158  BenchmarkEntries entries = 2;
159
160  // The configuration of the build (compiled opt? with cuda? any copts?)
161  BuildConfiguration build_configuration = 3;
162
163  // The commit id (git hash or changelist)
164  CommitId commit_id = 4;
165
166  // The time the run started (in seconds of UTC time since Unix epoch)
167  int64 start_time = 5;
168
169  // The amount of time the total run took (wall time in seconds)
170  double run_time = 6;
171
172  // Machine-specific parameters (Platform and CPU info)
173  MachineConfiguration machine_configuration = 7;
174
175  // Run-specific parameters (arguments, etc)
176  RunConfiguration run_configuration = 8;
177
178  // Benchmark target identifier.
179  string name = 9;
180
181  // The type of benchmark.
182  enum BenchmarkType {
183    UNKNOWN = 0;  // Fallback for protos written before Type was introduced.
184    CPP_MICROBENCHMARK = 1;
185    PYTHON_BENCHMARK = 2;
186    ANDROID_BENCHMARK = 3;
187  }
188  BenchmarkType benchmark_type = 10;
189
190  // Used for differentiating between continuous and debug builds.
191  // Must be one of:
192  // * cbuild: results from continuous build.
193  // * presubmit: results from oneshot requests.
194  // * culprit: results from culprit finder rerun.
195  string run_mode = 11;
196};
197