1 #include "dynamicproto.h"
2
3 #include "proto/tuningfork.pb.h"
4 #include "proto/tuningfork_clearcut_log.pb.h"
5 #include <google/protobuf/text_format.h>
6 #include <google/protobuf/io/zero_copy_stream_impl.h>
7
8 #include <iostream>
9 #include <fstream>
10 #include <numeric>
11 #include <functional>
12 #include <algorithm>
13 #include <random>
14 #include <sstream>
15
16 using namespace google::protobuf;
17 using namespace google::protobuf::io;
18
19 using ::com::google::tuningfork::FidelityParams;
20 using ::com::google::tuningfork::Annotation;
21 using ::com::google::tuningfork::Settings;
22 using ::com::google::tuningfork::Settings_Histogram;
23
24 using ::logs::proto::tuningfork::TuningForkLogEvent;
25 using ::logs::proto::tuningfork::TuningForkHistogram;
26
27 namespace {
28
29 std::default_random_engine generator;
30 std::uniform_real_distribution<double> zero_one_random_double_dist(0,1);
31
ZeroToOneRandomDouble()32 double ZeroToOneRandomDouble() {
33 return zero_one_random_double_dist(generator);
34 }
35
36 std::uniform_int_distribution<int32_t> int32_dist;
RandInt32()37 int32_t RandInt32() {
38 return int32_dist(generator);
39 }
40
Usage()41 void Usage() {
42 std::cerr << "Usage: sztool <your_tf_proto_name> <your_settings>\n";
43 }
44
45 constexpr size_t ERROR = -1;
46
FieldSize(const FieldDescriptor * fd,bool outputTypes)47 size_t FieldSize(const FieldDescriptor* fd, bool outputTypes) {
48 switch(fd->type()) {
49 case FieldDescriptor::TYPE_BOOL:
50 if(outputTypes)
51 std::cout << fd->name() << " " << fd->type_name() << std::endl;
52 // Do we want this?
53 return 2;
54 case FieldDescriptor::TYPE_ENUM:
55 if(outputTypes)
56 std::cout << fd->name() << " ";
57 dynamicproto::print(fd->enum_type(), std::cerr);
58 if(outputTypes)
59 std::cerr << std::endl;
60 // NB each field is optional, so there is another possibility,
61 // with the value unknown.
62 return fd->enum_type()->value_count() + 1;
63 default:
64 return ERROR;
65 }
66 }
67
RandomId()68 string RandomId() {
69 const int ID_LEN = 32;
70 char id[ID_LEN];
71 for(int i=0;i<ID_LEN-1;++i)
72 id[i] = 32+(RandInt32()%90);
73 id[ID_LEN-1]=0;
74 return id;
75 }
76
77 const string ANNOTATION_MESSAGE_NAME = "Annotation";
78 const string FIDELITYPARAMS_MESSAGE_NAME = "FidelityParams";
NextAnnotation(const std::vector<const FieldDescriptor * > & ann_extensions,std::vector<int> & ann_values)79 void NextAnnotation(const std::vector<const FieldDescriptor*>& ann_extensions,
80 std::vector<int>& ann_values) {
81 for(int i=0; i<ann_extensions.size(); ++i) {
82 auto fd = ann_extensions[i];
83 if(fd->type()==FieldDescriptor::TYPE_BOOL) {
84 if(ann_values[i]==0) {
85 ann_values[i]++;
86 return;
87 }
88 }
89 else {
90 if(ann_values[i]<fd->enum_type()->value_count()) {
91 ann_values[i]++;
92 return;
93 }
94 }
95 ann_values[i] = 0;
96 }
97 }
98
99 Annotation
MakeAnnotation(const FileDescriptor * fdesc,const std::vector<const FieldDescriptor * > & ann_extensions,const std::vector<int> & ann_values)100 MakeAnnotation(const FileDescriptor* fdesc,
101 const std::vector<const FieldDescriptor*>& ann_extensions,
102 const std::vector<int>& ann_values) {
103 Annotation a;
104 Message* m = dynamicproto::newMessage(fdesc, ANNOTATION_MESSAGE_NAME);
105 if(m) {
106 const Reflection* r = m->GetReflection();
107 for(int i=0; i<ann_extensions.size(); ++i) {
108 auto fd = ann_extensions[i];
109 if(fd->type()==FieldDescriptor::TYPE_BOOL) {
110 r->SetBool(m, fd, ann_values[i]!=0);
111 }
112 else {
113 // 0 means it's not set
114 if(ann_values[i]>0)
115 r->SetEnumValue(m, fd, ann_values[i]);
116 }
117 }
118 std::string s;
119 m->SerializeToString(&s);
120 a.ParseFromString(s);
121 delete m;
122 }
123 else
124 std::cerr << "Warning, could not find " << ANNOTATION_MESSAGE_NAME
125 << " in " << fdesc->name() << std::endl;
126 return a;
127 }
128
129 FidelityParams
RandomFidelityParams(const FileDescriptor * fdesc,const std::vector<const FieldDescriptor * > & fpExtensions)130 RandomFidelityParams(const FileDescriptor* fdesc,
131 const std::vector<const FieldDescriptor*>& fpExtensions) {
132 FidelityParams p;
133 Message* m = dynamicproto::newMessage(fdesc, FIDELITYPARAMS_MESSAGE_NAME);
134 if(m) {
135 const Reflection* r = m->GetReflection();
136 for(int i=0; i<fpExtensions.size(); ++i) {
137 auto fd = fpExtensions[i];
138 switch(fd->type()) {
139 case FieldDescriptor::TYPE_BOOL:
140 r->SetBool(m, fd, RandInt32()%2);
141 break;
142 case FieldDescriptor::TYPE_ENUM:
143 r->SetEnumValue(m, fd,
144 1 + (RandInt32() % fd->enum_type()->value_count()));
145 break;
146 case FieldDescriptor::TYPE_INT32:
147 r->SetInt32(m, fd, RandInt32());
148 break;
149 case FieldDescriptor::TYPE_INT64:
150 r->SetInt64(m, fd, RandInt32());
151 break;
152 case FieldDescriptor::TYPE_STRING:
153 r->SetString(m, fd, RandomId());
154 break;
155 default:
156 // Ignore the rest
157 break;
158 }
159 }
160 std::string s;
161 m->SerializeToString(&s);
162 p.ParseFromString(s);
163 delete m;
164 }
165 else
166 std::cerr << "Warning, could not find " << FIDELITYPARAMS_MESSAGE_NAME
167 << " in " << fdesc->name() << std::endl;
168 return p;
169 }
170
171 } // namespace
172
main(int argc,char * argv[])173 int main(int argc, char *argv[]) {
174
175 using namespace std::placeholders; // for _1, _2, _3...
176
177 if(argc!=3) {
178 Usage();
179 return 1;
180 }
181 dynamicproto::init({".", "proto"});
182 auto fdescs = dynamicproto::fileDescriptors(argv[1]);
183 if(fdescs.size()<2) {
184 std::cerr << "The proto file has errors: it must include one import "
185 << "of tuningfork.proto" << std::endl;
186 return -2;
187 }
188
189 // Get the annotation extensions and count the combinations
190 std::vector<const FieldDescriptor*> ann_extensions;
191 dynamicproto::extensionsOf(fdescs.back(), "com.google.tuningfork.Annotation",
192 ann_extensions);
193 auto enums = dynamicproto::enums(fdescs.back());
194 std::vector<size_t> ann_extensionsSize(ann_extensions.size());
195 std::cout << "Annotation extensions:" << std::endl;
196 std::transform(ann_extensions.begin(), ann_extensions.end(),
197 ann_extensionsSize.begin(),
198 std::bind(FieldSize, _1, true));
199 for(auto a: ann_extensionsSize) {
200 if(a==ERROR) {
201 std::cerr << "Bad annotation (only bool and enum are supported)"
202 << std::endl;
203 return 2;
204 }
205 }
206 size_t nAnnotationCombinations =
207 std::accumulate(ann_extensionsSize.begin(),
208 ann_extensionsSize.end(),
209 1,
210 std::multiplies<size_t>());
211 std::cerr << "Total number of annotation combinations = "
212 << nAnnotationCombinations << std::endl;
213
214 // FidelityParams extensions
215 std::vector<const FieldDescriptor*> fpExtensions;
216 dynamicproto::extensionsOf(fdescs.back(),
217 "com.google.tuningfork.FidelityParams",
218 fpExtensions);
219
220 // Get the settings
221 std::string settings_file_name = argv[2];
222 ifstream settings_file(settings_file_name);
223 IstreamInputStream settings_stream(&settings_file);
224 if(!settings_file.good()) {
225 std::cerr << "Bad file: " << argv[2] << std::endl;
226 return 2;
227 }
228 Settings settings;
229 TextFormat::Parse(&settings_stream, &settings);
230 std::cout << "Read settings file " << settings_file_name << " :\n";
231 size_t maxInstrumentationKeys =
232 settings.aggregation_strategy().max_instrumentation_keys();
233
234 const double fraction_of_annotations_explored = 0.2;
235 // Fill in a clearcut log event and get its size
236 TuningForkLogEvent ev;
237 ev.set_device_id(RandomId());
238 ev.set_apk_id(RandomId());
239 *ev.mutable_fidelityparams() = RandomFidelityParams(fdescs.front(),
240 fpExtensions);
241 ev.set_experiment_id(RandomId());
242 // This assumes we have histogram settings for each instrument key, i.e. not
243 // relying on defaults
244 int n = settings.histograms_size();
245 int nset = 0;
246 for(int i=0;i<n;++i) {
247 std::vector<int> ann_values(ann_extensions.size(),0);
248 const Settings_Histogram& h = settings.histograms(i);
249 for(int c=0;c<nAnnotationCombinations; ++c) {
250 TuningForkHistogram* live_h = ev.add_histograms();
251 live_h->set_instrument_id(i);
252 *live_h->mutable_annotation() = MakeAnnotation(fdescs.front(),
253 ann_extensions,
254 ann_values);
255 NextAnnotation(ann_extensions, ann_values);
256 // Only fill in some of the histograms
257 if(ZeroToOneRandomDouble()<fraction_of_annotations_explored) {
258 for(int i=0; i<h.n_buckets(); ++i)
259 live_h->add_counts(RandInt32());
260 nset++;
261 }
262 }
263 }
264 std::cout << "Assuming " << nset << " histograms are non-zero out of "
265 << n*nAnnotationCombinations << std::endl;
266 const std::string outfname = "clearcut_evt.bin";
267 const std::string dbgoutfname = "clearcut_evt.txt";
268 std::ofstream dbgfout(dbgoutfname);
269 dbgfout << ev.DebugString() << std::endl;
270 std::ofstream fout(outfname);
271 ev.SerializeToOstream(&fout);
272 std::cout << "Wrote example clearcut message to " << outfname << " and "
273 << dbgoutfname <<std::endl;
274 std::stringstream sout;
275 ev.SerializeToOstream(&sout);
276 std::cout << "Example message size = " << sout.str().length() << " bytes"
277 << std::endl;
278 return 0;
279 }
280