1 /* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/core/lib/monitoring/collection_registry.h"
17
18 #include "tensorflow/core/lib/monitoring/counter.h"
19 #include "tensorflow/core/lib/monitoring/gauge.h"
20 #include "tensorflow/core/lib/monitoring/sampler.h"
21 #include "tensorflow/core/lib/strings/strcat.h"
22 #include "tensorflow/core/platform/protobuf.h"
23 #include "tensorflow/core/platform/test.h"
24
25 namespace tensorflow {
26 namespace monitoring {
27
28 using histogram::Histogram;
29
30 namespace test_util {
31
32 class CollectionRegistryTestAccess {
33 public:
CreateRegistry(Env * const env)34 static std::unique_ptr<CollectionRegistry> CreateRegistry(Env* const env) {
35 return std::unique_ptr<CollectionRegistry>(new CollectionRegistry(env));
36 }
37 };
38
39 } // namespace test_util
40
41 namespace {
42
EmptyCollectionFunction(MetricCollectorGetter getter)43 void EmptyCollectionFunction(MetricCollectorGetter getter) {}
44
TEST(CollectionRegistryTest,RegistrationUnregistration)45 TEST(CollectionRegistryTest, RegistrationUnregistration) {
46 auto* collection_registry = CollectionRegistry::Default();
47 const MetricDef<MetricKind::kCumulative, int64, 0> metric_def0(
48 "/tensorflow/metric0", "An example metric with no labels.");
49 const MetricDef<MetricKind::kGauge, HistogramProto, 1> metric_def1(
50 "/tensorflow/metric1", "An example metric with one label.", "LabelName");
51
52 {
53 // Enclosed in a scope so that we unregister before the stack variables
54 // above are destroyed.
55
56 std::unique_ptr<CollectionRegistry::RegistrationHandle> handle0 =
57 collection_registry->Register(&metric_def0, EmptyCollectionFunction);
58 std::unique_ptr<CollectionRegistry::RegistrationHandle> handle1 =
59 collection_registry->Register(&metric_def1, EmptyCollectionFunction);
60
61 handle0.reset();
62
63 // Able to register again because it was unregistered earlier.
64 handle0 =
65 collection_registry->Register(&metric_def0, EmptyCollectionFunction);
66 }
67 }
68
TEST(CollectionRegistryDeathTest,DuplicateRegistration)69 TEST(CollectionRegistryDeathTest, DuplicateRegistration) {
70 auto* collection_registry = CollectionRegistry::Default();
71 const MetricDef<MetricKind::kCumulative, int64, 0> metric_def(
72 "/tensorflow/metric", "An example metric with no labels.");
73
74 auto handle =
75 collection_registry->Register(&metric_def, EmptyCollectionFunction);
76 EXPECT_DEATH(
77 {
78 auto duplicate_handle =
79 collection_registry->Register(&metric_def, EmptyCollectionFunction);
80 },
81 "/tensorflow/metric");
82 }
83
TEST(CollectMetricsTest,Counter)84 TEST(CollectMetricsTest, Counter) {
85 auto counter_with_labels = std::unique_ptr<Counter<2>>(
86 Counter<2>::New("/tensorflow/test/counter_with_labels",
87 "Counter with labels.", "MyLabel0", "MyLabel1"));
88 auto counter_without_labels = std::unique_ptr<Counter<0>>(Counter<0>::New(
89 "/tensorflow/test/counter_without_labels", "Counter without labels."));
90
91 counter_with_labels->GetCell("Label00", "Label10")->IncrementBy(42);
92 counter_with_labels->GetCell("Label01", "Label11")->IncrementBy(58);
93 counter_without_labels->GetCell()->IncrementBy(7);
94
95 for (const bool collect_metric_descriptors : {true, false}) {
96 SCOPED_TRACE(strings::StrCat("collect_metric_descriptors: ",
97 collect_metric_descriptors));
98
99 auto* collection_registry = CollectionRegistry::Default();
100 CollectionRegistry::CollectMetricsOptions options;
101 options.collect_metric_descriptors = collect_metric_descriptors;
102 const std::unique_ptr<CollectedMetrics> collected_metrics =
103 collection_registry->CollectMetrics(options);
104
105 if (collect_metric_descriptors) {
106 ASSERT_GE(collected_metrics->metric_descriptor_map.size(), 2);
107
108 const MetricDescriptor& ld = *collected_metrics->metric_descriptor_map.at(
109 "/tensorflow/test/counter_with_labels");
110 EXPECT_EQ("/tensorflow/test/counter_with_labels", ld.name);
111 EXPECT_EQ("Counter with labels.", ld.description);
112 ASSERT_EQ(2, ld.label_names.size());
113 EXPECT_EQ("MyLabel0", ld.label_names[0]);
114 EXPECT_EQ("MyLabel1", ld.label_names[1]);
115 EXPECT_EQ(MetricKind::kCumulative, ld.metric_kind);
116 EXPECT_EQ(ValueType::kInt64, ld.value_type);
117
118 const MetricDescriptor& ud = *collected_metrics->metric_descriptor_map.at(
119 "/tensorflow/test/counter_without_labels");
120 EXPECT_EQ("/tensorflow/test/counter_without_labels", ud.name);
121 EXPECT_EQ("Counter without labels.", ud.description);
122 ASSERT_EQ(0, ud.label_names.size());
123 EXPECT_EQ(MetricKind::kCumulative, ud.metric_kind);
124 EXPECT_EQ(ValueType::kInt64, ud.value_type);
125 } else {
126 EXPECT_EQ(0, collected_metrics->metric_descriptor_map.size());
127 }
128
129 ASSERT_GE(collected_metrics->point_set_map.size(), 2);
130
131 const PointSet& lps = *collected_metrics->point_set_map.at(
132 "/tensorflow/test/counter_with_labels");
133 EXPECT_EQ("/tensorflow/test/counter_with_labels", lps.metric_name);
134 ASSERT_EQ(2, lps.points.size());
135 ASSERT_EQ(2, lps.points[0]->labels.size());
136 EXPECT_EQ("MyLabel0", lps.points[0]->labels[0].name);
137 EXPECT_EQ("Label00", lps.points[0]->labels[0].value);
138 EXPECT_EQ("MyLabel1", lps.points[0]->labels[1].name);
139 EXPECT_EQ("Label10", lps.points[0]->labels[1].value);
140 EXPECT_EQ(ValueType::kInt64, lps.points[0]->value_type);
141 EXPECT_EQ(42, lps.points[0]->int64_value);
142 EXPECT_LT(0, lps.points[0]->start_timestamp_millis);
143 EXPECT_LT(0, lps.points[0]->end_timestamp_millis);
144 EXPECT_GE(lps.points[0]->end_timestamp_millis,
145 lps.points[0]->start_timestamp_millis);
146 ASSERT_EQ(2, lps.points[1]->labels.size());
147 EXPECT_EQ("MyLabel0", lps.points[1]->labels[0].name);
148 EXPECT_EQ("Label01", lps.points[1]->labels[0].value);
149 EXPECT_EQ("MyLabel1", lps.points[1]->labels[1].name);
150 EXPECT_EQ("Label11", lps.points[1]->labels[1].value);
151 EXPECT_EQ(ValueType::kInt64, lps.points[1]->value_type);
152 EXPECT_EQ(58, lps.points[1]->int64_value);
153 EXPECT_LT(0, lps.points[1]->start_timestamp_millis);
154 EXPECT_LT(0, lps.points[1]->end_timestamp_millis);
155 EXPECT_GE(lps.points[1]->end_timestamp_millis,
156 lps.points[1]->start_timestamp_millis);
157
158 const PointSet& ups = *collected_metrics->point_set_map.at(
159 "/tensorflow/test/counter_without_labels");
160 EXPECT_EQ("/tensorflow/test/counter_without_labels", ups.metric_name);
161 ASSERT_EQ(1, ups.points.size());
162 EXPECT_EQ(0, ups.points[0]->labels.size());
163 EXPECT_EQ(ValueType::kInt64, ups.points[0]->value_type);
164 EXPECT_EQ(7, ups.points[0]->int64_value);
165 EXPECT_LT(0, ups.points[0]->start_timestamp_millis);
166 EXPECT_LT(0, ups.points[0]->end_timestamp_millis);
167 EXPECT_GE(ups.points[0]->end_timestamp_millis,
168 ups.points[0]->start_timestamp_millis);
169 }
170 }
171
TEST(CollectMetricsTest,Gauge)172 TEST(CollectMetricsTest, Gauge) {
173 auto string_gauge_with_labels =
174 std::unique_ptr<Gauge<string, 2>>(Gauge<string, 2>::New(
175 "/tensorflow/test/string_gauge_with_labels",
176 "String gauge with labels.", "MyLabel0", "MyLabel1"));
177 auto inteter_gauge_without_labels = std::unique_ptr<Gauge<int64, 0>>(
178 Gauge<int64, 0>::New("/tensorflow/test/integer_gauge_without_labels",
179 "Integer gauge without labels."));
180
181 string_gauge_with_labels->GetCell("Label00", "Label10")->Set("test1");
182 string_gauge_with_labels->GetCell("Label01", "Label11")->Set("test2");
183 inteter_gauge_without_labels->GetCell()->Set(7);
184
185 for (const bool collect_metric_descriptors : {true, false}) {
186 SCOPED_TRACE(strings::StrCat("collect_metric_descriptors: ",
187 collect_metric_descriptors));
188
189 auto* collection_registry = CollectionRegistry::Default();
190 CollectionRegistry::CollectMetricsOptions options;
191 options.collect_metric_descriptors = collect_metric_descriptors;
192 const std::unique_ptr<CollectedMetrics> collected_metrics =
193 collection_registry->CollectMetrics(options);
194
195 if (collect_metric_descriptors) {
196 ASSERT_GE(collected_metrics->metric_descriptor_map.size(), 2);
197
198 const MetricDescriptor& ld = *collected_metrics->metric_descriptor_map.at(
199 "/tensorflow/test/string_gauge_with_labels");
200 EXPECT_EQ("/tensorflow/test/string_gauge_with_labels", ld.name);
201 EXPECT_EQ("String gauge with labels.", ld.description);
202 ASSERT_EQ(2, ld.label_names.size());
203 EXPECT_EQ("MyLabel0", ld.label_names[0]);
204 EXPECT_EQ("MyLabel1", ld.label_names[1]);
205 EXPECT_EQ(MetricKind::kGauge, ld.metric_kind);
206 EXPECT_EQ(ValueType::kString, ld.value_type);
207
208 const MetricDescriptor& ud = *collected_metrics->metric_descriptor_map.at(
209 "/tensorflow/test/integer_gauge_without_labels");
210 EXPECT_EQ("/tensorflow/test/integer_gauge_without_labels", ud.name);
211 EXPECT_EQ("Integer gauge without labels.", ud.description);
212 ASSERT_EQ(0, ud.label_names.size());
213 EXPECT_EQ(MetricKind::kGauge, ud.metric_kind);
214 EXPECT_EQ(ValueType::kInt64, ud.value_type);
215 } else {
216 EXPECT_EQ(0, collected_metrics->metric_descriptor_map.size());
217 }
218
219 ASSERT_GE(collected_metrics->point_set_map.size(), 2);
220
221 const PointSet& lps = *collected_metrics->point_set_map.at(
222 "/tensorflow/test/string_gauge_with_labels");
223 EXPECT_EQ("/tensorflow/test/string_gauge_with_labels", lps.metric_name);
224 ASSERT_EQ(2, lps.points.size());
225 ASSERT_EQ(2, lps.points[0]->labels.size());
226 EXPECT_EQ("MyLabel0", lps.points[0]->labels[0].name);
227 EXPECT_EQ("Label00", lps.points[0]->labels[0].value);
228 EXPECT_EQ("MyLabel1", lps.points[0]->labels[1].name);
229 EXPECT_EQ("Label10", lps.points[0]->labels[1].value);
230 EXPECT_EQ(ValueType::kString, lps.points[0]->value_type);
231 EXPECT_EQ("test1", lps.points[0]->string_value);
232 EXPECT_LT(0, lps.points[0]->start_timestamp_millis);
233 EXPECT_LT(0, lps.points[0]->end_timestamp_millis);
234 EXPECT_GE(lps.points[0]->end_timestamp_millis,
235 lps.points[0]->start_timestamp_millis);
236 ASSERT_EQ(2, lps.points[1]->labels.size());
237 EXPECT_EQ("MyLabel0", lps.points[1]->labels[0].name);
238 EXPECT_EQ("Label01", lps.points[1]->labels[0].value);
239 EXPECT_EQ("MyLabel1", lps.points[1]->labels[1].name);
240 EXPECT_EQ("Label11", lps.points[1]->labels[1].value);
241 EXPECT_EQ(ValueType::kString, lps.points[1]->value_type);
242 EXPECT_EQ("test2", lps.points[1]->string_value);
243 EXPECT_LT(0, lps.points[1]->start_timestamp_millis);
244 EXPECT_LT(0, lps.points[1]->end_timestamp_millis);
245 EXPECT_GE(lps.points[1]->end_timestamp_millis,
246 lps.points[1]->start_timestamp_millis);
247
248 const PointSet& ups = *collected_metrics->point_set_map.at(
249 "/tensorflow/test/integer_gauge_without_labels");
250 EXPECT_EQ("/tensorflow/test/integer_gauge_without_labels", ups.metric_name);
251 ASSERT_EQ(1, ups.points.size());
252 EXPECT_EQ(0, ups.points[0]->labels.size());
253 EXPECT_EQ(ValueType::kInt64, ups.points[0]->value_type);
254 EXPECT_EQ(7, ups.points[0]->int64_value);
255 EXPECT_LT(0, ups.points[0]->start_timestamp_millis);
256 EXPECT_LT(0, ups.points[0]->end_timestamp_millis);
257 EXPECT_GE(ups.points[0]->end_timestamp_millis,
258 ups.points[0]->start_timestamp_millis);
259 }
260 }
261
EqHistograms(const Histogram & expected,const HistogramProto & actual_proto)262 void EqHistograms(const Histogram& expected,
263 const HistogramProto& actual_proto) {
264 Histogram actual;
265 ASSERT_TRUE(actual.DecodeFromProto(actual_proto));
266
267 EXPECT_EQ(expected.ToString(), actual.ToString());
268 }
269
TEST(CollectMetricsTest,Sampler)270 TEST(CollectMetricsTest, Sampler) {
271 auto sampler_with_labels = std::unique_ptr<Sampler<2>>(
272 Sampler<2>::New({"/tensorflow/test/sampler_with_labels",
273 "Sampler with labels.", "MyLabel0", "MyLabel1"},
274 Buckets::Explicit({1.0, 2.0})));
275 auto sampler_without_labels = std::unique_ptr<Sampler<0>>(Sampler<0>::New(
276 {"/tensorflow/test/sampler_without_labels", "Sampler without labels."},
277 Buckets::Explicit({0.0})));
278
279 Histogram with_labels0({1.0, 2.0, DBL_MAX});
280 sampler_with_labels->GetCell("Label00", "Label10")->Add(0.7);
281 with_labels0.Add(0.7);
282
283 Histogram with_labels1({1.0, 2.0, DBL_MAX});
284 sampler_with_labels->GetCell("Label01", "Label11")->Add(1.5);
285 with_labels1.Add(1.5);
286
287 Histogram without_labels({0.0, DBL_MAX});
288 sampler_without_labels->GetCell()->Add(0.5);
289 without_labels.Add(0.5);
290
291 for (const bool collect_metric_descriptors : {true, false}) {
292 SCOPED_TRACE(strings::StrCat("collect_metric_descriptors: ",
293 collect_metric_descriptors));
294
295 auto* collection_registry = CollectionRegistry::Default();
296 CollectionRegistry::CollectMetricsOptions options;
297 options.collect_metric_descriptors = collect_metric_descriptors;
298 const std::unique_ptr<CollectedMetrics> collected_metrics =
299 collection_registry->CollectMetrics(options);
300
301 if (collect_metric_descriptors) {
302 ASSERT_GE(collected_metrics->metric_descriptor_map.size(), 2);
303
304 const MetricDescriptor& ld = *collected_metrics->metric_descriptor_map.at(
305 "/tensorflow/test/sampler_with_labels");
306 EXPECT_EQ("/tensorflow/test/sampler_with_labels", ld.name);
307 EXPECT_EQ("Sampler with labels.", ld.description);
308 ASSERT_EQ(2, ld.label_names.size());
309 EXPECT_EQ("MyLabel0", ld.label_names[0]);
310 EXPECT_EQ("MyLabel1", ld.label_names[1]);
311 EXPECT_EQ(MetricKind::kCumulative, ld.metric_kind);
312 EXPECT_EQ(ValueType::kHistogram, ld.value_type);
313
314 const MetricDescriptor& ud = *collected_metrics->metric_descriptor_map.at(
315 "/tensorflow/test/sampler_without_labels");
316 EXPECT_EQ("/tensorflow/test/sampler_without_labels", ud.name);
317 EXPECT_EQ("Sampler without labels.", ud.description);
318 ASSERT_EQ(0, ud.label_names.size());
319 EXPECT_EQ(MetricKind::kCumulative, ud.metric_kind);
320 EXPECT_EQ(ValueType::kHistogram, ud.value_type);
321 } else {
322 EXPECT_EQ(0, collected_metrics->metric_descriptor_map.size());
323 }
324
325 ASSERT_GE(collected_metrics->point_set_map.size(), 2);
326
327 const PointSet& lps = *collected_metrics->point_set_map.at(
328 "/tensorflow/test/sampler_with_labels");
329 EXPECT_EQ("/tensorflow/test/sampler_with_labels", lps.metric_name);
330 ASSERT_EQ(2, lps.points.size());
331 ASSERT_EQ(2, lps.points[0]->labels.size());
332 EXPECT_EQ("MyLabel0", lps.points[0]->labels[0].name);
333 EXPECT_EQ("Label00", lps.points[0]->labels[0].value);
334 EXPECT_EQ("MyLabel1", lps.points[0]->labels[1].name);
335 EXPECT_EQ("Label10", lps.points[0]->labels[1].value);
336 EXPECT_EQ(ValueType::kHistogram, lps.points[0]->value_type);
337 EqHistograms(with_labels0, lps.points[0]->histogram_value);
338 EXPECT_LT(0, lps.points[0]->start_timestamp_millis);
339 EXPECT_LT(0, lps.points[0]->end_timestamp_millis);
340 EXPECT_GE(lps.points[0]->end_timestamp_millis,
341 lps.points[0]->start_timestamp_millis);
342 ASSERT_EQ(2, lps.points[1]->labels.size());
343 EXPECT_EQ("MyLabel0", lps.points[1]->labels[0].name);
344 EXPECT_EQ("Label01", lps.points[1]->labels[0].value);
345 EXPECT_EQ("MyLabel1", lps.points[1]->labels[1].name);
346 EXPECT_EQ("Label11", lps.points[1]->labels[1].value);
347 EXPECT_EQ(ValueType::kHistogram, lps.points[1]->value_type);
348 EqHistograms(with_labels1, lps.points[1]->histogram_value);
349 EXPECT_LT(0, lps.points[1]->start_timestamp_millis);
350 EXPECT_LT(0, lps.points[1]->end_timestamp_millis);
351 EXPECT_GE(lps.points[1]->end_timestamp_millis,
352 lps.points[1]->start_timestamp_millis);
353
354 const PointSet& ups = *collected_metrics->point_set_map.at(
355 "/tensorflow/test/sampler_without_labels");
356 EXPECT_EQ("/tensorflow/test/sampler_without_labels", ups.metric_name);
357 ASSERT_EQ(1, ups.points.size());
358 EXPECT_EQ(0, ups.points[0]->labels.size());
359 EXPECT_EQ(ValueType::kHistogram, ups.points[0]->value_type);
360 EqHistograms(without_labels, ups.points[0]->histogram_value);
361 EXPECT_LT(0, ups.points[0]->start_timestamp_millis);
362 EXPECT_LT(0, ups.points[0]->end_timestamp_millis);
363 EXPECT_GE(ups.points[0]->end_timestamp_millis,
364 ups.points[0]->start_timestamp_millis);
365 }
366 }
367
368 // A FakeClockEnv to manually advance time.
369 class FakeClockEnv : public EnvWrapper {
370 public:
FakeClockEnv()371 FakeClockEnv() : EnvWrapper(Env::Default()), current_millis_(0) {}
372
373 // Manually advance the current time by 'millis' milliseconds.
AdvanceByMillis(const uint64 millis)374 void AdvanceByMillis(const uint64 millis) { current_millis_ += millis; }
375
376 // Method that this environment specifically overrides.
NowMicros()377 uint64 NowMicros() override { return current_millis_ * 1000; }
378
379 private:
380 uint64 current_millis_;
381 };
382
TEST(CollectionRegistryTest,WriteTimestamps)383 TEST(CollectionRegistryTest, WriteTimestamps) {
384 FakeClockEnv fake_clock_env;
385 auto collection_registry =
386 test_util::CollectionRegistryTestAccess::CreateRegistry(&fake_clock_env);
387
388 fake_clock_env.AdvanceByMillis(25);
389 {
390 const MetricDef<MetricKind::kCumulative, int64, 0> cumulative_metric(
391 "/tensorflow/cumulative/metric", "An example metric with no labels.");
392 auto handle = collection_registry->Register(
393 &cumulative_metric, [&](MetricCollectorGetter getter) {
394 auto metric_collector = getter.Get(&cumulative_metric);
395 metric_collector.CollectValue({}, 42);
396 });
397 fake_clock_env.AdvanceByMillis(75);
398 const std::unique_ptr<CollectedMetrics> collected_metrics =
399 collection_registry->CollectMetrics({});
400 const PointSet& point_set =
401 *collected_metrics->point_set_map.at("/tensorflow/cumulative/metric");
402 ASSERT_EQ(1, point_set.points.size());
403 EXPECT_EQ(25, point_set.points[0]->start_timestamp_millis);
404 EXPECT_EQ(100, point_set.points[0]->end_timestamp_millis);
405 }
406 {
407 const MetricDef<MetricKind::kGauge, int64, 0> gauge_metric(
408 "/tensorflow/gauge/metric", "An example metric with no labels.");
409 auto handle = collection_registry->Register(
410 &gauge_metric, [&](MetricCollectorGetter getter) {
411 auto metric_collector = getter.Get(&gauge_metric);
412 metric_collector.CollectValue({}, 42);
413 });
414 fake_clock_env.AdvanceByMillis(75);
415 const std::unique_ptr<CollectedMetrics> collected_metrics =
416 collection_registry->CollectMetrics({});
417 const PointSet& point_set =
418 *collected_metrics->point_set_map.at("/tensorflow/gauge/metric");
419 ASSERT_EQ(1, point_set.points.size());
420 EXPECT_EQ(175, point_set.points[0]->start_timestamp_millis);
421 EXPECT_EQ(175, point_set.points[0]->end_timestamp_millis);
422 }
423 }
424
425 } // namespace
426 } // namespace monitoring
427 } // namespace tensorflow
428