1 /* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include "tensorflow/core/data/metric_utils.h"
16
17 #include <cstdint>
18
19 #include "absl/memory/memory.h"
20 #include "absl/time/clock.h"
21 #include "absl/time/time.h"
22 #include "tensorflow/core/framework/types.h"
23 #include "tensorflow/core/lib/monitoring/cell_reader.h"
24 #include "tensorflow/core/lib/monitoring/test_utils.h"
25 #include "tensorflow/core/platform/env.h"
26 #include "tensorflow/core/platform/test.h"
27
28 namespace tensorflow {
29 namespace data {
30 namespace {
31
32 using tensorflow::monitoring::testing::CellReader;
33 using tensorflow::monitoring::testing::Histogram;
34
TEST(MetricUtilsTest,CollectMetrics)35 TEST(MetricUtilsTest, CollectMetrics) {
36 CellReader<Histogram> latency("/tensorflow/data/getnext_duration");
37 CellReader<int64_t> iterator_lifetime("/tensorflow/data/iterator_lifetime");
38 CellReader<int64_t> iterator_busy("/tensorflow/data/iterator_busy");
39 EXPECT_FLOAT_EQ(latency.Delta().num(), 0.0);
40 EXPECT_EQ(iterator_lifetime.Delta(), 0);
41 EXPECT_EQ(iterator_busy.Delta(), 0);
42
43 IteratorMetricsCollector metrics_collector(DEVICE_CPU, *Env::Default());
44 absl::Time start_time = metrics_collector.RecordStart();
45 absl::SleepFor(absl::Seconds(1));
46 metrics_collector.RecordStop(start_time, /*output=*/{});
47
48 Histogram latency_histogram = latency.Delta();
49 EXPECT_FLOAT_EQ(latency_histogram.num(), 1.0);
50 EXPECT_GT(latency_histogram.sum(), 0.0);
51 EXPECT_GT(iterator_lifetime.Delta(), 0);
52 EXPECT_GT(iterator_busy.Delta(), 0);
53 }
54
TEST(MetricUtilsTest,ShouldNotCollectMetrics)55 TEST(MetricUtilsTest, ShouldNotCollectMetrics) {
56 CellReader<Histogram> latency("/tensorflow/data/getnext_duration");
57 CellReader<int64_t> iterator_lifetime("/tensorflow/data/iterator_lifetime");
58 CellReader<int64_t> iterator_busy("/tensorflow/data/iterator_busy");
59 EXPECT_FLOAT_EQ(latency.Delta().num(), 0.0);
60 EXPECT_EQ(iterator_lifetime.Delta(), 0);
61 EXPECT_EQ(iterator_busy.Delta(), 0);
62
63 IteratorMetricsCollector metrics_collector(DEVICE_TPU, *Env::Default());
64 absl::Time start_time = metrics_collector.RecordStart();
65 absl::SleepFor(absl::Seconds(1));
66 metrics_collector.RecordStop(start_time, /*output=*/{});
67
68 EXPECT_FLOAT_EQ(latency.Delta().num(), 0.0);
69 EXPECT_EQ(iterator_lifetime.Delta(), 0);
70 EXPECT_EQ(iterator_busy.Delta(), 0);
71 }
72
TEST(MetricUtilsTest,ConcurrentThreads)73 TEST(MetricUtilsTest, ConcurrentThreads) {
74 CellReader<Histogram> latency("/tensorflow/data/getnext_duration");
75 CellReader<int64_t> iterator_lifetime("/tensorflow/data/iterator_lifetime");
76 CellReader<int64_t> iterator_busy("/tensorflow/data/iterator_busy");
77 EXPECT_FLOAT_EQ(latency.Delta().num(), 0.0);
78 EXPECT_EQ(iterator_lifetime.Delta(), 0);
79 EXPECT_EQ(iterator_busy.Delta(), 0);
80
81 IteratorMetricsCollector metrics_collector(DEVICE_CPU, *Env::Default());
82 absl::Time start_time = metrics_collector.RecordStart();
83 auto thread = absl::WrapUnique(Env::Default()->StartThread(
84 /*thread_options=*/{}, /*name=*/"Concurrent metric collection thread",
85 [&metrics_collector]() {
86 absl::Time concurrent_start_time = metrics_collector.RecordStart();
87 absl::SleepFor(absl::Seconds(1));
88 metrics_collector.RecordStop(concurrent_start_time, /*output=*/{});
89 }));
90 absl::SleepFor(absl::Seconds(1));
91 metrics_collector.RecordStop(start_time, /*output=*/{});
92 thread.reset();
93
94 Histogram latency_histogram = latency.Delta();
95 EXPECT_FLOAT_EQ(latency_histogram.num(), 2.0);
96 EXPECT_GT(latency_histogram.sum(),
97 absl::ToInt64Microseconds(absl::Seconds(2)));
98 // The iterator busy time and lifetime do not count the latency twice.
99 EXPECT_GE(iterator_lifetime.Delta(),
100 absl::ToInt64Microseconds(absl::Seconds(1)));
101 EXPECT_LT(iterator_lifetime.Delta(),
102 absl::ToInt64Microseconds(absl::Seconds(1.5)));
103 EXPECT_GE(iterator_busy.Delta(), absl::ToInt64Microseconds(absl::Seconds(1)));
104 EXPECT_LT(iterator_busy.Delta(),
105 absl::ToInt64Microseconds(absl::Seconds(1.5)));
106 }
107
TEST(MetricUtilsTest,OverlappingThreads)108 TEST(MetricUtilsTest, OverlappingThreads) {
109 CellReader<Histogram> latency("/tensorflow/data/getnext_duration");
110 CellReader<int64_t> iterator_lifetime("/tensorflow/data/iterator_lifetime");
111 CellReader<int64_t> iterator_busy("/tensorflow/data/iterator_busy");
112 EXPECT_FLOAT_EQ(latency.Delta().num(), 0.0);
113 EXPECT_EQ(iterator_lifetime.Delta(), 0);
114 EXPECT_EQ(iterator_busy.Delta(), 0);
115
116 // Two overlapping threads collect metrics:
117 // Thread 1 (end - start = 1 sec): |---------|
118 // Thread 2 (end - start = 2 sec): |------------------|
119 // Overlap: 0.5 sec.
120 // Iterator busy time: 2.5 sec.
121 IteratorMetricsCollector metrics_collector(DEVICE_CPU, *Env::Default());
122 absl::Time start_time = metrics_collector.RecordStart();
123 absl::SleepFor(absl::Seconds(0.5));
124 auto thread = absl::WrapUnique(Env::Default()->StartThread(
125 /*thread_options=*/{}, /*name=*/"Concurrent metric collection thread",
126 [&metrics_collector]() {
127 absl::Time concurrent_start_time = metrics_collector.RecordStart();
128 absl::SleepFor(absl::Seconds(2));
129 metrics_collector.RecordStop(concurrent_start_time, /*output=*/{});
130 }));
131 absl::SleepFor(absl::Seconds(0.5));
132 metrics_collector.RecordStop(start_time, /*output=*/{});
133 absl::SleepFor(absl::Seconds(1.5));
134 thread.reset();
135
136 Histogram latency_histogram = latency.Delta();
137 EXPECT_FLOAT_EQ(latency_histogram.num(), 2.0);
138 EXPECT_GT(latency_histogram.sum(),
139 absl::ToInt64Microseconds(absl::Seconds(3)));
140 // The iterator busy time and lifetime should not count the overlap twice.
141 EXPECT_GE(iterator_lifetime.Delta(),
142 absl::ToInt64Microseconds(absl::Seconds(2.5)));
143 EXPECT_LT(iterator_lifetime.Delta(),
144 absl::ToInt64Microseconds(absl::Seconds(2.9)));
145 EXPECT_GE(iterator_busy.Delta(),
146 absl::ToInt64Microseconds(absl::Seconds(2.5)));
147 EXPECT_LT(iterator_busy.Delta(),
148 absl::ToInt64Microseconds(absl::Seconds(2.9)));
149 }
150
151 } // namespace
152 } // namespace data
153 } // namespace tensorflow
154