1 //
2 // Copyright © 2017 Arm Ltd. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5
6 #include <armnn/IRuntime.hpp>
7 #include <armnn/TypesUtils.hpp>
8 #include <armnn/utility/IgnoreUnused.hpp>
9
10 #include <boost/test/unit_test.hpp>
11 #include <boost/test/tools/output_test_stream.hpp>
12
13 #include <memory>
14 #include <thread>
15 #include <ostream>
16
17 #include <Profiling.hpp>
18
19 namespace armnn
20 {
21
GetProfilerEventSequenceSize(armnn::Profiler * profiler)22 size_t GetProfilerEventSequenceSize(armnn::Profiler* profiler)
23 {
24 if (!profiler)
25 {
26 return static_cast<size_t>(-1);
27 }
28
29 return profiler->m_EventSequence.size();
30 }
31 } // namespace armnn
32
33 namespace
34 {
35
RegisterUnregisterProfilerSingleThreadImpl(bool & res)36 void RegisterUnregisterProfilerSingleThreadImpl(bool &res)
37 {
38 // Important! Don't use BOOST_TEST macros in this function as they
39 // seem to have problems when used in threads
40
41 // Get a reference to the profiler manager.
42 armnn::ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
43
44 // Check that there's no profiler registered for this thread.
45 res = !profilerManager.GetProfiler();
46
47 // Create and register a profiler for this thread.
48 std::unique_ptr<armnn::Profiler> profiler = std::make_unique<armnn::Profiler>();
49 profilerManager.RegisterProfiler(profiler.get());
50
51 // Check that on a single thread we get the same profiler we registered.
52 res &= profiler.get() == profilerManager.GetProfiler();
53
54 // Destroy the profiler.
55 profiler.reset();
56
57 // Check that the profiler has been un-registered for this thread.
58 res &= !profilerManager.GetProfiler();
59
60 armnn::ProfilerManager::GetInstance().RegisterProfiler(nullptr);
61 }
62
63 } // namespace
64
65 BOOST_AUTO_TEST_SUITE(Profiler)
66
BOOST_AUTO_TEST_CASE(EnableDisableProfiling)67 BOOST_AUTO_TEST_CASE(EnableDisableProfiling)
68 {
69 std::unique_ptr<armnn::Profiler> profiler = std::make_unique<armnn::Profiler>();
70
71 // Check that profiling is disabled by default.
72 BOOST_TEST(!profiler->IsProfilingEnabled());
73
74 // Enable profiling.
75 profiler->EnableProfiling(true);
76
77 // Check that profiling is enabled.
78 BOOST_TEST(profiler->IsProfilingEnabled());
79
80 // Disable profiling.
81 profiler->EnableProfiling(false);
82
83 // Check that profiling is disabled.
84 BOOST_TEST(!profiler->IsProfilingEnabled());
85 }
86
BOOST_AUTO_TEST_CASE(RegisterUnregisterProfilerSingleThread)87 BOOST_AUTO_TEST_CASE(RegisterUnregisterProfilerSingleThread)
88 {
89 bool res = false;
90 RegisterUnregisterProfilerSingleThreadImpl(res);
91 BOOST_TEST(res);
92 }
93
BOOST_AUTO_TEST_CASE(RegisterUnregisterProfilerMultipleThreads)94 BOOST_AUTO_TEST_CASE(RegisterUnregisterProfilerMultipleThreads)
95 {
96 bool res[3] = {false, false, false};
97 std::thread thread1([&res]() { RegisterUnregisterProfilerSingleThreadImpl(res[0]); });
98 std::thread thread2([&res]() { RegisterUnregisterProfilerSingleThreadImpl(res[1]); });
99 std::thread thread3([&res]() { RegisterUnregisterProfilerSingleThreadImpl(res[2]); });
100
101 thread1.join();
102 thread2.join();
103 thread3.join();
104
105 for (int i = 0 ; i < 3 ; i++)
106 {
107 BOOST_TEST(res[i]);
108 }
109 }
110
BOOST_AUTO_TEST_CASE(ProfilingMacros)111 BOOST_AUTO_TEST_CASE(ProfilingMacros)
112 {
113 // Get a reference to the profiler manager.
114 armnn::ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
115
116 { // --- No profiler ---
117
118 // Check that there's no profiler registered for this thread.
119 BOOST_TEST(!profilerManager.GetProfiler());
120
121 // Test scoped event.
122 { ARMNN_SCOPED_PROFILING_EVENT(armnn::Compute::CpuAcc, "test"); }
123
124 // Check that we still cannot get a profiler for this thread.
125 BOOST_TEST(!profilerManager.GetProfiler());
126 }
127
128 // Create and register a profiler for this thread.
129 std::unique_ptr<armnn::Profiler> profiler = std::make_unique<armnn::Profiler>();
130 profilerManager.RegisterProfiler(profiler.get());
131
132 { // --- Profiler, but profiling disabled ---
133
134 // Get current event sequence size.
135 size_t eventSequenceSizeBefore = armnn::GetProfilerEventSequenceSize(profiler.get());
136
137 // Test scoped macro.
138 { ARMNN_SCOPED_PROFILING_EVENT(armnn::Compute::CpuAcc, "test"); }
139
140 // Check that no profiling event has been added to the sequence.
141 size_t eventSequenceSizeAfter = armnn::GetProfilerEventSequenceSize(profiler.get());
142 BOOST_TEST(eventSequenceSizeBefore == eventSequenceSizeAfter);
143 }
144
145 // Enable profiling.
146 profiler->EnableProfiling(true);
147
148 { // --- Profiler, and profiling enabled ---
149
150 // Get current event sequence size.
151 size_t eventSequenceSizeBefore = armnn::GetProfilerEventSequenceSize(profiler.get());
152
153 // Test scoped macro.
154 { ARMNN_SCOPED_PROFILING_EVENT(armnn::Compute::CpuAcc, "test"); }
155
156 // Check that a profiling event has been added to the sequence.
157 size_t eventSequenceSizeAfter = armnn::GetProfilerEventSequenceSize(profiler.get());
158 BOOST_TEST(eventSequenceSizeAfter == eventSequenceSizeBefore + 1);
159 }
160
161 // Disable profiling here to not print out anything on stdout.
162 profiler->EnableProfiling(false);
163 }
164
165 #if defined(ARMNNREF_ENABLED)
166
167 // This test unit needs the reference backend, it's not available if the reference backend is not built
168
BOOST_AUTO_TEST_CASE(RuntimeLoadNetwork)169 BOOST_AUTO_TEST_CASE(RuntimeLoadNetwork)
170 {
171 // Get a reference to the profiler manager.
172 armnn::ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
173
174 // Check that there's no profiler registered for this thread.
175 BOOST_TEST(!profilerManager.GetProfiler());
176
177 // Build a mock-network and load it into the runtime.
178 armnn::IRuntime::CreationOptions options;
179 armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options));
180 armnn::NetworkId networkIdentifier = 1;
181 armnn::INetworkPtr mockNetwork(armnn::INetwork::Create());
182 mockNetwork->AddInputLayer(0, "test layer");
183 std::vector<armnn::BackendId> backends = { armnn::Compute::CpuRef };
184 runtime->LoadNetwork(networkIdentifier, armnn::Optimize(*mockNetwork, backends, runtime->GetDeviceSpec()));
185
186 // Check that now there's a profiler registered for this thread (created and registered by the loading the network).
187 BOOST_TEST(profilerManager.GetProfiler());
188
189 // Unload the network.
190 runtime->UnloadNetwork(networkIdentifier);
191
192 // Check that the profiler has been un-registered for this thread.
193 BOOST_TEST(!profilerManager.GetProfiler());
194 }
195
196 #endif
197
BOOST_AUTO_TEST_CASE(WriteEventResults)198 BOOST_AUTO_TEST_CASE(WriteEventResults)
199 {
200 // Get a reference to the profiler manager.
201 armnn::ProfilerManager& profileManager = armnn::ProfilerManager::GetInstance();
202
203 // Create and register a profiler for this thread.
204 std::unique_ptr<armnn::Profiler> profiler = std::make_unique<armnn::Profiler>();
205 profileManager.RegisterProfiler(profiler.get());
206
207 // Enable profiling.
208 profiler->EnableProfiling(true);
209
210 { // --- Profiler, and profiling enabled ---
211
212 // Get current event sequence size.
213 size_t eventSequenceSizeBefore = armnn::GetProfilerEventSequenceSize(profiler.get());
214
215 // Test scoped macro.
216 {
217 // Need to directly create a ScopedProfilingEvent as the one created by the macro falls out of scope
218 // immediately causing the Event.Stop() function method to be called immediately after the Event.Start()
219 // function resulting in periodic test failures on the Dent and Smith HiKeys
220 armnn::ScopedProfilingEvent testEvent(armnn::Compute::CpuAcc, "test", armnn::WallClockTimer());
221 std::this_thread::sleep_for(std::chrono::milliseconds(10));
222 }
223
224 // Check that a profiling event has been added to the sequence.
225 size_t eventSequenceSizeAfter = armnn::GetProfilerEventSequenceSize(profiler.get());
226 BOOST_TEST(eventSequenceSizeAfter == eventSequenceSizeBefore + 1);
227
228 boost::test_tools::output_test_stream output;
229 profiler->AnalyzeEventsAndWriteResults(output);
230 BOOST_TEST(!output.is_empty(false));
231
232 // output should contain event name 'test'
233 BOOST_CHECK(output.str().find("test") != std::string::npos);
234
235 // output should contain headers
236 BOOST_CHECK(output.str().find("Event Sequence - Name") != std::string::npos);
237 BOOST_CHECK(output.str().find("Event Stats - Name") != std::string::npos);
238 BOOST_CHECK(output.str().find("Total") != std::string::npos);
239 BOOST_CHECK(output.str().find("Device") != std::string::npos);
240 // output should contain compute device 'CpuAcc'
241 BOOST_CHECK(output.str().find("CpuAcc") != std::string::npos);
242 // output should not contain un-readable numbers
243 BOOST_CHECK(output.str().find("e+") == std::string::npos);
244 // output should not contain un-readable numbers
245 BOOST_CHECK(output.str().find("+") == std::string::npos);
246 // output should not contain zero value
247 BOOST_CHECK(output.str().find(" 0 ") == std::string::npos);
248 }
249
250 // Disable profiling here to not print out anything on stdout.
251 profiler->EnableProfiling(false);
252 }
253
BOOST_AUTO_TEST_CASE(ProfilerJsonPrinter)254 BOOST_AUTO_TEST_CASE(ProfilerJsonPrinter)
255 {
256 class TestInstrument : public armnn::Instrument
257 {
258 public:
259 virtual ~TestInstrument() {}
260 void Start() override {}
261 void Stop() override {}
262
263 std::vector<armnn::Measurement> GetMeasurements() const override
264 {
265 std::vector<armnn::Measurement> measurements;
266 measurements.emplace_back(armnn::Measurement("Measurement1",
267 1.0,
268 armnn::Measurement::Unit::TIME_MS));
269 measurements.emplace_back(armnn::Measurement("Measurement2",
270 2.0,
271 armnn::Measurement::Unit::TIME_US));
272 return measurements;
273 }
274
275 const char* GetName() const override
276 {
277 return "TestInstrument";
278 }
279 };
280
281 // Get a reference to the profiler manager.
282 armnn::ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
283
284 // Create and register a profiler for this thread.
285 std::unique_ptr<armnn::Profiler> profiler = std::make_unique<armnn::Profiler>();
286 profilerManager.RegisterProfiler(profiler.get());
287
288 profiler->EnableProfiling(true);
289
290 {
291 // Test scoped macro.
292 ARMNN_SCOPED_PROFILING_EVENT_WITH_INSTRUMENTS(armnn::Compute::CpuAcc, "EnqueueWorkload", TestInstrument())
293 ARMNN_SCOPED_PROFILING_EVENT_WITH_INSTRUMENTS(armnn::Compute::CpuAcc, "Level 0", TestInstrument())
294 {
295 {
296 ARMNN_SCOPED_PROFILING_EVENT_WITH_INSTRUMENTS(armnn::Compute::CpuAcc, "Level 1A", TestInstrument())
297 }
298
299 {
300 ARMNN_SCOPED_PROFILING_EVENT_WITH_INSTRUMENTS(armnn::Compute::CpuAcc, "Level 1B", TestInstrument())
301
302 {
303 ARMNN_SCOPED_PROFILING_EVENT_WITH_INSTRUMENTS(armnn::Compute::CpuAcc, "Level 2A", TestInstrument())
304 }
305 }
306 }
307 }
308
309 std::stringbuf buffer;
310 std::ostream json(&buffer);
311 profiler->Print(json);
312
313 std::string output = buffer.str();
314 armnn::IgnoreUnused(output);
315
316 // Disable profiling here to not print out anything on stdout.
317 profiler->EnableProfiling(false);
318
319 // blessed output validated by a human eyeballing the output to make sure it's ok and then copying it here.
320 // validation also included running the blessed output through an online json validation site
321 std::string blessedOutput("{\n\t\"ArmNN\": {\n\t\t\"inference_measurements_#1\": {\n\t\t\t\"type\": \""
322 "Event\",\n\t\t\t\"Measurement1_#1\": {\n\t\t\t\t\"type\": \""
323 "Measurement\",\n\t\t\t\t\"raw\": [\n\t\t\t\t\t1.000000\n\t\t\t\t],\n\t\t\t\t\""
324 "unit\": \"ms\"\n\t\t\t},\n\t\t\t\"Measurement2_#1\": {\n\t\t\t\t\"type\": \""
325 "Measurement\",\n\t\t\t\t\"raw\": [\n\t\t\t\t\t2.000000\n\t\t\t\t],\n\t\t\t\t\""
326 "unit\": \"us\"\n\t\t\t},\n\t\t\t\"Level 0_#2\": {\n\t\t\t\t\"type\": \""
327 "Event\",\n\t\t\t\t\"Measurement1_#2\": {\n\t\t\t\t\t\"type\": \""
328 "Measurement\",\n\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t1.000000\n\t\t\t\t\t],\n\t\t\t\t\t\""
329 "unit\": \"ms\"\n\t\t\t\t},\n\t\t\t\t\"Measurement2_#2\": {\n\t\t\t\t\t\"type\": \""
330 "Measurement\",\n\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t2.000000\n\t\t\t\t\t],\n\t\t\t\t\t\""
331 "unit\": \"us\"\n\t\t\t\t},\n\t\t\t\t\"Level 1A_#3\": {\n\t\t\t\t\t\"type\": \""
332 "Event\",\n\t\t\t\t\t\"Measurement1_#3\": {\n\t\t\t\t\t\t\"type\": \""
333 "Measurement\",\n\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t"
334 "1.000000\n\t\t\t\t\t\t],\n\t\t\t\t\t\t\""
335 "unit\": \"ms\"\n\t\t\t\t\t},\n\t\t\t\t\t\"Measurement2_#3\": {\n\t\t\t\t\t\t\"type\": \""
336 "Measurement\",\n\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t"
337 "2.000000\n\t\t\t\t\t\t],\n\t\t\t\t\t\t\""
338 "unit\": \"us\"\n\t\t\t\t\t}\n\t\t\t\t},\n\t\t\t\t\"Level 1B_#4\": {\n\t\t\t\t\t\""
339 "type\": \"Event\",\n\t\t\t\t\t\"Measurement1_#4\": {\n\t\t\t\t\t\t\"type\": \""
340 "Measurement\",\n\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t"
341 "1.000000\n\t\t\t\t\t\t],\n\t\t\t\t\t\t\""
342 "unit\": \"ms\"\n\t\t\t\t\t},\n\t\t\t\t\t\"Measurement2_#4\": {\n\t\t\t\t\t\t\""
343 "type\": \"Measurement\",\n\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t"
344 "2.000000\n\t\t\t\t\t\t],\n\t\t\t\t\t\t\""
345 "unit\": \"us\"\n\t\t\t\t\t},\n\t\t\t\t\t\"Level 2A_#5\": {\n\t\t\t\t\t\t\""
346 "type\": \"Event\",\n\t\t\t\t\t\t\"Measurement1_#5\": {\n\t\t\t\t\t\t\t\"type\": \""
347 "Measurement\",\n\t\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t\t"
348 "1.000000\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\""
349 "unit\": \"ms\"\n\t\t\t\t\t\t},\n\t\t\t\t\t\t\"Measurement2_#5\": {\n\t\t\t\t\t\t\t\""
350 "type\": \"Measurement\",\n\t\t\t\t\t\t\t\"raw\": [\n\t\t\t\t\t\t\t\t"
351 "2.000000\n\t\t\t\t\t\t\t],\n\t\t\t\t\t\t\t\""
352 "unit\": \"us\"\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n");
353
354 BOOST_CHECK(output == blessedOutput);
355 armnn::ProfilerManager::GetInstance().RegisterProfiler(nullptr);
356 }
357
358 BOOST_AUTO_TEST_SUITE_END();
359