1 #include <ATen/Functions.h>
2 #include <gtest/gtest.h>
3 #include <test/cpp/jit/test_utils.h>
4 #include <torch/csrc/jit/api/module.h>
5 #include <torch/csrc/jit/frontend/resolver.h>
6 #include <torch/csrc/jit/mobile/import.h>
7 #include <torch/csrc/jit/mobile/module.h>
8 #include <torch/csrc/jit/mobile/profiler_edge.h>
9 #include <fstream>
10
11 #include <unordered_set>
12
13 #include <torch/csrc/profiler/events.h>
14
15 #include "test/cpp/lite_interpreter_runtime/resources.h"
16
17 #ifdef EDGE_PROFILER_USE_KINETO
18 namespace torch {
19 namespace jit {
20 namespace mobile {
21
22 namespace {
checkMetaData(const std::string & op_name,const std::string & metadata_name,const std::string & metadata_val,std::ifstream & trace_file)23 bool checkMetaData(
24 const std::string& op_name,
25 const std::string& metadata_name,
26 const std::string& metadata_val,
27 std::ifstream& trace_file) {
28 std::string line;
29 while (std::getline(trace_file, line)) {
30 if (line.find(op_name) != std::string::npos) {
31 while (std::getline(trace_file, line)) {
32 if (line.find(metadata_name) != std::string::npos) {
33 if (line.find(metadata_val) != std::string::npos ||
34 !metadata_val.size()) {
35 /* if found the right metadata_val OR if expected
36 * metadata value is an empty string then ignore the metadata_val */
37 return true;
38 }
39 }
40 }
41 }
42 }
43 return false;
44 }
45 } // namespace
46
TEST(MobileProfiler,ModuleHierarchy)47 TEST(MobileProfiler, ModuleHierarchy) {
48 auto testModelFile = torch::testing::getResourcePath(
49 "test/cpp/lite_interpreter_runtime/to_be_profiled_module.ptl");
50
51 std::vector<IValue> inputs;
52 inputs.emplace_back(at::rand({64, 64}));
53 inputs.emplace_back(at::rand({64, 64}));
54 std::string trace_file_name("/tmp/test_trace.trace");
55
56 mobile::Module bc = _load_for_mobile(testModelFile.string());
57 {
58 KinetoEdgeCPUProfiler profiler(
59 bc,
60 trace_file_name,
61 false, // record input_shapes
62 false, // profile memory
63 true, // record callstack
64 false, // record flops
65 true, // record module hierarchy
66 {}, // events
67 false); // adjust_vulkan_timestamps
68 bc.forward(inputs);
69 } // End of profiler
70 std::ifstream trace_file(trace_file_name);
71 std::string line;
72 ASSERT_TRUE(trace_file.is_open());
73 trace_file.seekg(0, std::ios_base::beg);
74 const std::string metadata_name("Module Hierarchy");
75 ASSERT_TRUE(checkMetaData(
76 "aten::sub",
77 metadata_name,
78 "top(C)::<unknown>.A0(A)::forward.aten::sub",
79 trace_file));
80 trace_file.seekg(0, std::ios_base::beg);
81 ASSERT_TRUE(checkMetaData(
82 "aten::mul",
83 metadata_name,
84 "top(C)::<unknown>.A0(A)::forward.SELF(A)::forward_impl_.SELF(A)::my_new_method.aten::mul",
85 trace_file));
86 trace_file.seekg(0, std::ios_base::beg);
87 ASSERT_TRUE(checkMetaData(
88 "aten::add",
89 metadata_name,
90 "top(C)::<unknown>.A0(A)::forward.SELF(A)::forward_impl_.aten::add",
91 trace_file));
92 ASSERT_TRUE(checkMetaData(
93 "aten::add",
94 metadata_name,
95 "top(C)::<unknown>.SELF(C)::call_b.B0(B)::forward.aten::add",
96 trace_file));
97 ASSERT_TRUE(checkMetaData(
98 "aten::add", metadata_name, "top(C)::<unknown>.aten::add", trace_file));
99 }
100
TEST(MobileProfiler,Backend)101 TEST(MobileProfiler, Backend) {
102 auto testModelFile = torch::testing::getResourcePath(
103 "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl");
104
105 std::vector<IValue> inputs;
106 inputs.emplace_back(at::rand({64, 64}));
107 inputs.emplace_back(at::rand({64, 64}));
108 std::string trace_file_name("/tmp/test_trace_backend.trace");
109
110 mobile::Module bc = _load_for_mobile(testModelFile.string());
111 {
112 KinetoEdgeCPUProfiler profiler(
113 bc,
114 trace_file_name,
115 false, // record input_shapes
116 false, // profile memory
117 true, // record callstack
118 false, // record flops
119 true); // record module hierarchy
120 bc.forward(inputs);
121 } // End of profiler
122 std::ifstream trace_file(trace_file_name);
123 std::string line;
124 ASSERT_TRUE(trace_file.is_open());
125 trace_file.seekg(0, std::ios_base::beg);
126 std::string metadata_name("Module Hierarchy");
127 ASSERT_TRUE(checkMetaData(
128 "aten::add", metadata_name, "top(m)::<unknown>.aten::add", trace_file));
129 trace_file.seekg(0, std::ios_base::beg);
130 metadata_name = "Backend";
131 ASSERT_TRUE(
132 checkMetaData("aten::add", metadata_name, "test_backend", trace_file));
133 }
134
TEST(MobileProfiler,BackendMemoryEvents)135 TEST(MobileProfiler, BackendMemoryEvents) {
136 auto testModelFile = torch::testing::getResourcePath(
137 "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl");
138
139 std::vector<IValue> inputs;
140 inputs.emplace_back(at::rand({64, 64}));
141 inputs.emplace_back(at::rand({64, 64}));
142 std::string trace_file_name("/tmp/test_trace_backend_memory.trace");
143
144 mobile::Module bc = _load_for_mobile(testModelFile.string());
145 {
146 mobile::KinetoEdgeCPUProfiler profiler(
147 bc,
148 trace_file_name,
149 false, // record input_shapes
150 true, // profile memory
151 true, // record callstack
152 false, // record flops
153 true); // record module hierarchy
154 bc.forward(inputs);
155 }
156 std::ifstream trace_file(trace_file_name);
157 std::string line;
158 ASSERT_TRUE(trace_file.is_open());
159 trace_file.seekg(0, std::ios_base::beg);
160 std::string metadata_name("Bytes");
161 ASSERT_TRUE(checkMetaData("[memory]", metadata_name, "16384", trace_file));
162 trace_file.seekg(0, std::ios_base::beg);
163 metadata_name = "Total Reserved";
164 ASSERT_TRUE(checkMetaData("[memory]", metadata_name, "49152", trace_file));
165 }
166
TEST(MobileProfiler,ProfilerEvent)167 TEST(MobileProfiler, ProfilerEvent) {
168 auto testModelFile = torch::testing::getResourcePath(
169 "test/cpp/lite_interpreter_runtime/test_backend_for_profiling.ptl");
170
171 std::vector<IValue> inputs;
172 inputs.emplace_back(at::rand({64, 64}));
173 inputs.emplace_back(at::rand({64, 64}));
174 std::string trace_file_name("/tmp/test_trace_profiler_event.trace");
175
176 std::vector<std::string> events(
177 torch::profiler::ProfilerPerfEvents.begin(),
178 torch::profiler::ProfilerPerfEvents.end());
179
180 mobile::Module bc = _load_for_mobile(testModelFile.string());
181 {
182 // Bail if something goes wrong here
183 try {
184 KinetoEdgeCPUProfiler profiler(
185 bc,
186 trace_file_name,
187 false, // record input_shapes
188 false, // profile memory
189 true, // record callstack
190 false, // record flops
191 true, // record module hierarchy
192 events); // performance events
193 bc.forward(inputs);
194 } catch (...) {
195 return;
196 }
197 } // End of profiler
198 std::ifstream trace_file(trace_file_name);
199 std::string line;
200 ASSERT_TRUE(trace_file.is_open());
201
202 for (auto& event : events) {
203 trace_file.seekg(0, std::ios_base::beg);
204 /*
205 * Just checking if the event entry exists in the chrometrace.
206 * Checking the value in a hardware independent matter is tricky.
207 */
208 ASSERT_TRUE(checkMetaData("aten::__getitem__", event, "", trace_file));
209 }
210 }
211
212 } // namespace mobile
213 } // namespace jit
214 } // namespace torch
215 #endif
216