1 /**
2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "libpandabase/os/native_stack.h"
17 #include "monitor.h"
18 #include "os/mutex.h"
19 #include "runtime/include/runtime.h"
20 #include "runtime/include/thread_scopes.h"
21 #include "utils/logger.h"
22 #include "utils/span.h"
23 #include "verification/plugins.h"
24 #include "verification/util/is_system.h"
25 #include "verification/util/optional_ref.h"
26 #include "generated/base_options.h"
27
28 // generated
29 #include "ark_version.h"
30 #include "generated/verifier_options_gen.h"
31
32 #include <csignal>
33 #include <chrono>
34 #include <thread>
35
36 namespace panda::verifier {
37
38 size_t const MAX_THREADS = 64;
39
Worker(PandaDeque<Method * > * queue,os::memory::Mutex * lock,size_t threadNum,std::atomic<bool> * result)40 void Worker(PandaDeque<Method *> *queue, os::memory::Mutex *lock, size_t threadNum, std::atomic<bool> *result)
41 {
42 LOG(DEBUG, VERIFIER) << "Verifier thread " << threadNum << " starting";
43 {
44 std::stringstream ss;
45 ss << "Verifier #" << threadNum;
46 if (os::thread::SetThreadName(os::thread::GetNativeHandle(), ss.str().c_str()) != 0) {
47 LOG(ERROR, VERIFIER) << "Failed to set worker thread name " << ss.str();
48 }
49 }
50
51 auto *service = Runtime::GetCurrent()->GetVerifierService();
52 auto options = Runtime::GetCurrent()->GetOptions();
53 auto mode = options.GetVerificationMode();
54
55 ManagedThread *thread = nullptr;
56 panda_file::SourceLang currentLang = panda_file::SourceLang::INVALID;
57 for (;;) {
58 Method *method;
59 {
60 os::memory::LockHolder lh {*lock};
61 if (queue->empty()) {
62 break;
63 }
64 method = queue->front();
65 queue->pop_front();
66 }
67 auto methodLang = method->GetClass()->GetSourceLang();
68 if (methodLang != currentLang) {
69 if (thread != nullptr) {
70 plugin::GetLanguagePlugin(currentLang)->DestroyManagedThread(thread);
71 }
72 thread = plugin::GetLanguagePlugin(methodLang)->CreateManagedThread();
73 currentLang = methodLang;
74 }
75 if (panda::verifier::Verify(service, method, mode) != Status::OK) {
76 *result = false;
77 LOG(ERROR, VERIFIER) << "Error: method " << method->GetFullName(true) << " failed to verify";
78 }
79 }
80 if (thread != nullptr) {
81 plugin::GetLanguagePlugin(currentLang)->DestroyManagedThread(thread);
82 }
83 LOG(DEBUG, VERIFIER) << "Verifier thread " << threadNum << " finishing";
84 }
85
RunVerifier(const Options & cliOptions)86 bool RunVerifier(const Options &cliOptions)
87 {
88 auto isPerfMeasure = cliOptions.IsPerfMeasure();
89 std::chrono::steady_clock::time_point begin;
90 std::chrono::steady_clock::time_point end;
91
92 PandaDeque<Method *> queue;
93 os::memory::Mutex lock;
94
95 auto &runtime = *Runtime::GetCurrent();
96 auto &classLinker = *runtime.GetClassLinker();
97
98 const std::vector<std::string> &classNames = cliOptions.GetClasses();
99 const std::vector<std::string> &methodNames = cliOptions.GetMethods();
100
101 std::atomic<bool> result = true;
102
103 PandaUnorderedSet<Method *> methodsSet;
104
105 auto enqueueMethod = [&methodsSet, &queue](Method *method) {
106 if (methodsSet.count(method) == 0) {
107 queue.push_back(method);
108 methodsSet.insert(method);
109 }
110 };
111
112 bool verifyLibraries = runtime.GetOptions().IsVerifyRuntimeLibraries();
113
114 auto enqueueClass = [&verifyLibraries, &enqueueMethod](const Class &klass) {
115 if (!verifyLibraries && IsSystemClass(&klass)) {
116 LOG(INFO, VERIFIER) << klass.GetName() << " is a system class, skipping";
117 return;
118 }
119 LOG(INFO, VERIFIER) << "Begin verification of class " << klass.GetName();
120 for (auto &method : klass.GetMethods()) {
121 enqueueMethod(&method);
122 }
123 };
124
125 // we need ScopedManagedCodeThread for the verifier since it can allocate objects
126 {
127 ScopedManagedCodeThread managedObjThread(ManagedThread::GetCurrent());
128 if (classNames.empty() && methodNames.empty()) {
129 auto handleFile = [&result, &classLinker, &enqueueClass](const panda_file::File &file) {
130 LOG(INFO, VERIFIER) << "Processing file" << file.GetFilename();
131 for (auto id : file.GetClasses()) {
132 panda_file::File::EntityId entityId {id};
133 if (!file.IsExternal(entityId)) {
134 auto optLang = panda_file::ClassDataAccessor {file, entityId}.GetSourceLang();
135 if (optLang.has_value() && !IsValidSourceLang(optLang.value())) {
136 LOG(ERROR, VERIFIER) << "Unknown SourceLang";
137 result = false;
138 return false;
139 }
140 ClassLinkerExtension *ext =
141 classLinker.GetExtension(optLang.value_or(panda_file::SourceLang::PANDA_ASSEMBLY));
142 if (ext == nullptr) {
143 LOG(ERROR, VERIFIER) << "Error: Class Linker Extension failed to initialize";
144 result = false;
145 return false;
146 }
147 const Class *klass = ext->GetClass(file, entityId);
148
149 if (klass != nullptr) {
150 enqueueClass(*klass);
151 }
152 }
153 }
154 return true;
155 };
156
157 classLinker.EnumeratePandaFiles(handleFile);
158
159 if (verifyLibraries) {
160 classLinker.EnumerateBootPandaFiles(handleFile);
161 } else if (runtime.GetPandaFiles().empty()) {
162 // in this case the last boot-panda-file and only it is actually not a system file and should be
163 // verified
164 OptionalConstRef<panda_file::File> file;
165 classLinker.EnumerateBootPandaFiles([&file](const panda_file::File &pf) {
166 file = std::cref(pf);
167 return true;
168 });
169 if (file.HasRef()) {
170 handleFile(*file);
171 } else {
172 LOG(ERROR, VERIFIER) << "No files given to verify";
173 }
174 }
175 } else {
176 PandaUnorderedMap<std::string, Class *> classesByName;
177 ClassLinkerContext *ctx =
178 classLinker.GetExtension(runtime.GetLanguageContext(runtime.GetRuntimeType()))->GetBootContext();
179
180 auto getClassByName = [&classesByName, &classLinker, &ctx,
181 &result](const std::string &className) -> Class * {
182 auto it = classesByName.find(className);
183 if (it != classesByName.end()) {
184 return it->second;
185 }
186
187 PandaString descriptor;
188 const uint8_t *classNameBytes =
189 ClassHelper::GetDescriptor(utf::CStringAsMutf8(className.c_str()), &descriptor);
190 Class *klass = classLinker.GetClass(classNameBytes, true, ctx);
191 if (klass == nullptr) {
192 LOG(ERROR, VERIFIER) << "Error: Cannot resolve class with name " << className;
193 result = false;
194 }
195
196 classesByName.emplace(className, klass);
197 return klass;
198 };
199
200 for (const auto &className : classNames) {
201 Class *klass = getClassByName(className);
202 // the bad case is already handled in get_class_by_name
203 if (klass != nullptr) {
204 enqueueClass(*klass);
205 }
206 }
207
208 for (const std::string &fqMethodName : methodNames) {
209 size_t pos = fqMethodName.find_last_of("::");
210 if (pos == std::string::npos) {
211 LOG(ERROR, VERIFIER) << "Error: Fully qualified method name must contain '::', was "
212 << fqMethodName;
213 result = false;
214 break;
215 }
216 std::string className = fqMethodName.substr(0, pos - 1);
217 std::string_view unqualifiedMethodName = std::string_view(fqMethodName).substr(pos + 1);
218 if (std::find(classNames.begin(), classNames.end(), className) != classNames.end()) {
219 // this method was already verified while enumerating class_names
220 continue;
221 }
222 Class *klass = getClassByName(className);
223 if (klass != nullptr) {
224 bool methodFound = false;
225 for (auto &method : klass->GetMethods()) {
226 const char *nameData = utf::Mutf8AsCString(method.GetName().data);
227 if (std::string_view(nameData) == unqualifiedMethodName) {
228 methodFound = true;
229 LOG(INFO, VERIFIER) << "Verification of method '" << fqMethodName << "'";
230 enqueueMethod(&method);
231 }
232 }
233 if (!methodFound) {
234 LOG(ERROR, VERIFIER) << "Error: Cannot resolve method with name " << unqualifiedMethodName
235 << " in class " << className;
236 result = false;
237 }
238 }
239 }
240 }
241 }
242
243 if (isPerfMeasure) {
244 begin = std::chrono::steady_clock::now();
245 }
246
247 size_t nthreads = runtime.GetOptions().GetVerificationThreads();
248 std::vector<std::thread *> threads;
249 for (size_t i = 0; i < nthreads; i++) {
250 auto *worker = runtime.GetInternalAllocator()->New<std::thread>(Worker, &queue, &lock, i, &result);
251 threads.push_back(worker);
252 }
253
254 for (auto *thr : threads) {
255 thr->join();
256 runtime.GetInternalAllocator()->Delete(thr);
257 }
258
259 if (isPerfMeasure) {
260 end = std::chrono::steady_clock::now();
261 std::cout << "Verification time = "
262 << std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count() << " us" << std::endl;
263 }
264
265 return result;
266 }
267
PrintHelp(const PandArgParser & paParser)268 void PrintHelp(const PandArgParser &paParser)
269 {
270 std::string error = paParser.GetErrorString();
271 if (!error.empty()) {
272 error += "\n\n";
273 }
274 std::cerr << error << "Usage: verifier [option...] [file]\n"
275 << "Verify specified Panda files (given by file and --panda-files) "
276 << "or certain classes/methods in them.\n\n"
277 << paParser.GetHelpString() << std::endl;
278 }
279
Main(int argc,const char ** argv)280 int Main(int argc, const char **argv)
281 {
282 Span<const char *> sp(argv, argc);
283 RuntimeOptions runtimeOptions(sp[0]);
284 Options cliOptions(sp[0]);
285 PandArgParser paParser;
286 base_options::Options baseOptions("");
287
288 PandArg<bool> help("help", false, "Print this message and exit");
289 PandArg<bool> options("options", false, "Print verifier options");
290 // tail argument
291 PandArg<std::string> file("file", "", "path to pandafile");
292
293 cliOptions.AddOptions(&paParser);
294
295 paParser.Add(&help);
296 paParser.Add(&options);
297 paParser.PushBackTail(&file);
298 paParser.EnableTail();
299
300 if (!paParser.Parse(argc, argv)) {
301 PrintHelp(paParser);
302 return 1;
303 }
304
305 if (runtimeOptions.IsVersion()) {
306 PrintPandaVersion();
307 return 0;
308 }
309
310 if (help.GetValue()) {
311 PrintHelp(paParser);
312 return 0;
313 }
314
315 if (options.GetValue()) {
316 std::cout << paParser.GetRegularArgs() << std::endl;
317 return 0;
318 }
319
320 auto cliOptionsErr = cliOptions.Validate();
321 if (cliOptionsErr) {
322 std::cerr << "Error: " << cliOptionsErr.value().GetMessage() << std::endl;
323 return 1;
324 }
325
326 auto bootPandaFiles = cliOptions.GetBootPandaFiles();
327 auto pandaFiles = cliOptions.GetPandaFiles();
328 std::string mainFileName = file.GetValue();
329 if (!mainFileName.empty()) {
330 if (pandaFiles.empty()) {
331 bootPandaFiles.push_back(mainFileName);
332 } else {
333 auto foundIter = std::find_if(pandaFiles.begin(), pandaFiles.end(),
334 [&mainFileName](auto &fileName) { return mainFileName == fileName; });
335 if (foundIter == pandaFiles.end()) {
336 pandaFiles.push_back(mainFileName);
337 }
338 }
339 }
340
341 runtimeOptions.SetBootPandaFiles(bootPandaFiles);
342 runtimeOptions.SetPandaFiles(pandaFiles);
343 runtimeOptions.SetLoadRuntimes(cliOptions.GetLoadRuntimes());
344 runtimeOptions.SetGcType(cliOptions.GetGcType());
345
346 baseOptions.SetLogComponents(cliOptions.GetLogComponents());
347 baseOptions.SetLogLevel(cliOptions.GetLogLevel());
348 baseOptions.SetLogStream(cliOptions.GetLogStream());
349 baseOptions.SetLogFile(cliOptions.GetLogFile());
350 Logger::Initialize(baseOptions);
351
352 runtimeOptions.SetLimitStandardAlloc(cliOptions.IsLimitStandardAlloc());
353 runtimeOptions.SetInternalAllocatorType(cliOptions.GetInternalAllocatorType());
354 runtimeOptions.SetInternalMemorySizeLimit(cliOptions.GetInternalMemorySizeLimit());
355
356 runtimeOptions.SetVerificationMode(cliOptions.IsDebugMode() ? VerificationMode::DEBUG
357 : VerificationMode::AHEAD_OF_TIME);
358 runtimeOptions.SetVerificationConfigFile(cliOptions.GetConfigFile());
359 runtimeOptions.SetVerificationCacheFile(cliOptions.GetCacheFile());
360 runtimeOptions.SetVerificationUpdateCache(cliOptions.IsUpdateCache());
361 runtimeOptions.SetVerifyRuntimeLibraries(cliOptions.IsVerifyRuntimeLibraries());
362 uint32_t threads = cliOptions.GetThreads();
363 if (threads == 0) {
364 threads = std::thread::hardware_concurrency();
365 // hardware_concurrency can return 0 if the value is not computable or well defined
366 if (threads == 0) {
367 threads = 1;
368 } else if (threads > MAX_THREADS) {
369 threads = MAX_THREADS;
370 }
371 }
372 runtimeOptions.SetVerificationThreads(threads);
373
374 if (!Runtime::Create(runtimeOptions)) {
375 std::cerr << "Error: cannot create runtime" << std::endl;
376 return -1;
377 }
378
379 int ret = RunVerifier(cliOptions) ? 0 : -1;
380
381 if (cliOptions.IsPrintMemoryStatistics()) {
382 std::cout << Runtime::GetCurrent()->GetMemoryStatistics();
383 }
384 if (!Runtime::Destroy()) {
385 std::cerr << "Error: cannot destroy runtime" << std::endl;
386 return -1;
387 }
388 return ret;
389 }
390
391 } // namespace panda::verifier
392
main(int argc,const char ** argv)393 int main(int argc, const char **argv)
394 {
395 return panda::verifier::Main(argc, argv);
396 }
397