1 /*
2 * Copyright (c) 2017-2021 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "Framework.h"
25
26 #include "arm_compute/runtime/Scheduler.h"
27 #include "tests/framework/ParametersLibrary.h"
28 #include "tests/framework/TestFilter.h"
29
30 #ifdef ARM_COMPUTE_CL
31 #include "arm_compute/runtime/CL/CLRuntimeContext.h"
32 #include "arm_compute/runtime/CL/CLScheduler.h"
33
34 #endif /* ARM_COMPUTE_CL */
35
36 #include <chrono>
37 #include <iostream>
38 #include <memory>
39 #include <sstream>
40 #include <type_traits>
41
42 namespace arm_compute
43 {
44 namespace test
45 {
46 std::unique_ptr<ParametersLibrary> parameters;
47
48 namespace framework
49 {
50 std::unique_ptr<InstrumentsInfo> instruments_info;
51
Framework()52 Framework::Framework()
53 : _test_filter(nullptr)
54 {
55 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimestamps, ScaleFactor::NONE>);
56 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_MS),
57 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
58 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_S),
59 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
60 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimer, ScaleFactor::NONE>);
61 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_MS>);
62 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_S>);
63 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::NONE>);
64 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_MS),
65 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
66 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_S),
67 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
68 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimer, ScaleFactor::NONE>);
69 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_MS>);
70 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_S>);
71 #ifdef PMU_ENABLED
72 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::NONE), Instrument::make_instrument<PMUCounter, ScaleFactor::NONE>);
73 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1K), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1K>);
74 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1M), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1M>);
75 #endif /* PMU_ENABLED */
76 #ifdef MALI_ENABLED
77 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::NONE), Instrument::make_instrument<MaliCounter, ScaleFactor::NONE>);
78 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1K), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1K>);
79 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1M), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1M>);
80 #endif /* MALI_ENABLED */
81 #ifdef ARM_COMPUTE_CL
82 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::NONE>);
83 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_US>);
84 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_MS>);
85 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_S>);
86 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimer, ScaleFactor::NONE>);
87 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_US>);
88 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_MS>);
89 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_S>);
90 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::NONE), Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::NONE>);
91 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1K),
92 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
93 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1M),
94 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
95 #endif /* ARM_COMPUTE_CL */
96
97 instruments_info = std::make_unique<InstrumentsInfo>();
98 }
99
available_instruments() const100 std::set<InstrumentsDescription> Framework::available_instruments() const
101 {
102 std::set<InstrumentsDescription> types;
103
104 for(const auto &instrument : _available_instruments)
105 {
106 types.emplace(instrument.first);
107 }
108
109 return types;
110 }
111
count_test_results() const112 std::map<TestResult::Status, int> Framework::count_test_results() const
113 {
114 std::map<TestResult::Status, int> counts;
115
116 for(const auto &test : _test_results)
117 {
118 ++counts[test.second.status];
119 }
120
121 return counts;
122 }
123
get()124 Framework &Framework::get()
125 {
126 static Framework instance;
127 return instance;
128 }
129
init(const FrameworkConfig & config)130 void Framework::init(const FrameworkConfig &config)
131 {
132 _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter));
133 _num_iterations = config.num_iterations;
134 _log_level = config.log_level;
135 _cooldown_sec = config.cooldown_sec;
136 _configure_only = config.configure_only;
137
138 _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
139 }
140
current_suite_name() const141 std::string Framework::current_suite_name() const
142 {
143 return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/");
144 }
145
push_suite(std::string name)146 void Framework::push_suite(std::string name)
147 {
148 _test_suite_name.emplace_back(std::move(name));
149 }
150
pop_suite()151 void Framework::pop_suite()
152 {
153 _test_suite_name.pop_back();
154 }
155
add_test_info(std::string info)156 void Framework::add_test_info(std::string info)
157 {
158 _test_info.emplace_back(std::move(info));
159 }
160
clear_test_info()161 void Framework::clear_test_info()
162 {
163 _test_info.clear();
164 }
165
has_test_info() const166 bool Framework::has_test_info() const
167 {
168 return !_test_info.empty();
169 }
170
print_test_info(std::ostream & os) const171 void Framework::print_test_info(std::ostream &os) const
172 {
173 if(!_test_info.empty())
174 {
175 os << "CONTEXT:\n";
176
177 for(const auto &str : _test_info)
178 {
179 os << " " << str << "\n";
180 }
181 }
182 }
183
184 template <typename F>
func_on_all_printers(F && func)185 void Framework::func_on_all_printers(F &&func)
186 {
187 std::for_each(std::begin(_printers), std::end(_printers), func);
188 }
189
log_test_start(const TestInfo & info)190 void Framework::log_test_start(const TestInfo &info)
191 {
192 if(_log_level >= LogLevel::TESTS)
193 {
194 func_on_all_printers([&](Printer * p)
195 {
196 p->print_test_header(info);
197 });
198 }
199 }
200
log_test_skipped(const TestInfo & info)201 void Framework::log_test_skipped(const TestInfo &info)
202 {
203 static_cast<void>(info);
204 }
205
log_test_end(const TestInfo & info)206 void Framework::log_test_end(const TestInfo &info)
207 {
208 if(_log_level >= LogLevel::MEASUREMENTS)
209 {
210 func_on_all_printers([&](Printer * p)
211 {
212 p->print_profiler_header(_test_results.at(info).header_data);
213 p->print_measurements(_test_results.at(info).measurements);
214 });
215 }
216
217 if(_log_level >= LogLevel::TESTS)
218 {
219 func_on_all_printers([](Printer * p)
220 {
221 p->print_test_footer();
222 });
223 }
224 }
225
log_failed_expectation(const TestError & error)226 void Framework::log_failed_expectation(const TestError &error)
227 {
228 ARM_COMPUTE_ERROR_ON(_current_test_info == nullptr);
229 ARM_COMPUTE_ERROR_ON(_current_test_result == nullptr);
230
231 const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
232
233 if(_log_level >= error.level())
234 {
235 func_on_all_printers([&](Printer * p)
236 {
237 p->print_error(error, is_expected_failure);
238 });
239 }
240
241 _current_test_result->status = TestResult::Status::FAILED;
242 }
243
log_info(const std::string & info)244 void Framework::log_info(const std::string &info)
245 {
246 if(_log_level >= LogLevel::DEBUG)
247 {
248 func_on_all_printers([&](Printer * p)
249 {
250 p->print_info(info);
251 });
252 }
253 }
254
num_iterations() const255 int Framework::num_iterations() const
256 {
257 return _num_iterations;
258 }
259
set_num_iterations(int num_iterations)260 void Framework::set_num_iterations(int num_iterations)
261 {
262 _num_iterations = num_iterations;
263 }
264
set_throw_errors(bool throw_errors)265 void Framework::set_throw_errors(bool throw_errors)
266 {
267 _throw_errors = throw_errors;
268 }
269
throw_errors() const270 bool Framework::throw_errors() const
271 {
272 return _throw_errors;
273 }
274
set_stop_on_error(bool stop_on_error)275 void Framework::set_stop_on_error(bool stop_on_error)
276 {
277 _stop_on_error = stop_on_error;
278 }
279
stop_on_error() const280 bool Framework::stop_on_error() const
281 {
282 return _stop_on_error;
283 }
284
set_error_on_missing_assets(bool error_on_missing_assets)285 void Framework::set_error_on_missing_assets(bool error_on_missing_assets)
286 {
287 _error_on_missing_assets = error_on_missing_assets;
288 }
289
error_on_missing_assets() const290 bool Framework::error_on_missing_assets() const
291 {
292 return _error_on_missing_assets;
293 }
294
run_test(const TestInfo & info,TestCaseFactory & test_factory)295 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
296 {
297 if(test_factory.status() == TestCaseFactory::Status::DISABLED)
298 {
299 log_test_skipped(info);
300 set_test_result(info, TestResult(TestResult::Status::DISABLED));
301 return;
302 }
303
304 log_test_start(info);
305
306 Profiler profiler = get_profiler();
307 TestResult result(TestResult::Status::NOT_RUN);
308
309 _current_test_info = &info;
310 _current_test_result = &result;
311
312 if(_log_level >= LogLevel::ERRORS)
313 {
314 func_on_all_printers([](Printer * p)
315 {
316 p->print_errors_header();
317 });
318 }
319
320 const bool is_expected_failure = info.status == TestCaseFactory::Status::EXPECTED_FAILURE;
321
322 try
323 {
324 std::unique_ptr<TestCase> test_case = test_factory.make();
325
326 try
327 {
328 profiler.test_start();
329
330 test_case->do_setup();
331
332 for(int i = 0; i < _num_iterations; ++i)
333 {
334 //Start the profiler if:
335 //- there is only one iteration
336 //- it's not the first iteration of a multi-iterations run.
337 //
338 //Reason: if the CLTuner is enabled then the first run will be really messy
339 //as each kernel will be executed several times, messing up the instruments like OpenCL timers.
340 if(_num_iterations == 1 || i != 0)
341 {
342 profiler.start();
343 }
344 test_case->do_run();
345 test_case->do_sync();
346 if(_num_iterations == 1 || i != 0)
347 {
348 profiler.stop();
349 }
350 }
351
352 test_case->do_teardown();
353
354 profiler.test_stop();
355
356 // Change status to success if no error has happend
357 if(result.status == TestResult::Status::NOT_RUN)
358 {
359 result.status = TestResult::Status::SUCCESS;
360 }
361 }
362 catch(const FileNotFound &error)
363 {
364 profiler.test_stop();
365 if(_error_on_missing_assets)
366 {
367 if(_log_level >= LogLevel::ERRORS)
368 {
369 TestError test_error(error.what(), LogLevel::ERRORS);
370 func_on_all_printers([&](Printer * p)
371 {
372 p->print_error(test_error, is_expected_failure);
373 });
374 }
375
376 result.status = TestResult::Status::FAILED;
377
378 if(_throw_errors)
379 {
380 throw;
381 }
382 }
383 else
384 {
385 if(_log_level >= LogLevel::DEBUG)
386 {
387 func_on_all_printers([&](Printer * p)
388 {
389 p->print_info(error.what());
390 });
391 }
392
393 result.status = TestResult::Status::NOT_RUN;
394 }
395 }
396 catch(const TestError &error)
397 {
398 profiler.test_stop();
399 if(_log_level >= error.level())
400 {
401 func_on_all_printers([&](Printer * p)
402 {
403 p->print_error(error, is_expected_failure);
404 });
405 }
406
407 result.status = TestResult::Status::FAILED;
408
409 if(_throw_errors)
410 {
411 throw;
412 }
413 }
414 #ifdef ARM_COMPUTE_CL
415 catch(const ::cl::Error &error)
416 {
417 profiler.test_stop();
418 if(_log_level >= LogLevel::ERRORS)
419 {
420 std::stringstream stream;
421 stream << "Error code: " << error.err();
422 TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
423 func_on_all_printers([&](Printer * p)
424 {
425 p->print_error(test_error, is_expected_failure);
426 });
427 }
428
429 result.status = TestResult::Status::FAILED;
430
431 if(_throw_errors)
432 {
433 throw;
434 }
435 }
436 #endif /* ARM_COMPUTE_CL */
437 catch(const std::exception &error)
438 {
439 profiler.test_stop();
440 if(_log_level >= LogLevel::ERRORS)
441 {
442 func_on_all_printers([&](Printer * p)
443 {
444 p->print_error(error, is_expected_failure);
445 });
446 }
447
448 result.status = TestResult::Status::CRASHED;
449
450 if(_throw_errors)
451 {
452 throw;
453 }
454 }
455 catch(...)
456 {
457 profiler.test_stop();
458 if(_log_level >= LogLevel::ERRORS)
459 {
460 func_on_all_printers([&](Printer * p)
461 {
462 p->print_error(TestError("Received unknown exception"), is_expected_failure);
463 });
464 }
465
466 result.status = TestResult::Status::CRASHED;
467
468 if(_throw_errors)
469 {
470 throw;
471 }
472 }
473 }
474 catch(const std::exception &error)
475 {
476 if(_log_level >= LogLevel::ERRORS)
477 {
478 func_on_all_printers([&](Printer * p)
479 {
480 p->print_error(error, is_expected_failure);
481 });
482 }
483
484 result.status = TestResult::Status::CRASHED;
485
486 if(_throw_errors)
487 {
488 throw;
489 }
490 }
491 catch(...)
492 {
493 if(_log_level >= LogLevel::ERRORS)
494 {
495 func_on_all_printers([&](Printer * p)
496 {
497 p->print_error(TestError("Received unknown exception"), is_expected_failure);
498 });
499 }
500
501 result.status = TestResult::Status::CRASHED;
502
503 if(_throw_errors)
504 {
505 throw;
506 }
507 }
508
509 if(_log_level >= LogLevel::ERRORS)
510 {
511 func_on_all_printers([](Printer * p)
512 {
513 p->print_errors_footer();
514 });
515 }
516
517 _current_test_info = nullptr;
518 _current_test_result = nullptr;
519
520 if(result.status == TestResult::Status::FAILED)
521 {
522 if(info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
523 {
524 result.status = TestResult::Status::EXPECTED_FAILURE;
525 }
526 }
527
528 if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED)
529 {
530 if(_stop_on_error)
531 {
532 throw std::runtime_error("Abandon on first error.");
533 }
534 }
535
536 result.header_data = profiler.header();
537 result.measurements = profiler.measurements();
538
539 set_test_result(info, result);
540 log_test_end(info);
541 }
542
run()543 bool Framework::run()
544 {
545 // Clear old test results
546 _test_results.clear();
547
548 if(_log_level >= LogLevel::TESTS)
549 {
550 func_on_all_printers([](Printer * p)
551 {
552 p->print_run_header();
553 });
554 }
555
556 const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
557
558 int id = 0;
559 int id_run_test = 0;
560
561 for(auto &test_factory : _test_factories)
562 {
563 const std::string test_case_name = test_factory->name();
564 const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
565
566 if(_test_filter->is_selected(test_info))
567 {
568 #ifdef ARM_COMPUTE_CL
569 // Every 100 tests, reset the OpenCL context to release the allocated memory
570 if(opencl_is_available() && (id_run_test % 100) == 0)
571 {
572 auto ctx_properties = CLScheduler::get().context().getInfo<CL_CONTEXT_PROPERTIES>(nullptr);
573 auto queue_properties = CLScheduler::get().queue().getInfo<CL_QUEUE_PROPERTIES>(nullptr);
574
575 cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
576 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx, CLKernelLibrary::get().get_device(), queue_properties);
577
578 CLKernelLibrary::get().clear_programs_cache();
579 CLScheduler::get().set_context(new_ctx);
580 CLScheduler::get().set_queue(new_queue);
581 }
582 #endif // ARM_COMPUTE_CL
583
584 run_test(test_info, *test_factory);
585
586 ++id_run_test;
587
588 // Run test delay
589 sleep_in_seconds(_cooldown_sec);
590 }
591
592 ++id;
593 }
594
595 const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
596
597 if(_log_level >= LogLevel::TESTS)
598 {
599 func_on_all_printers([](Printer * p)
600 {
601 p->print_run_footer();
602 });
603 }
604
605 auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
606 std::map<TestResult::Status, int> results = count_test_results();
607
608 if(_log_level > LogLevel::NONE)
609 {
610 std::cout << "Executed " << _test_results.size() << " test(s) ("
611 << results[TestResult::Status::SUCCESS] << " passed, "
612 << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
613 << results[TestResult::Status::FAILED] << " failed, "
614 << results[TestResult::Status::CRASHED] << " crashed, "
615 << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
616 }
617
618 int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
619
620 return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
621 }
622
set_test_result(TestInfo info,TestResult result)623 void Framework::set_test_result(TestInfo info, TestResult result)
624 {
625 _test_results.emplace(std::move(info), std::move(result));
626 }
627
print_test_results(Printer & printer) const628 void Framework::print_test_results(Printer &printer) const
629 {
630 printer.print_run_header();
631
632 for(const auto &test : _test_results)
633 {
634 printer.print_test_header(test.first);
635 printer.print_profiler_header(test.second.header_data);
636 printer.print_measurements(test.second.measurements);
637 printer.print_test_footer();
638 }
639
640 printer.print_run_footer();
641 }
642
get_profiler() const643 Profiler Framework::get_profiler() const
644 {
645 Profiler profiler;
646
647 const bool all_instruments = std::any_of(
648 _instruments.begin(),
649 _instruments.end(),
650 [](InstrumentsDescription type) -> bool { return type.first == InstrumentType::ALL; });
651
652 auto is_selected = [&](InstrumentsDescription instrument) -> bool
653 {
654 return std::find_if(_instruments.begin(), _instruments.end(), [&](InstrumentsDescription type) -> bool {
655 const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(type.first) & 0xFF00);
656 return (group == instrument.first) && (instrument.second == type.second);
657 })
658 != _instruments.end();
659 };
660
661 for(const auto &instrument : _available_instruments)
662 {
663 if(all_instruments || is_selected(instrument.first))
664 {
665 profiler.add(instrument.second());
666 }
667 }
668
669 return profiler;
670 }
671
add_printer(Printer * printer)672 void Framework::add_printer(Printer *printer)
673 {
674 _printers.push_back(printer);
675 }
676
test_infos() const677 std::vector<TestInfo> Framework::test_infos() const
678 {
679 std::vector<TestInfo> ids;
680
681 int id = 0;
682
683 for(const auto &factory : _test_factories)
684 {
685 const TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
686
687 if(_test_filter->is_selected(test_info))
688 {
689 ids.emplace_back(std::move(test_info));
690 }
691
692 ++id;
693 }
694
695 return ids;
696 }
697
log_level() const698 LogLevel Framework::log_level() const
699 {
700 return _log_level;
701 }
702
set_instruments_info(InstrumentsInfo instr_info)703 void Framework::set_instruments_info(InstrumentsInfo instr_info)
704 {
705 ARM_COMPUTE_ERROR_ON(instruments_info == nullptr);
706 *instruments_info = instr_info;
707 }
708
configure_only() const709 bool Framework::configure_only() const
710 {
711 return _configure_only;
712 }
713
new_fixture_call() const714 bool Framework::new_fixture_call() const
715 {
716 return _new_fixture_call;
717 }
718
set_new_fixture_call(bool val)719 void Framework::set_new_fixture_call(bool val)
720 {
721 _new_fixture_call = val;
722 }
723 } // namespace framework
724 } // namespace test
725 } // namespace arm_compute
726