1 /*
2 * Copyright (c) 2017-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "Framework.h"
25
26 #include "arm_compute/runtime/Scheduler.h"
27 #include "support/MemorySupport.h"
28 #include "tests/framework/ParametersLibrary.h"
29 #include "tests/framework/TestFilter.h"
30
31 #ifdef ARM_COMPUTE_CL
32 #include "arm_compute/runtime/CL/CLRuntimeContext.h"
33 #include "arm_compute/runtime/CL/CLScheduler.h"
34
35 #endif /* ARM_COMPUTE_CL */
36
37 #include <chrono>
38 #include <iostream>
39 #include <sstream>
40 #include <type_traits>
41
42 namespace arm_compute
43 {
44 namespace test
45 {
46 std::unique_ptr<ParametersLibrary> parameters;
47
48 namespace framework
49 {
50 std::unique_ptr<InstrumentsInfo> instruments_info;
51
Framework()52 Framework::Framework()
53 : _test_filter(nullptr)
54 {
55 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimestamps, ScaleFactor::NONE>);
56 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_MS),
57 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_MS>);
58 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMESTAMPS, ScaleFactor::TIME_S),
59 Instrument::make_instrument<WallClockTimestamps, ScaleFactor::TIME_S>);
60 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::NONE), Instrument::make_instrument<WallClockTimer, ScaleFactor::NONE>);
61 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_MS>);
62 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::WALL_CLOCK_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<WallClockTimer, ScaleFactor::TIME_S>);
63 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::NONE>);
64 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_MS),
65 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_MS>);
66 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMESTAMPS, ScaleFactor::TIME_S),
67 Instrument::make_instrument<SchedulerTimestamps, ScaleFactor::TIME_S>);
68 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::NONE), Instrument::make_instrument<SchedulerTimer, ScaleFactor::NONE>);
69 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_MS>);
70 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::SCHEDULER_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<SchedulerTimer, ScaleFactor::TIME_S>);
71 #ifdef PMU_ENABLED
72 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::NONE), Instrument::make_instrument<PMUCounter, ScaleFactor::NONE>);
73 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1K), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1K>);
74 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::PMU, ScaleFactor::SCALE_1M), Instrument::make_instrument<PMUCounter, ScaleFactor::SCALE_1M>);
75 #endif /* PMU_ENABLED */
76 #ifdef MALI_ENABLED
77 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::NONE), Instrument::make_instrument<MaliCounter, ScaleFactor::NONE>);
78 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1K), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1K>);
79 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::MALI, ScaleFactor::SCALE_1M), Instrument::make_instrument<MaliCounter, ScaleFactor::SCALE_1M>);
80 #endif /* MALI_ENABLED */
81 #ifdef ARM_COMPUTE_CL
82 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::NONE>);
83 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_US>);
84 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_MS>);
85 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMESTAMPS, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimestamps, ScaleFactor::TIME_S>);
86 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::NONE), Instrument::make_instrument<OpenCLTimer, ScaleFactor::NONE>);
87 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_US), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_US>);
88 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_MS), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_MS>);
89 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_TIMER, ScaleFactor::TIME_S), Instrument::make_instrument<OpenCLTimer, ScaleFactor::TIME_S>);
90 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::NONE), Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::NONE>);
91 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1K),
92 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1K>);
93 _available_instruments.emplace(std::pair<InstrumentType, ScaleFactor>(InstrumentType::OPENCL_MEMORY_USAGE, ScaleFactor::SCALE_1M),
94 Instrument::make_instrument<OpenCLMemoryUsage, ScaleFactor::SCALE_1M>);
95 #endif /* ARM_COMPUTE_CL */
96
97 instruments_info = support::cpp14::make_unique<InstrumentsInfo>();
98 }
99
available_instruments() const100 std::set<InstrumentsDescription> Framework::available_instruments() const
101 {
102 std::set<InstrumentsDescription> types;
103
104 for(const auto &instrument : _available_instruments)
105 {
106 types.emplace(instrument.first);
107 }
108
109 return types;
110 }
111
count_test_results() const112 std::map<TestResult::Status, int> Framework::count_test_results() const
113 {
114 std::map<TestResult::Status, int> counts;
115
116 for(const auto &test : _test_results)
117 {
118 ++counts[test.second.status];
119 }
120
121 return counts;
122 }
123
get()124 Framework &Framework::get()
125 {
126 static Framework instance;
127 return instance;
128 }
129
init(const FrameworkConfig & config)130 void Framework::init(const FrameworkConfig &config)
131 {
132 _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter));
133 _num_iterations = config.num_iterations;
134 _log_level = config.log_level;
135 _cooldown_sec = config.cooldown_sec;
136
137 _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
138 }
139
current_suite_name() const140 std::string Framework::current_suite_name() const
141 {
142 return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/");
143 }
144
push_suite(std::string name)145 void Framework::push_suite(std::string name)
146 {
147 _test_suite_name.emplace_back(std::move(name));
148 }
149
pop_suite()150 void Framework::pop_suite()
151 {
152 _test_suite_name.pop_back();
153 }
154
add_test_info(std::string info)155 void Framework::add_test_info(std::string info)
156 {
157 _test_info.emplace_back(std::move(info));
158 }
159
clear_test_info()160 void Framework::clear_test_info()
161 {
162 _test_info.clear();
163 }
164
has_test_info() const165 bool Framework::has_test_info() const
166 {
167 return !_test_info.empty();
168 }
169
print_test_info(std::ostream & os) const170 void Framework::print_test_info(std::ostream &os) const
171 {
172 if(!_test_info.empty())
173 {
174 os << "CONTEXT:\n";
175
176 for(const auto &str : _test_info)
177 {
178 os << " " << str << "\n";
179 }
180 }
181 }
182
183 template <typename F>
func_on_all_printers(F && func)184 void Framework::func_on_all_printers(F &&func)
185 {
186 std::for_each(std::begin(_printers), std::end(_printers), func);
187 }
188
log_test_start(const TestInfo & info)189 void Framework::log_test_start(const TestInfo &info)
190 {
191 if(_log_level >= LogLevel::TESTS)
192 {
193 func_on_all_printers([&](Printer * p)
194 {
195 p->print_test_header(info);
196 });
197 }
198 }
199
log_test_skipped(const TestInfo & info)200 void Framework::log_test_skipped(const TestInfo &info)
201 {
202 static_cast<void>(info);
203 }
204
log_test_end(const TestInfo & info)205 void Framework::log_test_end(const TestInfo &info)
206 {
207 if(_log_level >= LogLevel::MEASUREMENTS)
208 {
209 func_on_all_printers([&](Printer * p)
210 {
211 p->print_measurements(_test_results.at(info).measurements);
212 });
213 }
214
215 if(_log_level >= LogLevel::TESTS)
216 {
217 func_on_all_printers([](Printer * p)
218 {
219 p->print_test_footer();
220 });
221 }
222 }
223
log_failed_expectation(const TestError & error)224 void Framework::log_failed_expectation(const TestError &error)
225 {
226 ARM_COMPUTE_ERROR_ON(_current_test_info == nullptr);
227 ARM_COMPUTE_ERROR_ON(_current_test_result == nullptr);
228
229 const bool is_expected_failure = _current_test_info->status == TestCaseFactory::Status::EXPECTED_FAILURE;
230
231 if(_log_level >= error.level())
232 {
233 func_on_all_printers([&](Printer * p)
234 {
235 p->print_error(error, is_expected_failure);
236 });
237 }
238
239 _current_test_result->status = TestResult::Status::FAILED;
240 }
241
log_info(const std::string & info)242 void Framework::log_info(const std::string &info)
243 {
244 if(_log_level >= LogLevel::DEBUG)
245 {
246 func_on_all_printers([&](Printer * p)
247 {
248 p->print_info(info);
249 });
250 }
251 }
252
num_iterations() const253 int Framework::num_iterations() const
254 {
255 return _num_iterations;
256 }
257
set_num_iterations(int num_iterations)258 void Framework::set_num_iterations(int num_iterations)
259 {
260 _num_iterations = num_iterations;
261 }
262
set_throw_errors(bool throw_errors)263 void Framework::set_throw_errors(bool throw_errors)
264 {
265 _throw_errors = throw_errors;
266 }
267
throw_errors() const268 bool Framework::throw_errors() const
269 {
270 return _throw_errors;
271 }
272
set_stop_on_error(bool stop_on_error)273 void Framework::set_stop_on_error(bool stop_on_error)
274 {
275 _stop_on_error = stop_on_error;
276 }
277
stop_on_error() const278 bool Framework::stop_on_error() const
279 {
280 return _stop_on_error;
281 }
282
set_error_on_missing_assets(bool error_on_missing_assets)283 void Framework::set_error_on_missing_assets(bool error_on_missing_assets)
284 {
285 _error_on_missing_assets = error_on_missing_assets;
286 }
287
error_on_missing_assets() const288 bool Framework::error_on_missing_assets() const
289 {
290 return _error_on_missing_assets;
291 }
292
run_test(const TestInfo & info,TestCaseFactory & test_factory)293 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
294 {
295 if(test_factory.status() == TestCaseFactory::Status::DISABLED)
296 {
297 log_test_skipped(info);
298 set_test_result(info, TestResult(TestResult::Status::DISABLED));
299 return;
300 }
301
302 log_test_start(info);
303
304 Profiler profiler = get_profiler();
305 TestResult result(TestResult::Status::NOT_RUN);
306
307 _current_test_info = &info;
308 _current_test_result = &result;
309
310 if(_log_level >= LogLevel::ERRORS)
311 {
312 func_on_all_printers([](Printer * p)
313 {
314 p->print_errors_header();
315 });
316 }
317
318 const bool is_expected_failure = info.status == TestCaseFactory::Status::EXPECTED_FAILURE;
319
320 try
321 {
322 std::unique_ptr<TestCase> test_case = test_factory.make();
323
324 try
325 {
326 profiler.test_start();
327
328 test_case->do_setup();
329
330 for(int i = 0; i < _num_iterations; ++i)
331 {
332 //Start the profiler if:
333 //- there is only one iteration
334 //- it's not the first iteration of a multi-iterations run.
335 //
336 //Reason: if the CLTuner is enabled then the first run will be really messy
337 //as each kernel will be executed several times, messing up the instruments like OpenCL timers.
338 if(_num_iterations == 1 || i != 0)
339 {
340 profiler.start();
341 }
342 test_case->do_run();
343 test_case->do_sync();
344 if(_num_iterations == 1 || i != 0)
345 {
346 profiler.stop();
347 }
348 }
349
350 test_case->do_teardown();
351
352 profiler.test_stop();
353
354 // Change status to success if no error has happend
355 if(result.status == TestResult::Status::NOT_RUN)
356 {
357 result.status = TestResult::Status::SUCCESS;
358 }
359 }
360 catch(const FileNotFound &error)
361 {
362 profiler.test_stop();
363 if(_error_on_missing_assets)
364 {
365 if(_log_level >= LogLevel::ERRORS)
366 {
367 TestError test_error(error.what(), LogLevel::ERRORS);
368 func_on_all_printers([&](Printer * p)
369 {
370 p->print_error(test_error, is_expected_failure);
371 });
372 }
373
374 result.status = TestResult::Status::FAILED;
375
376 if(_throw_errors)
377 {
378 throw;
379 }
380 }
381 else
382 {
383 if(_log_level >= LogLevel::DEBUG)
384 {
385 func_on_all_printers([&](Printer * p)
386 {
387 p->print_info(error.what());
388 });
389 }
390
391 result.status = TestResult::Status::NOT_RUN;
392 }
393 }
394 catch(const TestError &error)
395 {
396 profiler.test_stop();
397 if(_log_level >= error.level())
398 {
399 func_on_all_printers([&](Printer * p)
400 {
401 p->print_error(error, is_expected_failure);
402 });
403 }
404
405 result.status = TestResult::Status::FAILED;
406
407 if(_throw_errors)
408 {
409 throw;
410 }
411 }
412 #ifdef ARM_COMPUTE_CL
413 catch(const ::cl::Error &error)
414 {
415 profiler.test_stop();
416 if(_log_level >= LogLevel::ERRORS)
417 {
418 std::stringstream stream;
419 stream << "Error code: " << error.err();
420 TestError test_error(error.what(), LogLevel::ERRORS, stream.str());
421 func_on_all_printers([&](Printer * p)
422 {
423 p->print_error(test_error, is_expected_failure);
424 });
425 }
426
427 result.status = TestResult::Status::FAILED;
428
429 if(_throw_errors)
430 {
431 throw;
432 }
433 }
434 #endif /* ARM_COMPUTE_CL */
435 catch(const std::exception &error)
436 {
437 profiler.test_stop();
438 if(_log_level >= LogLevel::ERRORS)
439 {
440 func_on_all_printers([&](Printer * p)
441 {
442 p->print_error(error, is_expected_failure);
443 });
444 }
445
446 result.status = TestResult::Status::CRASHED;
447
448 if(_throw_errors)
449 {
450 throw;
451 }
452 }
453 catch(...)
454 {
455 profiler.test_stop();
456 if(_log_level >= LogLevel::ERRORS)
457 {
458 func_on_all_printers([&](Printer * p)
459 {
460 p->print_error(TestError("Received unknown exception"), is_expected_failure);
461 });
462 }
463
464 result.status = TestResult::Status::CRASHED;
465
466 if(_throw_errors)
467 {
468 throw;
469 }
470 }
471 }
472 catch(const std::exception &error)
473 {
474 if(_log_level >= LogLevel::ERRORS)
475 {
476 func_on_all_printers([&](Printer * p)
477 {
478 p->print_error(error, is_expected_failure);
479 });
480 }
481
482 result.status = TestResult::Status::CRASHED;
483
484 if(_throw_errors)
485 {
486 throw;
487 }
488 }
489 catch(...)
490 {
491 if(_log_level >= LogLevel::ERRORS)
492 {
493 func_on_all_printers([&](Printer * p)
494 {
495 p->print_error(TestError("Received unknown exception"), is_expected_failure);
496 });
497 }
498
499 result.status = TestResult::Status::CRASHED;
500
501 if(_throw_errors)
502 {
503 throw;
504 }
505 }
506
507 if(_log_level >= LogLevel::ERRORS)
508 {
509 func_on_all_printers([](Printer * p)
510 {
511 p->print_errors_footer();
512 });
513 }
514
515 _current_test_info = nullptr;
516 _current_test_result = nullptr;
517
518 if(result.status == TestResult::Status::FAILED)
519 {
520 if(info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
521 {
522 result.status = TestResult::Status::EXPECTED_FAILURE;
523 }
524 }
525
526 if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED)
527 {
528 if(_stop_on_error)
529 {
530 throw std::runtime_error("Abort on first error.");
531 }
532 }
533
534 result.measurements = profiler.measurements();
535
536 set_test_result(info, result);
537 log_test_end(info);
538 }
539
run()540 bool Framework::run()
541 {
542 // Clear old test results
543 _test_results.clear();
544
545 if(_log_level >= LogLevel::TESTS)
546 {
547 func_on_all_printers([](Printer * p)
548 {
549 p->print_run_header();
550 });
551 }
552
553 const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
554
555 int id = 0;
556 int id_run_test = 0;
557
558 for(auto &test_factory : _test_factories)
559 {
560 const std::string test_case_name = test_factory->name();
561 const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
562
563 if(_test_filter->is_selected(test_info))
564 {
565 #ifdef ARM_COMPUTE_CL
566 // Every 100 tests, reset the OpenCL context to release the allocated memory
567 if(opencl_is_available() && (id_run_test % 100) == 0)
568 {
569 auto ctx_properties = CLScheduler::get().context().getInfo<CL_CONTEXT_PROPERTIES>(nullptr);
570 auto queue_properties = CLScheduler::get().queue().getInfo<CL_QUEUE_PROPERTIES>(nullptr);
571
572 cl::Context new_ctx = cl::Context(CL_DEVICE_TYPE_DEFAULT, ctx_properties.data());
573 cl::CommandQueue new_queue = cl::CommandQueue(new_ctx, CLKernelLibrary::get().get_device(), queue_properties);
574
575 CLKernelLibrary::get().clear_programs_cache();
576 CLScheduler::get().set_context(new_ctx);
577 CLScheduler::get().set_queue(new_queue);
578 }
579 #endif // ARM_COMPUTE_CL
580
581 run_test(test_info, *test_factory);
582
583 ++id_run_test;
584
585 // Run test delay
586 sleep_in_seconds(_cooldown_sec);
587 }
588
589 ++id;
590 }
591
592 const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
593
594 if(_log_level >= LogLevel::TESTS)
595 {
596 func_on_all_printers([](Printer * p)
597 {
598 p->print_run_footer();
599 });
600 }
601
602 auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
603 std::map<TestResult::Status, int> results = count_test_results();
604
605 if(_log_level > LogLevel::NONE)
606 {
607 std::cout << "Executed " << _test_results.size() << " test(s) ("
608 << results[TestResult::Status::SUCCESS] << " passed, "
609 << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
610 << results[TestResult::Status::FAILED] << " failed, "
611 << results[TestResult::Status::CRASHED] << " crashed, "
612 << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
613 }
614
615 int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE] + results[TestResult::Status::DISABLED];
616
617 return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
618 }
619
set_test_result(TestInfo info,TestResult result)620 void Framework::set_test_result(TestInfo info, TestResult result)
621 {
622 _test_results.emplace(std::move(info), std::move(result));
623 }
624
print_test_results(Printer & printer) const625 void Framework::print_test_results(Printer &printer) const
626 {
627 printer.print_run_header();
628
629 for(const auto &test : _test_results)
630 {
631 printer.print_test_header(test.first);
632 printer.print_measurements(test.second.measurements);
633 printer.print_test_footer();
634 }
635
636 printer.print_run_footer();
637 }
638
get_profiler() const639 Profiler Framework::get_profiler() const
640 {
641 Profiler profiler;
642
643 const bool all_instruments = std::any_of(
644 _instruments.begin(),
645 _instruments.end(),
646 [](InstrumentsDescription type) -> bool { return type.first == InstrumentType::ALL; });
647
648 auto is_selected = [&](InstrumentsDescription instrument) -> bool
649 {
650 return std::find_if(_instruments.begin(), _instruments.end(), [&](InstrumentsDescription type) -> bool {
651 const auto group = static_cast<InstrumentType>(static_cast<uint64_t>(type.first) & 0xFF00);
652 return (group == instrument.first) && (instrument.second == type.second);
653 })
654 != _instruments.end();
655 };
656
657 for(const auto &instrument : _available_instruments)
658 {
659 if(all_instruments || is_selected(instrument.first))
660 {
661 profiler.add(instrument.second());
662 }
663 }
664
665 return profiler;
666 }
667
add_printer(Printer * printer)668 void Framework::add_printer(Printer *printer)
669 {
670 _printers.push_back(printer);
671 }
672
test_infos() const673 std::vector<TestInfo> Framework::test_infos() const
674 {
675 std::vector<TestInfo> ids;
676
677 int id = 0;
678
679 for(const auto &factory : _test_factories)
680 {
681 TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
682
683 if(_test_filter->is_selected(test_info))
684 {
685 ids.emplace_back(std::move(test_info));
686 }
687
688 ++id;
689 }
690
691 return ids;
692 }
693
log_level() const694 LogLevel Framework::log_level() const
695 {
696 return _log_level;
697 }
698
set_instruments_info(InstrumentsInfo instr_info)699 void Framework::set_instruments_info(InstrumentsInfo instr_info)
700 {
701 ARM_COMPUTE_ERROR_ON(instruments_info == nullptr);
702 *instruments_info = instr_info;
703 }
704 } // namespace framework
705 } // namespace test
706 } // namespace arm_compute
707