1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "dl/sp/src/test/test_util.h"
11
12 #include "dl/sp/src/test/compare.h"
13
14 /*
15 * Test results from running either forward or inverse FFT tests
16 */
17 struct TestResult {
18 /* Number of tests that failed */
19 int failed_count_;
20
21 /* Number of tests run */
22 int test_count_;
23
24 /* Number of tests that were expected to fail */
25 int expected_failure_count_;
26
27 /* Number of tests that were expected to fail but didn't */
28 int unexpected_pass_count_;
29
30 /* Number of tests that unexpectedly failed */
31 int unexpected_failure_count_;
32
33 /* The minimum SNR found for all of the tests */
34 float min_snr_;
35 };
36
37 /*
38 * Run one FFT test
39 */
TestOneFFT(int fft_log_size,int signal_type,float signal_value,const struct TestInfo * info,const char * message)40 void TestOneFFT(int fft_log_size,
41 int signal_type,
42 float signal_value,
43 const struct TestInfo* info,
44 const char* message) {
45 struct SnrResult snr;
46
47 if (info->do_forward_tests_) {
48 RunOneForwardTest(fft_log_size, signal_type, signal_value, &snr);
49 printf("Forward %s\n", message);
50 printf("SNR: real part %10.3f dB\n", snr.real_snr_);
51 printf(" imag part %10.3f dB\n", snr.imag_snr_);
52 printf(" complex part %10.3f dB\n", snr.complex_snr_);
53 }
54
55 if (info->do_inverse_tests_) {
56 RunOneInverseTest(fft_log_size, signal_type, signal_value, &snr);
57 printf("Inverse %s\n", message);
58 if (info->real_only_) {
59 printf("SNR: real %10.3f dB\n", snr.real_snr_);
60 } else {
61 printf("SNR: real part %10.3f dB\n", snr.real_snr_);
62 printf(" imag part %10.3f dB\n", snr.imag_snr_);
63 printf(" complex part %10.3f dB\n", snr.complex_snr_);
64 }
65 }
66 }
67
68 /*
69 * Run a set of tests, printing out the result of each test.
70 */
RunTests(struct TestResult * result,float (* test_function)(int,int,float,struct SnrResult *),const char * id,int is_inverse_test,const struct TestInfo * info,float snr_threshold)71 void RunTests(struct TestResult* result,
72 float (*test_function)(int, int, float, struct SnrResult*),
73 const char* id,
74 int is_inverse_test,
75 const struct TestInfo* info,
76 float snr_threshold) {
77 int fft_order;
78 int signal_type;
79 float snr;
80 int tests = 0;
81 int failures = 0;
82 int expected_failures = 0;
83 int unexpected_failures = 0;
84 int unexpected_passes = 0;
85 float min_snr = 1e10;
86 struct SnrResult snrResults;
87
88 for (fft_order = info->min_fft_order_; fft_order <= info->max_fft_order_;
89 ++fft_order) {
90 for (signal_type = 0; signal_type < MaxSignalType(info->real_only_);
91 ++signal_type) {
92 int known_failure = 0;
93 int test_failed = 0;
94 ++tests;
95 snr = test_function(fft_order, signal_type, 1024.0, &snrResults);
96 if (snr < min_snr)
97 min_snr = snr;
98 known_failure = IsKnownFailure(fft_order, is_inverse_test,
99 signal_type, info->known_failures_);
100 if (snr < snr_threshold) {
101 ++failures;
102 test_failed = 1;
103 if (known_failure) {
104 ++expected_failures;
105 printf(" *FAILED: %s ", id);
106 } else {
107 ++unexpected_failures;
108 printf("**FAILED: %s ", id);
109 }
110 } else {
111 test_failed = 0;
112 printf(" PASSED: %s ", id);
113 }
114 printf("order %2d signal %d: SNR = %9.3f",
115 fft_order, signal_type, snr);
116 if (known_failure) {
117 if (test_failed) {
118 printf(" (expected failure)");
119 } else {
120 ++unexpected_passes;
121 printf(" (**Expected to fail, but passed)");
122 }
123 }
124 printf("\n");
125 }
126 }
127
128 printf("%sSummary: %d %s tests failed out of %d tests. "
129 "(Success rate %.2f%%.)\n",
130 failures ? "**" : "",
131 failures,
132 id,
133 tests,
134 (100.0 * (tests - failures)) / tests);
135 if (expected_failures || unexpected_passes || unexpected_failures) {
136 printf(" (%d expected failures)\n", expected_failures);
137 printf(" (%d unexpected failures)\n", unexpected_failures);
138 printf(" (%d unexpected passes)\n", unexpected_passes);
139 }
140
141 printf(" (Minimum SNR = %.3f dB)\n", min_snr);
142
143 result->failed_count_ = failures;
144 result->test_count_ = tests;
145 result->expected_failure_count_ = expected_failures;
146 result->unexpected_pass_count_ = unexpected_passes;
147 result->unexpected_failure_count_ = unexpected_failures;
148 result->min_snr_ = min_snr;
149 }
150
151 /*
152 * For all FFT orders and signal types, run the forward FFT.
153 * runOneForwardTest must be defined to compute the forward FFT and
154 * return the SNR beween the actual and expected FFT.
155 *
156 * Also finds the minium SNR from all of the tests and returns the
157 * minimum SNR value.
158 */
RunForwardTests(struct TestResult * result,const struct TestInfo * info,float snr_threshold)159 void RunForwardTests(struct TestResult* result, const struct TestInfo* info,
160 float snr_threshold) {
161 RunTests(result, RunOneForwardTest, "FwdFFT", 0, info, snr_threshold);
162 }
163
initializeTestResult(struct TestResult * result)164 void initializeTestResult(struct TestResult *result) {
165 result->failed_count_ = 0;
166 result->test_count_ = 0;
167 result->expected_failure_count_ = 0;
168 result->min_snr_ = 1000;
169 }
170
171 /*
172 * For all FFT orders and signal types, run the inverse FFT.
173 * runOneInverseTest must be defined to compute the forward FFT and
174 * return the SNR beween the actual and expected FFT.
175 *
176 * Also finds the minium SNR from all of the tests and returns the
177 * minimum SNR value.
178 */
RunInverseTests(struct TestResult * result,const struct TestInfo * info,float snr_threshold)179 void RunInverseTests(struct TestResult* result, const struct TestInfo* info,
180 float snr_threshold) {
181 RunTests(result, RunOneInverseTest, "InvFFT", 1, info, snr_threshold);
182 }
183
184 /*
185 * Run all forward and inverse FFT tests, printing a summary of the
186 * results.
187 */
RunAllTests(const struct TestInfo * info)188 int RunAllTests(const struct TestInfo* info) {
189 int failed;
190 int total;
191 float min_forward_snr;
192 float min_inverse_snr;
193 struct TestResult forward_results;
194 struct TestResult inverse_results;
195
196 initializeTestResult(&forward_results);
197 initializeTestResult(&inverse_results);
198
199 if (info->do_forward_tests_)
200 RunForwardTests(&forward_results, info, info->forward_threshold_);
201 if (info->do_inverse_tests_)
202 RunInverseTests(&inverse_results, info, info->inverse_threshold_);
203
204 failed = forward_results.failed_count_ + inverse_results.failed_count_;
205 total = forward_results.test_count_ + inverse_results.test_count_;
206 min_forward_snr = forward_results.min_snr_;
207 min_inverse_snr = inverse_results.min_snr_;
208
209 if (total) {
210 printf("%sTotal: %d tests failed out of %d tests. "
211 "(Success rate = %.2f%%.)\n",
212 failed ? "**" : "",
213 failed,
214 total,
215 (100.0 * (total - failed)) / total);
216 if (forward_results.expected_failure_count_
217 + inverse_results.expected_failure_count_) {
218 printf(" (%d expected failures)\n",
219 forward_results.expected_failure_count_
220 + inverse_results.expected_failure_count_);
221 printf(" (%d unexpected failures)\n",
222 forward_results.unexpected_failure_count_
223 + inverse_results.unexpected_failure_count_);
224 printf(" (%d unexpected passes)\n",
225 forward_results.unexpected_pass_count_
226 + inverse_results.unexpected_pass_count_);
227 }
228 printf(" Min forward SNR = %.3f dB, min inverse SNR = %.3f dB\n",
229 min_forward_snr,
230 min_inverse_snr);
231 } else {
232 printf("No tests run\n");
233 }
234
235 return failed;
236 }
237