1 /*
2 * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11 #include "test/test_main_lib.h"
12
13 #include <fstream>
14 #include <memory>
15 #include <string>
16 #include <vector>
17
18 #include "absl/flags/flag.h"
19 #include "absl/memory/memory.h"
20 #include "absl/strings/match.h"
21 #include "absl/types/optional.h"
22 #include "api/test/metrics/chrome_perf_dashboard_metrics_exporter.h"
23 #include "api/test/metrics/global_metrics_logger_and_exporter.h"
24 #include "api/test/metrics/metrics_exporter.h"
25 #include "api/test/metrics/metrics_set_proto_file_exporter.h"
26 #include "api/test/metrics/print_result_proxy_metrics_exporter.h"
27 #include "api/test/metrics/stdout_metrics_exporter.h"
28 #include "rtc_base/checks.h"
29 #include "rtc_base/event_tracer.h"
30 #include "rtc_base/logging.h"
31 #include "rtc_base/ssl_adapter.h"
32 #include "rtc_base/ssl_stream_adapter.h"
33 #include "rtc_base/thread.h"
34 #include "system_wrappers/include/field_trial.h"
35 #include "system_wrappers/include/metrics.h"
36 #include "test/field_trial.h"
37 #include "test/gtest.h"
38 #include "test/test_flags.h"
39 #include "test/testsupport/perf_test.h"
40 #include "test/testsupport/resources_dir_flag.h"
41
42 #if defined(WEBRTC_WIN)
43 #include "rtc_base/win32_socket_init.h"
44 #endif
45
46 #if defined(WEBRTC_IOS)
47 #include "test/ios/test_support.h"
48
49 ABSL_FLAG(std::string,
50 NSTreatUnknownArgumentsAsOpen,
51 "",
52 "Intentionally ignored flag intended for iOS test runner.");
53 ABSL_FLAG(std::string,
54 ApplePersistenceIgnoreState,
55 "",
56 "Intentionally ignored flag intended for iOS test runner.");
57 ABSL_FLAG(bool,
58 enable_run_ios_unittests_with_xctest,
59 false,
60 "Intentionally ignored flag intended for iOS test runner.");
61 ABSL_FLAG(bool,
62 write_compiled_tests_json_to_writable_path,
63 false,
64 "Intentionally ignored flag intended for iOS test runner.");
65
66 // This is the cousin of isolated_script_test_perf_output, but we can't dictate
67 // where to write on iOS so the semantics of this flag are a bit different.
68 ABSL_FLAG(
69 bool,
70 write_perf_output_on_ios,
71 false,
72 "Store the perf results in Documents/perftest_result.pb in the format "
73 "described by histogram.proto in "
74 "https://chromium.googlesource.com/catapult/.");
75
76 #elif defined(WEBRTC_FUCHSIA)
77 ABSL_FLAG(std::string, use_vulkan, "", "Intentionally ignored flag.");
78 #else
79 // TODO(bugs.webrtc.org/8115): Remove workaround when fixed.
80 ABSL_FLAG(bool, no_sandbox, false, "Intentionally ignored flag.");
81 ABSL_FLAG(bool, test_launcher_bot_mode, false, "Intentionally ignored flag.");
82 #endif
83
84 ABSL_FLAG(std::string,
85 isolated_script_test_output,
86 "",
87 "Path to output an empty JSON file which Chromium infra requires.");
88
89 ABSL_FLAG(bool, logs, true, "print logs to stderr");
90 ABSL_FLAG(bool, verbose, false, "verbose logs to stderr");
91
92 ABSL_FLAG(std::string,
93 trace_event,
94 "",
95 "Path to collect trace events (json file) for chrome://tracing. "
96 "If not set, events aren't captured.");
97
98 ABSL_FLAG(std::string,
99 test_launcher_shard_index,
100 "",
101 "Index of the test shard to run, from 0 to "
102 "the value specified with --test_launcher_total_shards.");
103
104 ABSL_FLAG(std::string,
105 test_launcher_total_shards,
106 "",
107 "Total number of shards.");
108
109 namespace webrtc {
110
111 namespace {
112
113 constexpr char kPlotAllMetrics[] = "all";
114
115 class TestMainImpl : public TestMain {
116 public:
Init(int * argc,char * argv[])117 int Init(int* argc, char* argv[]) override { return Init(); }
118
Init()119 int Init() override {
120 // Make sure we always pull in the --resources_dir flag, even if the test
121 // binary doesn't link with fileutils (downstream expects all test mains to
122 // have this flag).
123 (void)absl::GetFlag(FLAGS_resources_dir);
124
125 // Default to LS_INFO, even for release builds to provide better test
126 // logging.
127 if (rtc::LogMessage::GetLogToDebug() > rtc::LS_INFO)
128 rtc::LogMessage::LogToDebug(rtc::LS_INFO);
129
130 if (absl::GetFlag(FLAGS_verbose))
131 rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
132
133 rtc::LogMessage::SetLogToStderr(absl::GetFlag(FLAGS_logs) ||
134 absl::GetFlag(FLAGS_verbose));
135
136 // The sharding arguments take precedence over the sharding environment
137 // variables.
138 if (!absl::GetFlag(FLAGS_test_launcher_shard_index).empty() &&
139 !absl::GetFlag(FLAGS_test_launcher_total_shards).empty()) {
140 std::string shard_index =
141 "GTEST_SHARD_INDEX=" + absl::GetFlag(FLAGS_test_launcher_shard_index);
142 std::string total_shards =
143 "GTEST_TOTAL_SHARDS=" +
144 absl::GetFlag(FLAGS_test_launcher_total_shards);
145 putenv(shard_index.data());
146 putenv(total_shards.data());
147 }
148
149 // InitFieldTrialsFromString stores the char*, so the char array must
150 // outlive the application.
151 field_trials_ = absl::GetFlag(FLAGS_force_fieldtrials);
152 webrtc::field_trial::InitFieldTrialsFromString(field_trials_.c_str());
153 webrtc::metrics::Enable();
154
155 #if defined(WEBRTC_WIN)
156 winsock_init_ = std::make_unique<rtc::WinsockInitializer>();
157 #endif
158
159 // Initialize SSL which are used by several tests.
160 rtc::InitializeSSL();
161 rtc::SSLStreamAdapter::EnableTimeCallbackForTesting();
162
163 return 0;
164 }
165
Run(int argc,char * argv[])166 int Run(int argc, char* argv[]) override {
167 std::string trace_event_path = absl::GetFlag(FLAGS_trace_event);
168 const bool capture_events = !trace_event_path.empty();
169 if (capture_events) {
170 rtc::tracing::SetupInternalTracer();
171 rtc::tracing::StartInternalCapture(trace_event_path);
172 }
173
174 absl::optional<std::vector<std::string>> metrics_to_plot =
175 absl::GetFlag(FLAGS_plot);
176
177 if (metrics_to_plot->empty()) {
178 metrics_to_plot = absl::nullopt;
179 } else {
180 if (metrics_to_plot->size() == 1 &&
181 (*metrics_to_plot)[0] == kPlotAllMetrics) {
182 metrics_to_plot->clear();
183 }
184 }
185
186 #if defined(WEBRTC_IOS)
187 rtc::test::InitTestSuite(
188 RUN_ALL_TESTS, argc, argv,
189 absl::GetFlag(FLAGS_write_perf_output_on_ios),
190 absl::GetFlag(FLAGS_export_perf_results_new_api),
191 absl::GetFlag(FLAGS_webrtc_test_metrics_output_path), metrics_to_plot);
192 rtc::test::RunTestsFromIOSApp();
193 int exit_code = 0;
194 #else
195 int exit_code = RUN_ALL_TESTS();
196
197 std::vector<std::unique_ptr<test::MetricsExporter>> exporters;
198 if (absl::GetFlag(FLAGS_export_perf_results_new_api)) {
199 exporters.push_back(std::make_unique<test::StdoutMetricsExporter>());
200 if (!absl::GetFlag(FLAGS_webrtc_test_metrics_output_path).empty()) {
201 exporters.push_back(
202 std::make_unique<webrtc::test::MetricsSetProtoFileExporter>(
203 webrtc::test::MetricsSetProtoFileExporter::Options(
204 absl::GetFlag(FLAGS_webrtc_test_metrics_output_path))));
205 }
206 if (!absl::GetFlag(FLAGS_isolated_script_test_perf_output).empty()) {
207 exporters.push_back(
208 std::make_unique<test::ChromePerfDashboardMetricsExporter>(
209 absl::GetFlag(FLAGS_isolated_script_test_perf_output)));
210 }
211 } else {
212 exporters.push_back(
213 std::make_unique<test::PrintResultProxyMetricsExporter>());
214 }
215 test::ExportPerfMetric(*test::GetGlobalMetricsLogger(),
216 std::move(exporters));
217 if (!absl::GetFlag(FLAGS_export_perf_results_new_api)) {
218 std::string perf_output_file =
219 absl::GetFlag(FLAGS_isolated_script_test_perf_output);
220 if (!perf_output_file.empty()) {
221 if (!webrtc::test::WritePerfResults(perf_output_file)) {
222 return 1;
223 }
224 }
225 if (metrics_to_plot) {
226 webrtc::test::PrintPlottableResults(*metrics_to_plot);
227 }
228 }
229
230 std::string result_filename =
231 absl::GetFlag(FLAGS_isolated_script_test_output);
232 if (!result_filename.empty()) {
233 std::ofstream result_file(result_filename);
234 result_file << "{\"version\": 3}";
235 }
236 #endif
237
238 if (capture_events) {
239 rtc::tracing::StopInternalCapture();
240 }
241
242 #if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) || \
243 defined(MEMORY_SANITIZER) || defined(THREAD_SANITIZER) || \
244 defined(UNDEFINED_SANITIZER)
245 // We want the test flagged as failed only for sanitizer defects,
246 // in which case the sanitizer will override exit code with 66.
247 exit_code = 0;
248 #endif
249
250 return exit_code;
251 }
252
253 ~TestMainImpl() override = default;
254
255 private:
256 #if defined(WEBRTC_WIN)
257 std::unique_ptr<rtc::WinsockInitializer> winsock_init_;
258 #endif
259 };
260
261 } // namespace
262
Create()263 std::unique_ptr<TestMain> TestMain::Create() {
264 return std::make_unique<TestMainImpl>();
265 }
266
267 } // namespace webrtc
268