1 // Copyright 2013 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "base/test/launcher/test_results_tracker.h"
6
7 #include "base/base64.h"
8 #include "base/command_line.h"
9 #include "base/file_util.h"
10 #include "base/files/file_path.h"
11 #include "base/format_macros.h"
12 #include "base/json/json_file_value_serializer.h"
13 #include "base/json/string_escape.h"
14 #include "base/logging.h"
15 #include "base/strings/string_util.h"
16 #include "base/strings/stringprintf.h"
17 #include "base/test/launcher/test_launcher.h"
18 #include "base/values.h"
19
20 namespace base {
21
22 // See https://groups.google.com/a/chromium.org/d/msg/chromium-dev/nkdTP7sstSc/uT3FaE_sgkAJ .
23 using ::operator<<;
24
25 namespace {
26
27 // The default output file for XML output.
28 const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
29 "test_detail.xml");
30
31 // Utility function to print a list of test names. Uses iterator to be
32 // compatible with different containers, like vector and set.
33 template<typename InputIterator>
PrintTests(InputIterator first,InputIterator last,const std::string & description)34 void PrintTests(InputIterator first,
35 InputIterator last,
36 const std::string& description) {
37 size_t count = std::distance(first, last);
38 if (count == 0)
39 return;
40
41 fprintf(stdout,
42 "%" PRIuS " test%s %s:\n",
43 count,
44 count != 1 ? "s" : "",
45 description.c_str());
46 for (InputIterator i = first; i != last; ++i)
47 fprintf(stdout, " %s\n", (*i).c_str());
48 fflush(stdout);
49 }
50
TestNameWithoutDisabledPrefix(const std::string & test_name)51 std::string TestNameWithoutDisabledPrefix(const std::string& test_name) {
52 std::string test_name_no_disabled(test_name);
53 ReplaceSubstringsAfterOffset(&test_name_no_disabled, 0, "DISABLED_", "");
54 return test_name_no_disabled;
55 }
56
57 } // namespace
58
TestResultsTracker()59 TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(NULL) {
60 }
61
~TestResultsTracker()62 TestResultsTracker::~TestResultsTracker() {
63 DCHECK(thread_checker_.CalledOnValidThread());
64
65 if (!out_)
66 return;
67 fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
68 fprintf(out_, "<testsuites name=\"AllTests\" tests=\"\" failures=\"\""
69 " disabled=\"\" errors=\"\" time=\"\">\n");
70
71 // Maps test case names to test results.
72 typedef std::map<std::string, std::vector<TestResult> > TestCaseMap;
73 TestCaseMap test_case_map;
74
75 for (PerIterationData::ResultsMap::iterator i =
76 per_iteration_data_[iteration_].results.begin();
77 i != per_iteration_data_[iteration_].results.end();
78 ++i) {
79 // Use the last test result as the final one.
80 TestResult result = i->second.test_results.back();
81 test_case_map[result.GetTestCaseName()].push_back(result);
82 }
83 for (TestCaseMap::iterator i = test_case_map.begin();
84 i != test_case_map.end();
85 ++i) {
86 fprintf(out_, " <testsuite name=\"%s\" tests=\"%" PRIuS "\" failures=\"\""
87 " disabled=\"\" errors=\"\" time=\"\">\n",
88 i->first.c_str(), i->second.size());
89 for (size_t j = 0; j < i->second.size(); ++j) {
90 const TestResult& result = i->second[j];
91 fprintf(out_, " <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
92 " classname=\"%s\">\n",
93 result.GetTestName().c_str(),
94 result.elapsed_time.InSecondsF(),
95 result.GetTestCaseName().c_str());
96 if (result.status != TestResult::TEST_SUCCESS)
97 fprintf(out_, " <failure message=\"\" type=\"\"></failure>\n");
98 fprintf(out_, " </testcase>\n");
99 }
100 fprintf(out_, " </testsuite>\n");
101 }
102 fprintf(out_, "</testsuites>\n");
103 fclose(out_);
104 }
105
Init(const CommandLine & command_line)106 bool TestResultsTracker::Init(const CommandLine& command_line) {
107 DCHECK(thread_checker_.CalledOnValidThread());
108
109 // Prevent initializing twice.
110 if (out_) {
111 NOTREACHED();
112 return false;
113 }
114
115 if (!command_line.HasSwitch(kGTestOutputFlag))
116 return true;
117
118 std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
119 size_t colon_pos = flag.find(':');
120 FilePath path;
121 if (colon_pos != std::string::npos) {
122 FilePath flag_path =
123 command_line.GetSwitchValuePath(kGTestOutputFlag);
124 FilePath::StringType path_string = flag_path.value();
125 path = FilePath(path_string.substr(colon_pos + 1));
126 // If the given path ends with '/', consider it is a directory.
127 // Note: This does NOT check that a directory (or file) actually exists
128 // (the behavior is same as what gtest does).
129 if (path.EndsWithSeparator()) {
130 FilePath executable = command_line.GetProgram().BaseName();
131 path = path.Append(executable.ReplaceExtension(
132 FilePath::StringType(FILE_PATH_LITERAL("xml"))));
133 }
134 }
135 if (path.value().empty())
136 path = FilePath(kDefaultOutputFile);
137 FilePath dir_name = path.DirName();
138 if (!DirectoryExists(dir_name)) {
139 LOG(WARNING) << "The output directory does not exist. "
140 << "Creating the directory: " << dir_name.value();
141 // Create the directory if necessary (because the gtest does the same).
142 if (!base::CreateDirectory(dir_name)) {
143 LOG(ERROR) << "Failed to created directory " << dir_name.value();
144 return false;
145 }
146 }
147 out_ = OpenFile(path, "w");
148 if (!out_) {
149 LOG(ERROR) << "Cannot open output file: "
150 << path.value() << ".";
151 return false;
152 }
153
154 return true;
155 }
156
OnTestIterationStarting()157 void TestResultsTracker::OnTestIterationStarting() {
158 DCHECK(thread_checker_.CalledOnValidThread());
159
160 // Start with a fresh state for new iteration.
161 iteration_++;
162 per_iteration_data_.push_back(PerIterationData());
163 }
164
AddTest(const std::string & test_name)165 void TestResultsTracker::AddTest(const std::string& test_name) {
166 // Record disabled test names without DISABLED_ prefix so that they are easy
167 // to compare with regular test names, e.g. before or after disabling.
168 all_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
169 }
170
AddDisabledTest(const std::string & test_name)171 void TestResultsTracker::AddDisabledTest(const std::string& test_name) {
172 // Record disabled test names without DISABLED_ prefix so that they are easy
173 // to compare with regular test names, e.g. before or after disabling.
174 disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
175 }
176
AddTestResult(const TestResult & result)177 void TestResultsTracker::AddTestResult(const TestResult& result) {
178 DCHECK(thread_checker_.CalledOnValidThread());
179
180 per_iteration_data_[iteration_].results[
181 result.full_name].test_results.push_back(result);
182 }
183
PrintSummaryOfCurrentIteration() const184 void TestResultsTracker::PrintSummaryOfCurrentIteration() const {
185 TestStatusMap tests_by_status(GetTestStatusMapForCurrentIteration());
186
187 PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
188 tests_by_status[TestResult::TEST_FAILURE].end(),
189 "failed");
190 PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
191 tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
192 "failed on exit");
193 PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
194 tests_by_status[TestResult::TEST_TIMEOUT].end(),
195 "timed out");
196 PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
197 tests_by_status[TestResult::TEST_CRASH].end(),
198 "crashed");
199 PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
200 tests_by_status[TestResult::TEST_SKIPPED].end(),
201 "skipped");
202 PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
203 tests_by_status[TestResult::TEST_UNKNOWN].end(),
204 "had unknown result");
205 }
206
PrintSummaryOfAllIterations() const207 void TestResultsTracker::PrintSummaryOfAllIterations() const {
208 DCHECK(thread_checker_.CalledOnValidThread());
209
210 TestStatusMap tests_by_status(GetTestStatusMapForAllIterations());
211
212 fprintf(stdout, "Summary of all test iterations:\n");
213 fflush(stdout);
214
215 PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
216 tests_by_status[TestResult::TEST_FAILURE].end(),
217 "failed");
218 PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
219 tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
220 "failed on exit");
221 PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
222 tests_by_status[TestResult::TEST_TIMEOUT].end(),
223 "timed out");
224 PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
225 tests_by_status[TestResult::TEST_CRASH].end(),
226 "crashed");
227 PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
228 tests_by_status[TestResult::TEST_SKIPPED].end(),
229 "skipped");
230 PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
231 tests_by_status[TestResult::TEST_UNKNOWN].end(),
232 "had unknown result");
233
234 fprintf(stdout, "End of the summary.\n");
235 fflush(stdout);
236 }
237
AddGlobalTag(const std::string & tag)238 void TestResultsTracker::AddGlobalTag(const std::string& tag) {
239 global_tags_.insert(tag);
240 }
241
SaveSummaryAsJSON(const FilePath & path) const242 bool TestResultsTracker::SaveSummaryAsJSON(const FilePath& path) const {
243 scoped_ptr<DictionaryValue> summary_root(new DictionaryValue);
244
245 ListValue* global_tags = new ListValue;
246 summary_root->Set("global_tags", global_tags);
247
248 for (std::set<std::string>::const_iterator i = global_tags_.begin();
249 i != global_tags_.end();
250 ++i) {
251 global_tags->AppendString(*i);
252 }
253
254 ListValue* all_tests = new ListValue;
255 summary_root->Set("all_tests", all_tests);
256
257 for (std::set<std::string>::const_iterator i = all_tests_.begin();
258 i != all_tests_.end();
259 ++i) {
260 all_tests->AppendString(*i);
261 }
262
263 ListValue* disabled_tests = new ListValue;
264 summary_root->Set("disabled_tests", disabled_tests);
265
266 for (std::set<std::string>::const_iterator i = disabled_tests_.begin();
267 i != disabled_tests_.end();
268 ++i) {
269 disabled_tests->AppendString(*i);
270 }
271
272 ListValue* per_iteration_data = new ListValue;
273 summary_root->Set("per_iteration_data", per_iteration_data);
274
275 for (int i = 0; i <= iteration_; i++) {
276 DictionaryValue* current_iteration_data = new DictionaryValue;
277 per_iteration_data->Append(current_iteration_data);
278
279 for (PerIterationData::ResultsMap::const_iterator j =
280 per_iteration_data_[i].results.begin();
281 j != per_iteration_data_[i].results.end();
282 ++j) {
283 ListValue* test_results = new ListValue;
284 current_iteration_data->SetWithoutPathExpansion(j->first, test_results);
285
286 for (size_t k = 0; k < j->second.test_results.size(); k++) {
287 const TestResult& test_result = j->second.test_results[k];
288
289 DictionaryValue* test_result_value = new DictionaryValue;
290 test_results->Append(test_result_value);
291
292 test_result_value->SetString("status", test_result.StatusAsString());
293 test_result_value->SetInteger(
294 "elapsed_time_ms", test_result.elapsed_time.InMilliseconds());
295
296 // There are no guarantees about character encoding of the output
297 // snippet. Escape it and record whether it was losless.
298 // It's useful to have the output snippet as string in the summary
299 // for easy viewing.
300 std::string escaped_output_snippet;
301 bool losless_snippet = EscapeJSONString(
302 test_result.output_snippet, false, &escaped_output_snippet);
303 test_result_value->SetString("output_snippet",
304 escaped_output_snippet);
305 test_result_value->SetBoolean("losless_snippet", losless_snippet);
306
307 // Also include the raw version (base64-encoded so that it can be safely
308 // JSON-serialized - there are no guarantees about character encoding
309 // of the snippet). This can be very useful piece of information when
310 // debugging a test failure related to character encoding.
311 std::string base64_output_snippet;
312 Base64Encode(test_result.output_snippet, &base64_output_snippet);
313 test_result_value->SetString("output_snippet_base64",
314 base64_output_snippet);
315 }
316 }
317 }
318
319 JSONFileValueSerializer serializer(path);
320 return serializer.Serialize(*summary_root);
321 }
322
323 TestResultsTracker::TestStatusMap
GetTestStatusMapForCurrentIteration() const324 TestResultsTracker::GetTestStatusMapForCurrentIteration() const {
325 TestStatusMap tests_by_status;
326 GetTestStatusForIteration(iteration_, &tests_by_status);
327 return tests_by_status;
328 }
329
330 TestResultsTracker::TestStatusMap
GetTestStatusMapForAllIterations() const331 TestResultsTracker::GetTestStatusMapForAllIterations() const {
332 TestStatusMap tests_by_status;
333 for (int i = 0; i <= iteration_; i++)
334 GetTestStatusForIteration(i, &tests_by_status);
335 return tests_by_status;
336 }
337
GetTestStatusForIteration(int iteration,TestStatusMap * map) const338 void TestResultsTracker::GetTestStatusForIteration(
339 int iteration, TestStatusMap* map) const {
340 for (PerIterationData::ResultsMap::const_iterator j =
341 per_iteration_data_[iteration].results.begin();
342 j != per_iteration_data_[iteration].results.end();
343 ++j) {
344 // Use the last test result as the final one.
345 const TestResult& result = j->second.test_results.back();
346 (*map)[result.status].insert(result.full_name);
347 }
348 }
349
AggregateTestResult()350 TestResultsTracker::AggregateTestResult::AggregateTestResult() {
351 }
352
~AggregateTestResult()353 TestResultsTracker::AggregateTestResult::~AggregateTestResult() {
354 }
355
PerIterationData()356 TestResultsTracker::PerIterationData::PerIterationData() {
357 }
358
~PerIterationData()359 TestResultsTracker::PerIterationData::~PerIterationData() {
360 }
361
362 } // namespace base
363