1 // Copyright 2019 The Chromium Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #include "testing/perf/luci_test_result.h"
6
7 #include "base/files/file_path.h"
8 #include "base/files/file_util.h"
9 #include "base/files/scoped_temp_dir.h"
10 #include "base/json/json_reader.h"
11 #include "base/strings/stringprintf.h"
12 #include "base/time/time.h"
13 #include "testing/gtest/include/gtest/gtest.h"
14 #include "third_party/abseil-cpp/absl/types/optional.h"
15
16 namespace perf_test {
17
18 class LuciTestResultTest : public testing::Test {
19 public:
20 LuciTestResultTest() = default;
21
22 LuciTestResultTest(const LuciTestResultTest&) = delete;
23 LuciTestResultTest& operator=(const LuciTestResultTest&) = delete;
24
25 ~LuciTestResultTest() override = default;
26
27 // testing::Test:
SetUp()28 void SetUp() override {
29 testing::Test::SetUp();
30 ASSERT_TRUE(temp_dir_.CreateUniqueTempDir());
31 }
32
GetResultFilePath() const33 base::FilePath GetResultFilePath() const {
34 return temp_dir_.GetPath().AppendASCII("luci_test_results.json");
35 }
36
37 // Validates that |result| is written to file that contains an equivalent JSON
38 // as |expected_json|.
ValidateResult(const LuciTestResult & result,const std::string & expected_json)39 void ValidateResult(const LuciTestResult& result,
40 const std::string& expected_json) {
41 const base::FilePath result_file = GetResultFilePath();
42 result.WriteToFile(result_file);
43
44 std::string json;
45 ASSERT_TRUE(ReadFileToString(GetResultFilePath(), &json));
46 absl::optional<base::Value> value = base::JSONReader::Read(json);
47 ASSERT_TRUE(value.has_value());
48
49 absl::optional<base::Value> expected_value =
50 base::JSONReader::Read(expected_json);
51 ASSERT_TRUE(expected_value.has_value());
52
53 EXPECT_EQ(expected_value, value) << "Expected:\n====\n"
54 << expected_json << "\nActual:\n====\n"
55 << json;
56 }
57
58 private:
59 base::ScopedTempDir temp_dir_;
60 };
61
TEST_F(LuciTestResultTest,Basic)62 TEST_F(LuciTestResultTest, Basic) {
63 LuciTestResult result;
64 result.set_test_path("FakeTestSuite.FakeTest");
65 result.set_status(LuciTestResult::Status::kPass);
66 result.set_is_expected(true);
67
68 result.AddVariant("variantKey", "variantValue");
69 result.AddVariant("param/instantiation", "FooType");
70 result.AddVariant("param/index", "0");
71
72 // 2019/9/11 12:30 UTC
73 base::Time start_time;
74 ASSERT_TRUE(
75 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
76 result.set_start_time(start_time);
77
78 result.set_duration(base::Milliseconds(1500));
79
80 result.AddOutputArtifactContents("plain", "plain data", "text/plain");
81 result.AddOutputArtifactContents("new_line", "first\nsecond", "text/plain");
82 result.AddOutputArtifactFile(
83 "file.json", base::FilePath(FILE_PATH_LITERAL("/tmp/file.json")),
84 "application/json");
85 result.AddTag("tbmv2", "umaMetric");
86
87 const std::string expected_json =
88 R"({
89 "testResult":{
90 "outputArtifacts":{
91 "file.json":{
92 "contentType":"application/json",
93 "filePath":"/tmp/file.json"
94 },
95 "new_line":{
96 "contentType":"text/plain",
97 "contents":"first\nsecond"
98 },
99 "plain":{
100 "contentType":"text/plain",
101 "contents":"plain data"
102 }
103 },
104 "expected":true,
105 "runDuration":"1.50s",
106 "startTime":"2019-09-11T12:30:00.000Z",
107 "status":"PASS",
108 "tags":[
109 {"key":"tbmv2","value":"umaMetric"}
110 ],
111 "variant":{
112 "variantKey": "variantValue",
113 "param/instantiation": "FooType",
114 "param/index": "0"
115 },
116 "testPath":"FakeTestSuite.FakeTest"
117 }
118 })";
119 ValidateResult(result, expected_json);
120 }
121
TEST_F(LuciTestResultTest,Status)122 TEST_F(LuciTestResultTest, Status) {
123 using Status = LuciTestResult::Status;
124
125 LuciTestResult result;
126 result.set_test_path("FakeTestSuite.Status");
127
128 const std::string json_template =
129 R"({
130 "testResult":{
131 "expected":false,
132 "status":"%s",
133 "testPath":"FakeTestSuite.Status"
134 }
135 })";
136
137 const struct {
138 Status status;
139 const char* status_text;
140 } kTestCases[] = {
141 {Status::kUnspecified, "UNSPECIFIED"},
142 {Status::kPass, "PASS"},
143 {Status::kFail, "FAIL"},
144 {Status::kCrash, "CRASH"},
145 {Status::kAbort, "ABORT"},
146 {Status::kSkip, "SKIP"},
147 };
148
149 for (const auto& test_case : kTestCases) {
150 result.set_status(test_case.status);
151 const std::string expected_json =
152 base::StringPrintf(json_template.c_str(), test_case.status_text);
153 ValidateResult(result, expected_json);
154 }
155 }
156
157 ///////////////////////////////////////////////////////////////////////////////
158
159 class LuciTestResultParameterizedTest
160 : public LuciTestResultTest,
161 public testing::WithParamInterface<int> {
162 public:
163 LuciTestResultParameterizedTest() = default;
164 ~LuciTestResultParameterizedTest() override = default;
165 };
166
TEST_P(LuciTestResultParameterizedTest,Variant)167 TEST_P(LuciTestResultParameterizedTest, Variant) {
168 LuciTestResult result = LuciTestResult::CreateForGTest();
169
170 // 2019/9/11 12:30 UTC
171 base::Time start_time;
172 ASSERT_TRUE(
173 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
174 result.set_start_time(start_time);
175
176 result.set_duration(base::Milliseconds(1500));
177
178 const std::string json_template =
179 R"({
180 "testResult":{
181 "expected":true,
182 "runDuration":"1.50s",
183 "startTime":"2019-09-11T12:30:00.000Z",
184 "status":"PASS",
185 "testPath":
186 "ZeroToFiveSequence/LuciTestResultParameterizedTest.Variant",
187 "variant":{"param/index":"%d"}
188 }
189 })";
190 const std::string expected_json =
191 base::StringPrintf(json_template.c_str(), GetParam());
192 ValidateResult(result, expected_json);
193 }
194 INSTANTIATE_TEST_SUITE_P(ZeroToFiveSequence,
195 LuciTestResultParameterizedTest,
196 testing::Range(0, 5));
197
198 ///////////////////////////////////////////////////////////////////////////////
199
200 template <typename T>
201 class LuciTestResultTypedTest : public LuciTestResultTest {
202 public:
203 LuciTestResultTypedTest() = default;
204 ~LuciTestResultTypedTest() override = default;
205 };
206
207 TYPED_TEST_SUITE_P(LuciTestResultTypedTest);
208
TYPED_TEST_P(LuciTestResultTypedTest,Variant)209 TYPED_TEST_P(LuciTestResultTypedTest, Variant) {
210 LuciTestResult result = LuciTestResult::CreateForGTest();
211
212 // 2019/9/11 12:30 UTC
213 base::Time start_time;
214 ASSERT_TRUE(
215 base::Time::FromUTCExploded({2019, 9, 3, 11, 12, 30, 0}, &start_time));
216 result.set_start_time(start_time);
217
218 result.set_duration(base::Milliseconds(1500));
219
220 std::string test_suite_name =
221 testing::UnitTest::GetInstance()->current_test_info()->test_suite_name();
222 auto pos = test_suite_name.rfind('/');
223 ASSERT_NE(pos, std::string::npos);
224 std::string type_param_name = test_suite_name.substr(pos + 1);
225
226 const std::string json_template =
227 R"({
228 "testResult":{
229 "expected":true,
230 "runDuration":"1.50s",
231 "startTime":"2019-09-11T12:30:00.000Z",
232 "status":"PASS",
233 "testPath":"SomeTypes/LuciTestResultTypedTest/%s.Variant",
234 "variant":{"param/instantiation":"%s"}
235 }
236 })";
237 // Note that chromium has RTTI disabled. As a result, type_param() and
238 // GetTypeName<> always returns a generic "<type>".
239 const std::string expected_json =
240 base::StringPrintf(json_template.c_str(), type_param_name.c_str(),
241 testing::internal::GetTypeName<TypeParam>().c_str());
242 this->ValidateResult(result, expected_json);
243 }
244
245 REGISTER_TYPED_TEST_SUITE_P(LuciTestResultTypedTest, Variant);
246
247 using SomeTypes = testing::Types<int, double>;
248 INSTANTIATE_TYPED_TEST_SUITE_P(SomeTypes, LuciTestResultTypedTest, SomeTypes);
249
250 } // namespace perf_test
251