1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <errno.h>
18 #include <getopt.h>
19 #include <inttypes.h>
20 #include <libgen.h>
21 #include <stdarg.h>
22 #include <stdio.h>
23 #include <stdlib.h>
24 #include <string.h>
25 #include <sys/wait.h>
26 #include <time.h>
27 #include <unistd.h>
28
29 #include <string>
30 #include <vector>
31
32 #include <android-base/chrono_utils.h>
33 #include <android-base/file.h>
34 #include <android-base/stringprintf.h>
35 #include <android-base/strings.h>
36 #include <android-base/test_utils.h>
37
38 // Example:
39
40 // name: unzip -n
41 // before: mkdir -p d1/d2
42 // before: echo b > d1/d2/a.txt
43 // command: unzip -q -n $FILES/zip/example.zip d1/d2/a.txt && cat d1/d2/a.txt
44 // expected-stdout:
45 // b
46
47 struct Test {
48 std::string test_filename;
49 std::string name;
50 std::string command;
51 std::vector<std::string> befores;
52 std::vector<std::string> afters;
53 std::string expected_stdout;
54 std::string expected_stderr;
55 int exit_status = 0;
56 };
57
58 static const char* g_progname;
59 static bool g_verbose;
60
61 static const char* g_file;
62 static size_t g_line;
63
64 enum Color { kRed, kGreen };
65
Print(Color c,const char * lhs,const char * fmt,...)66 static void Print(Color c, const char* lhs, const char* fmt, ...) {
67 va_list ap;
68 va_start(ap, fmt);
69 if (isatty(0)) printf("%s", (c == kRed) ? "\e[31m" : "\e[32m");
70 printf("%s%s", lhs, isatty(0) ? "\e[0m" : "");
71 vfprintf(stdout, fmt, ap);
72 putchar('\n');
73 va_end(ap);
74 }
75
Die(int error,const char * fmt,...)76 static void Die(int error, const char* fmt, ...) {
77 va_list ap;
78 va_start(ap, fmt);
79 fprintf(stderr, "%s: ", g_progname);
80 vfprintf(stderr, fmt, ap);
81 if (error != 0) fprintf(stderr, ": %s", strerror(error));
82 fprintf(stderr, "\n");
83 va_end(ap);
84 _exit(1);
85 }
86
V(const char * fmt,...)87 static void V(const char* fmt, ...) {
88 if (!g_verbose) return;
89
90 va_list ap;
91 va_start(ap, fmt);
92 fprintf(stderr, " - ");
93 vfprintf(stderr, fmt, ap);
94 fprintf(stderr, "\n");
95 va_end(ap);
96 }
97
SetField(const char * what,std::string * field,std::string_view value)98 static void SetField(const char* what, std::string* field, std::string_view value) {
99 if (!field->empty()) {
100 Die(0, "%s:%zu: %s already set to '%s'", g_file, g_line, what, field->c_str());
101 }
102 field->assign(value);
103 }
104
105 // Similar to ConsumePrefix, but also trims, so "key:value" and "key: value"
106 // are equivalent.
Match(std::string * s,const std::string & prefix)107 static bool Match(std::string* s, const std::string& prefix) {
108 if (!android::base::StartsWith(*s, prefix)) return false;
109 s->assign(android::base::Trim(s->substr(prefix.length())));
110 return true;
111 }
112
CollectTests(std::vector<Test> * tests,const char * test_filename)113 static void CollectTests(std::vector<Test>* tests, const char* test_filename) {
114 std::string absolute_test_filename;
115 if (!android::base::Realpath(test_filename, &absolute_test_filename)) {
116 Die(errno, "realpath '%s'", test_filename);
117 }
118
119 std::string content;
120 if (!android::base::ReadFileToString(test_filename, &content)) {
121 Die(errno, "couldn't read '%s'", test_filename);
122 }
123
124 size_t count = 0;
125 g_file = test_filename;
126 g_line = 0;
127 auto lines = android::base::Split(content, "\n");
128 std::unique_ptr<Test> test(new Test);
129 while (g_line < lines.size()) {
130 auto line = lines[g_line++];
131 if (line.empty() || line[0] == '#') continue;
132
133 if (line[0] == '-') {
134 if (test->name.empty() || test->command.empty()) {
135 Die(0, "%s:%zu: each test requires both a name and a command", g_file, g_line);
136 }
137 test->test_filename = absolute_test_filename;
138 tests->push_back(*test.release());
139 test.reset(new Test);
140 ++count;
141 } else if (Match(&line, "name:")) {
142 SetField("name", &test->name, line);
143 } else if (Match(&line, "command:")) {
144 SetField("command", &test->command, line);
145 } else if (Match(&line, "before:")) {
146 test->befores.push_back(line);
147 } else if (Match(&line, "after:")) {
148 test->afters.push_back(line);
149 } else if (Match(&line, "expected-exit-status:")) {
150 char* end_p;
151 errno = 0;
152 test->exit_status = strtol(line.c_str(), &end_p, 10);
153 if (errno != 0 || *end_p != '\0') {
154 Die(0, "%s:%zu: bad exit status: \"%s\"", g_file, g_line, line.c_str());
155 }
156 } else if (Match(&line, "expected-stdout:")) {
157 // Collect tab-indented lines.
158 std::string text;
159 while (g_line < lines.size() && !lines[g_line].empty() && lines[g_line][0] == '\t') {
160 text += lines[g_line++].substr(1) + "\n";
161 }
162 SetField("expected stdout", &test->expected_stdout, text);
163 } else {
164 Die(0, "%s:%zu: syntax error: \"%s\"", g_file, g_line, line.c_str());
165 }
166 }
167 if (count == 0) Die(0, "no tests found in '%s'", g_file);
168 }
169
Plural(size_t n)170 static const char* Plural(size_t n) {
171 return (n == 1) ? "" : "s";
172 }
173
ExitStatusToString(int status)174 static std::string ExitStatusToString(int status) {
175 if (WIFSIGNALED(status)) {
176 return android::base::StringPrintf("was killed by signal %d (%s)", WTERMSIG(status),
177 strsignal(WTERMSIG(status)));
178 }
179 if (WIFSTOPPED(status)) {
180 return android::base::StringPrintf("was stopped by signal %d (%s)", WSTOPSIG(status),
181 strsignal(WSTOPSIG(status)));
182 }
183 return android::base::StringPrintf("exited with status %d", WEXITSTATUS(status));
184 }
185
RunCommands(const char * what,const std::vector<std::string> & commands)186 static bool RunCommands(const char* what, const std::vector<std::string>& commands) {
187 bool result = true;
188 for (auto& command : commands) {
189 V("running %s \"%s\"", what, command.c_str());
190 int exit_status = system(command.c_str());
191 if (exit_status != 0) {
192 result = false;
193 fprintf(stderr, "Command (%s) \"%s\" %s\n", what, command.c_str(),
194 ExitStatusToString(exit_status).c_str());
195 }
196 }
197 return result;
198 }
199
CheckOutput(const char * what,std::string actual_output,const std::string & expected_output,const std::string & FILES)200 static bool CheckOutput(const char* what, std::string actual_output,
201 const std::string& expected_output, const std::string& FILES) {
202 // Rewrite the output to reverse any expansion of $FILES.
203 actual_output = android::base::StringReplace(actual_output, FILES, "$FILES", true);
204
205 bool result = (actual_output == expected_output);
206 if (!result) {
207 fprintf(stderr, "Incorrect %s.\nExpected:\n%s\nActual:\n%s\n", what, expected_output.c_str(),
208 actual_output.c_str());
209 }
210 return result;
211 }
212
RunTests(const std::vector<Test> & tests)213 static int RunTests(const std::vector<Test>& tests) {
214 std::vector<std::string> failures;
215
216 Print(kGreen, "[==========]", " Running %zu tests.", tests.size());
217 android::base::Timer total_timer;
218 for (const auto& test : tests) {
219 bool failed = false;
220
221 Print(kGreen, "[ RUN ]", " %s", test.name.c_str());
222 android::base::Timer test_timer;
223
224 // Set $FILES for this test.
225 std::string FILES = android::base::Dirname(test.test_filename) + "/files";
226 V("setenv(\"FILES\", \"%s\")", FILES.c_str());
227 setenv("FILES", FILES.c_str(), 1);
228
229 // Make a safe space to run the test.
230 TemporaryDir td;
231 V("chdir(\"%s\")", td.path);
232 if (chdir(td.path)) Die(errno, "chdir(\"%s\")", td.path);
233
234 // Perform any setup specified for this test.
235 if (!RunCommands("before", test.befores)) failed = true;
236
237 if (!failed) {
238 V("running command \"%s\"", test.command.c_str());
239 CapturedStdout test_stdout;
240 CapturedStderr test_stderr;
241 int status = system(test.command.c_str());
242 test_stdout.Stop();
243 test_stderr.Stop();
244
245 V("system() returned status %d", status);
246 if (WEXITSTATUS(status) != test.exit_status) {
247 failed = true;
248 fprintf(stderr, "Incorrect exit status: expected %d but %s\n", test.exit_status,
249 ExitStatusToString(status).c_str());
250 }
251
252 if (!CheckOutput("stdout", test_stdout.str(), test.expected_stdout, FILES)) failed = true;
253 if (!CheckOutput("stderr", test_stderr.str(), test.expected_stderr, FILES)) failed = true;
254
255 if (!RunCommands("after", test.afters)) failed = true;
256 }
257
258 std::stringstream duration;
259 duration << test_timer;
260 if (failed) {
261 failures.push_back(test.name);
262 Print(kRed, "[ FAILED ]", " %s (%s)", test.name.c_str(), duration.str().c_str());
263 } else {
264 Print(kGreen, "[ OK ]", " %s (%s)", test.name.c_str(), duration.str().c_str());
265 }
266 }
267
268 // Summarize the whole run and explicitly list all the failures.
269
270 std::stringstream duration;
271 duration << total_timer;
272 Print(kGreen, "[==========]", " %zu tests ran. (%s total)", tests.size(), duration.str().c_str());
273
274 size_t fail_count = failures.size();
275 size_t pass_count = tests.size() - fail_count;
276 Print(kGreen, "[ PASSED ]", " %zu test%s.", pass_count, Plural(pass_count));
277 if (!failures.empty()) {
278 Print(kRed, "[ FAILED ]", " %zu test%s.", fail_count, Plural(fail_count));
279 for (auto& failure : failures) {
280 Print(kRed, "[ FAILED ]", " %s", failure.c_str());
281 }
282 }
283 return (fail_count == 0) ? 0 : 1;
284 }
285
ShowHelp(bool full)286 static void ShowHelp(bool full) {
287 fprintf(full ? stdout : stderr, "usage: %s [-v] FILE...\n", g_progname);
288 if (!full) exit(EXIT_FAILURE);
289
290 printf(
291 "\n"
292 "Run tests.\n"
293 "\n"
294 "-v\tVerbose (show workings)\n");
295 exit(EXIT_SUCCESS);
296 }
297
main(int argc,char * argv[])298 int main(int argc, char* argv[]) {
299 g_progname = basename(argv[0]);
300
301 static const struct option opts[] = {
302 {"help", no_argument, 0, 'h'},
303 {"verbose", no_argument, 0, 'v'},
304 {},
305 };
306
307 int opt;
308 while ((opt = getopt_long(argc, argv, "hv", opts, nullptr)) != -1) {
309 switch (opt) {
310 case 'h':
311 ShowHelp(true);
312 break;
313 case 'v':
314 g_verbose = true;
315 break;
316 default:
317 ShowHelp(false);
318 break;
319 }
320 }
321
322 argv += optind;
323 if (!*argv) Die(0, "no test files provided");
324 std::vector<Test> tests;
325 for (; *argv; ++argv) CollectTests(&tests, *argv);
326 return RunTests(tests);
327 }
328