1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 // * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 // * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 // * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 // This file contains a program for running the test suite in a separate
32 // process. The other alternative is to run the suite in-process. See
33 // conformance.proto for pros/cons of these two options.
34 //
35 // This program will fork the process under test and communicate with it over
36 // its stdin/stdout:
37 //
38 // +--------+ pipe +----------+
39 // | tester | <------> | testee |
40 // | | | |
41 // | C++ | | any lang |
42 // +--------+ +----------+
43 //
44 // The tester contains all of the test cases and their expected output.
45 // The testee is a simple program written in the target language that reads
46 // each test case and attempts to produce acceptable output for it.
47 //
48 // Every test consists of a ConformanceRequest/ConformanceResponse
49 // request/reply pair. The protocol on the pipe is simply:
50 //
51 // 1. tester sends 4-byte length N (little endian)
52 // 2. tester sends N bytes representing a ConformanceRequest proto
53 // 3. testee sends 4-byte length M (little endian)
54 // 4. testee sends M bytes representing a ConformanceResponse proto
55
56 #include <errno.h>
57 #include <sys/types.h>
58 #include <sys/wait.h>
59 #include <unistd.h>
60
61 #include <algorithm>
62 #include <fstream>
63 #include <vector>
64
65 #include <google/protobuf/stubs/stringprintf.h>
66 #include "conformance.pb.h"
67 #include "conformance_test.h"
68
69 using conformance::ConformanceResponse;
70 using google::protobuf::ConformanceTestSuite;
71 using std::string;
72 using std::vector;
73
74 #define STRINGIFY(x) #x
75 #define TOSTRING(x) STRINGIFY(x)
76 #define GOOGLE_CHECK_SYSCALL(call) \
77 if (call < 0) { \
78 perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
79 exit(1); \
80 }
81
82 namespace google {
83 namespace protobuf {
84
ParseFailureList(const char * filename,conformance::FailureSet * failure_list)85 void ParseFailureList(const char *filename,
86 conformance::FailureSet *failure_list) {
87 std::ifstream infile(filename);
88
89 if (!infile.is_open()) {
90 fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
91 exit(1);
92 }
93
94 for (string line; getline(infile, line);) {
95 // Remove whitespace.
96 line.erase(std::remove_if(line.begin(), line.end(), ::isspace),
97 line.end());
98
99 // Remove comments.
100 line = line.substr(0, line.find("#"));
101
102 if (!line.empty()) {
103 failure_list->add_failure(line);
104 }
105 }
106 }
107
UsageError()108 void UsageError() {
109 fprintf(stderr,
110 "Usage: conformance-test-runner [options] <test-program>\n");
111 fprintf(stderr, "\n");
112 fprintf(stderr, "Options:\n");
113 fprintf(stderr,
114 " --failure_list <filename> Use to specify list of tests\n");
115 fprintf(stderr,
116 " that are expected to fail. File\n");
117 fprintf(stderr,
118 " should contain one test name per\n");
119 fprintf(stderr,
120 " line. Use '#' for comments.\n");
121 fprintf(stderr,
122 " --text_format_failure_list <filename> Use to specify list \n");
123 fprintf(stderr,
124 " of tests that are expected to \n");
125 fprintf(stderr,
126 " fail in the \n");
127 fprintf(stderr,
128 " text_format_conformance_suite. \n");
129 fprintf(stderr,
130 " File should contain one test name \n");
131 fprintf(stderr,
132 " per line. Use '#' for comments.\n");
133
134 fprintf(stderr,
135 " --enforce_recommended Enforce that recommended test\n");
136 fprintf(stderr,
137 " cases are also passing. Specify\n");
138 fprintf(stderr,
139 " this flag if you want to be\n");
140 fprintf(stderr,
141 " strictly conforming to protobuf\n");
142 fprintf(stderr,
143 " spec.\n");
144 fprintf(stderr,
145 " --output_dir <dirname> Directory to write\n"
146 " output files.\n");
147 exit(1);
148 }
149
RunTest(const std::string & test_name,const std::string & request,std::string * response)150 void ForkPipeRunner::RunTest(
151 const std::string& test_name,
152 const std::string& request,
153 std::string* response) {
154 if (child_pid_ < 0) {
155 SpawnTestProgram();
156 }
157
158 current_test_name_ = test_name;
159
160 uint32_t len = request.size();
161 CheckedWrite(write_fd_, &len, sizeof(uint32_t));
162 CheckedWrite(write_fd_, request.c_str(), request.size());
163
164 if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
165 // We failed to read from the child, assume a crash and try to reap.
166 GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
167
168 int status = 0;
169 waitpid(child_pid_, &status, WEXITED);
170
171 string error_msg;
172 if (WIFEXITED(status)) {
173 StringAppendF(&error_msg,
174 "child exited, status=%d", WEXITSTATUS(status));
175 } else if (WIFSIGNALED(status)) {
176 StringAppendF(&error_msg,
177 "child killed by signal %d", WTERMSIG(status));
178 }
179 GOOGLE_LOG(INFO) << error_msg;
180 child_pid_ = -1;
181
182 conformance::ConformanceResponse response_obj;
183 response_obj.set_runtime_error(error_msg);
184 response_obj.SerializeToString(response);
185 return;
186 }
187
188 response->resize(len);
189 CheckedRead(read_fd_, (void*)response->c_str(), len);
190 }
191
Run(int argc,char * argv[],const std::vector<ConformanceTestSuite * > & suites)192 int ForkPipeRunner::Run(
193 int argc, char *argv[], const std::vector<ConformanceTestSuite*>& suites) {
194 if (suites.empty()) {
195 fprintf(stderr, "No test suites found.\n");
196 return EXIT_FAILURE;
197 }
198 bool all_ok = true;
199 for (ConformanceTestSuite* suite : suites) {
200 string program;
201 std::vector<string> program_args;
202 string failure_list_filename;
203 conformance::FailureSet failure_list;
204
205 for (int arg = 1; arg < argc; ++arg) {
206 if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
207 if (++arg == argc) UsageError();
208 failure_list_filename = argv[arg];
209 ParseFailureList(argv[arg], &failure_list);
210 } else if (strcmp(argv[arg], "--verbose") == 0) {
211 suite->SetVerbose(true);
212 } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
213 suite->SetEnforceRecommended(true);
214 } else if (strcmp(argv[arg], "--output_dir") == 0) {
215 if (++arg == argc) UsageError();
216 suite->SetOutputDir(argv[arg]);
217 } else if (argv[arg][0] == '-') {
218 bool recognized_flag = false;
219 for (ConformanceTestSuite* suite : suites) {
220 if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
221 if (++arg == argc) UsageError();
222 recognized_flag = true;
223 }
224 }
225 if (!recognized_flag) {
226 fprintf(stderr, "Unknown option: %s\n", argv[arg]);
227 UsageError();
228 }
229 } else {
230 program += argv[arg];
231 while (arg < argc) {
232 program_args.push_back(argv[arg]);
233 arg++;
234 }
235 }
236 }
237
238 ForkPipeRunner runner(program, program_args);
239
240 std::string output;
241 all_ok = all_ok &&
242 suite->RunSuite(&runner, &output, failure_list_filename, &failure_list);
243
244 fwrite(output.c_str(), 1, output.size(), stderr);
245 }
246 return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
247 }
248
249 // TODO(haberman): make this work on Windows, instead of using these
250 // UNIX-specific APIs.
251 //
252 // There is a platform-agnostic API in
253 // src/google/protobuf/compiler/subprocess.h
254 //
255 // However that API only supports sending a single message to the subprocess.
256 // We really want to be able to send messages and receive responses one at a
257 // time:
258 //
259 // 1. Spawning a new process for each test would take way too long for thousands
260 // of tests and subprocesses like java that can take 100ms or more to start
261 // up.
262 //
263 // 2. Sending all the tests in one big message and receiving all results in one
264 // big message would take away our visibility about which test(s) caused a
265 // crash or other fatal error. It would also give us only a single failure
266 // instead of all of them.
SpawnTestProgram()267 void ForkPipeRunner::SpawnTestProgram() {
268 int toproc_pipe_fd[2];
269 int fromproc_pipe_fd[2];
270 if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
271 perror("pipe");
272 exit(1);
273 }
274
275 pid_t pid = fork();
276 if (pid < 0) {
277 perror("fork");
278 exit(1);
279 }
280
281 if (pid) {
282 // Parent.
283 GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
284 GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
285 write_fd_ = toproc_pipe_fd[1];
286 read_fd_ = fromproc_pipe_fd[0];
287 child_pid_ = pid;
288 } else {
289 // Child.
290 GOOGLE_CHECK_SYSCALL(close(STDIN_FILENO));
291 GOOGLE_CHECK_SYSCALL(close(STDOUT_FILENO));
292 GOOGLE_CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
293 GOOGLE_CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
294
295 GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
296 GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
297 GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[1]));
298 GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
299
300 std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
301 memcpy(executable.get(), executable_.c_str(), executable_.size());
302 executable[executable_.size()] = '\0';
303
304 std::vector<const char *> argv;
305 argv.push_back(executable.get());
306 for (size_t i = 0; i < executable_args_.size(); ++i) {
307 argv.push_back(executable_args_[i].c_str());
308 }
309 argv.push_back(nullptr);
310 // Never returns.
311 GOOGLE_CHECK_SYSCALL(execv(executable.get(), const_cast<char **>(argv.data())));
312 }
313 }
314
CheckedWrite(int fd,const void * buf,size_t len)315 void ForkPipeRunner::CheckedWrite(int fd, const void *buf, size_t len) {
316 if (static_cast<size_t>(write(fd, buf, len)) != len) {
317 GOOGLE_LOG(FATAL) << current_test_name_
318 << ": error writing to test program: " << strerror(errno);
319 }
320 }
321
TryRead(int fd,void * buf,size_t len)322 bool ForkPipeRunner::TryRead(int fd, void *buf, size_t len) {
323 size_t ofs = 0;
324 while (len > 0) {
325 ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
326
327 if (bytes_read == 0) {
328 GOOGLE_LOG(ERROR) << current_test_name_ << ": unexpected EOF from test program";
329 return false;
330 } else if (bytes_read < 0) {
331 GOOGLE_LOG(ERROR) << current_test_name_
332 << ": error reading from test program: " << strerror(errno);
333 return false;
334 }
335
336 len -= bytes_read;
337 ofs += bytes_read;
338 }
339
340 return true;
341 }
342
CheckedRead(int fd,void * buf,size_t len)343 void ForkPipeRunner::CheckedRead(int fd, void *buf, size_t len) {
344 if (!TryRead(fd, buf, len)) {
345 GOOGLE_LOG(FATAL) << current_test_name_
346 << ": error reading from test program: " << strerror(errno);
347 }
348 }
349
350 } // namespace protobuf
351 } // namespace google
352