• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc.  All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 //     * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //     * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 //     * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 
31 // This file contains a program for running the test suite in a separate
32 // process.  The other alternative is to run the suite in-process.  See
33 // conformance.proto for pros/cons of these two options.
34 //
35 // This program will fork the process under test and communicate with it over
36 // its stdin/stdout:
37 //
38 //     +--------+   pipe   +----------+
39 //     | tester | <------> | testee   |
40 //     |        |          |          |
41 //     |  C++   |          | any lang |
42 //     +--------+          +----------+
43 //
44 // The tester contains all of the test cases and their expected output.
45 // The testee is a simple program written in the target language that reads
46 // each test case and attempts to produce acceptable output for it.
47 //
48 // Every test consists of a ConformanceRequest/ConformanceResponse
49 // request/reply pair.  The protocol on the pipe is simply:
50 //
51 //   1. tester sends 4-byte length N (little endian)
52 //   2. tester sends N bytes representing a ConformanceRequest proto
53 //   3. testee sends 4-byte length M (little endian)
54 //   4. testee sends M bytes representing a ConformanceResponse proto
55 
56 #include <errno.h>
57 #include <sys/types.h>
58 #include <sys/wait.h>
59 #include <unistd.h>
60 
61 #include <algorithm>
62 #include <fstream>
63 #include <vector>
64 
65 #include <google/protobuf/stubs/stringprintf.h>
66 #include "conformance.pb.h"
67 #include "conformance_test.h"
68 
69 using conformance::ConformanceResponse;
70 using google::protobuf::ConformanceTestSuite;
71 using std::string;
72 using std::vector;
73 
74 #define STRINGIFY(x) #x
75 #define TOSTRING(x) STRINGIFY(x)
76 #define GOOGLE_CHECK_SYSCALL(call) \
77   if (call < 0) { \
78     perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
79     exit(1); \
80   }
81 
82 namespace google {
83 namespace protobuf {
84 
ParseFailureList(const char * filename,conformance::FailureSet * failure_list)85 void ParseFailureList(const char *filename,
86                       conformance::FailureSet *failure_list) {
87   std::ifstream infile(filename);
88 
89   if (!infile.is_open()) {
90     fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
91     exit(1);
92   }
93 
94   for (string line; getline(infile, line);) {
95     // Remove whitespace.
96     line.erase(std::remove_if(line.begin(), line.end(), ::isspace),
97                line.end());
98 
99     // Remove comments.
100     line = line.substr(0, line.find("#"));
101 
102     if (!line.empty()) {
103       failure_list->add_failure(line);
104     }
105   }
106 }
107 
UsageError()108 void UsageError() {
109   fprintf(stderr,
110           "Usage: conformance-test-runner [options] <test-program>\n");
111   fprintf(stderr, "\n");
112   fprintf(stderr, "Options:\n");
113   fprintf(stderr,
114           "  --failure_list <filename>   Use to specify list of tests\n");
115   fprintf(stderr,
116           "                              that are expected to fail.  File\n");
117   fprintf(stderr,
118           "                              should contain one test name per\n");
119   fprintf(stderr,
120           "                              line.  Use '#' for comments.\n");
121   fprintf(stderr,
122           "  --text_format_failure_list <filename>   Use to specify list \n");
123   fprintf(stderr,
124           "                              of tests that are expected to \n");
125   fprintf(stderr,
126           "                              fail in the \n");
127   fprintf(stderr,
128           "                              text_format_conformance_suite.  \n");
129   fprintf(stderr,
130           "                              File should contain one test name \n");
131   fprintf(stderr,
132           "                              per line.  Use '#' for comments.\n");
133 
134   fprintf(stderr,
135           "  --enforce_recommended       Enforce that recommended test\n");
136   fprintf(stderr,
137           "                              cases are also passing. Specify\n");
138   fprintf(stderr,
139           "                              this flag if you want to be\n");
140   fprintf(stderr,
141           "                              strictly conforming to protobuf\n");
142   fprintf(stderr,
143           "                              spec.\n");
144   exit(1);
145 }
146 
RunTest(const std::string & test_name,const std::string & request,std::string * response)147 void ForkPipeRunner::RunTest(
148     const std::string& test_name,
149     const std::string& request,
150     std::string* response) {
151   if (child_pid_ < 0) {
152     SpawnTestProgram();
153   }
154 
155   current_test_name_ = test_name;
156 
157   uint32_t len = request.size();
158   CheckedWrite(write_fd_, &len, sizeof(uint32_t));
159   CheckedWrite(write_fd_, request.c_str(), request.size());
160 
161   if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
162     // We failed to read from the child, assume a crash and try to reap.
163     GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
164 
165     int status;
166     waitpid(child_pid_, &status, WEXITED);
167 
168     string error_msg;
169     if (WIFEXITED(status)) {
170       StringAppendF(&error_msg,
171                     "child exited, status=%d", WEXITSTATUS(status));
172     } else if (WIFSIGNALED(status)) {
173       StringAppendF(&error_msg,
174                     "child killed by signal %d", WTERMSIG(status));
175     }
176     GOOGLE_LOG(INFO) << error_msg;
177     child_pid_ = -1;
178 
179     conformance::ConformanceResponse response_obj;
180     response_obj.set_runtime_error(error_msg);
181     response_obj.SerializeToString(response);
182     return;
183   }
184 
185   response->resize(len);
186   CheckedRead(read_fd_, (void*)response->c_str(), len);
187 }
188 
Run(int argc,char * argv[],const std::vector<ConformanceTestSuite * > & suites)189 int ForkPipeRunner::Run(
190     int argc, char *argv[], const std::vector<ConformanceTestSuite*>& suites) {
191   if (suites.empty()) {
192     fprintf(stderr, "No test suites found.\n");
193     return EXIT_FAILURE;
194   }
195   bool all_ok = true;
196   for (ConformanceTestSuite* suite : suites) {
197     string program;
198     std::vector<string> program_args;
199     string failure_list_filename;
200     conformance::FailureSet failure_list;
201 
202     for (int arg = 1; arg < argc; ++arg) {
203       if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
204         if (++arg == argc) UsageError();
205         failure_list_filename = argv[arg];
206         ParseFailureList(argv[arg], &failure_list);
207       } else if (strcmp(argv[arg], "--verbose") == 0) {
208         suite->SetVerbose(true);
209       } else if (strcmp(argv[arg], "--enforce_recommended") == 0) {
210         suite->SetEnforceRecommended(true);
211       } else if (argv[arg][0] == '-') {
212         bool recognized_flag = false;
213         for (ConformanceTestSuite* suite : suites) {
214           if (strcmp(argv[arg], suite->GetFailureListFlagName().c_str()) == 0) {
215             if (++arg == argc) UsageError();
216             recognized_flag = true;
217           }
218         }
219         if (!recognized_flag) {
220           fprintf(stderr, "Unknown option: %s\n", argv[arg]);
221           UsageError();
222         }
223       } else {
224         program += argv[arg];
225         while (arg < argc) {
226           program_args.push_back(argv[arg]);
227           arg++;
228         }
229       }
230     }
231 
232     ForkPipeRunner runner(program, program_args);
233 
234     std::string output;
235     all_ok = all_ok &&
236         suite->RunSuite(&runner, &output, failure_list_filename, &failure_list);
237 
238     fwrite(output.c_str(), 1, output.size(), stderr);
239   }
240   return all_ok ? EXIT_SUCCESS : EXIT_FAILURE;
241 }
242 
243 // TODO(haberman): make this work on Windows, instead of using these
244 // UNIX-specific APIs.
245 //
246 // There is a platform-agnostic API in
247 //    src/google/protobuf/compiler/subprocess.h
248 //
249 // However that API only supports sending a single message to the subprocess.
250 // We really want to be able to send messages and receive responses one at a
251 // time:
252 //
253 // 1. Spawning a new process for each test would take way too long for thousands
254 //    of tests and subprocesses like java that can take 100ms or more to start
255 //    up.
256 //
257 // 2. Sending all the tests in one big message and receiving all results in one
258 //    big message would take away our visibility about which test(s) caused a
259 //    crash or other fatal error.  It would also give us only a single failure
260 //    instead of all of them.
SpawnTestProgram()261 void ForkPipeRunner::SpawnTestProgram() {
262   int toproc_pipe_fd[2];
263   int fromproc_pipe_fd[2];
264   if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
265     perror("pipe");
266     exit(1);
267   }
268 
269   pid_t pid = fork();
270   if (pid < 0) {
271     perror("fork");
272     exit(1);
273   }
274 
275   if (pid) {
276     // Parent.
277     GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
278     GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
279     write_fd_ = toproc_pipe_fd[1];
280     read_fd_ = fromproc_pipe_fd[0];
281     child_pid_ = pid;
282   } else {
283     // Child.
284     GOOGLE_CHECK_SYSCALL(close(STDIN_FILENO));
285     GOOGLE_CHECK_SYSCALL(close(STDOUT_FILENO));
286     GOOGLE_CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
287     GOOGLE_CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
288 
289     GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[0]));
290     GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
291     GOOGLE_CHECK_SYSCALL(close(toproc_pipe_fd[1]));
292     GOOGLE_CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
293 
294     std::unique_ptr<char[]> executable(new char[executable_.size() + 1]);
295     memcpy(executable.get(), executable_.c_str(), executable_.size());
296     executable[executable_.size()] = '\0';
297 
298     std::vector<const char *> argv;
299     argv.push_back(executable.get());
300     for (int i = 0; i < executable_args_.size(); ++i) {
301       argv.push_back(executable_args_[i].c_str());
302     }
303     argv.push_back(nullptr);
304     // Never returns.
305     GOOGLE_CHECK_SYSCALL(execv(executable.get(), const_cast<char **>(argv.data())));
306   }
307 }
308 
CheckedWrite(int fd,const void * buf,size_t len)309 void ForkPipeRunner::CheckedWrite(int fd, const void *buf, size_t len) {
310   if (write(fd, buf, len) != len) {
311     GOOGLE_LOG(FATAL) << current_test_name_
312                << ": error writing to test program: " << strerror(errno);
313   }
314 }
315 
TryRead(int fd,void * buf,size_t len)316 bool ForkPipeRunner::TryRead(int fd, void *buf, size_t len) {
317   size_t ofs = 0;
318   while (len > 0) {
319     ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
320 
321     if (bytes_read == 0) {
322       GOOGLE_LOG(ERROR) << current_test_name_ << ": unexpected EOF from test program";
323       return false;
324     } else if (bytes_read < 0) {
325       GOOGLE_LOG(ERROR) << current_test_name_
326                  << ": error reading from test program: " << strerror(errno);
327       return false;
328     }
329 
330     len -= bytes_read;
331     ofs += bytes_read;
332   }
333 
334   return true;
335 }
336 
CheckedRead(int fd,void * buf,size_t len)337 void ForkPipeRunner::CheckedRead(int fd, void *buf, size_t len) {
338   if (!TryRead(fd, buf, len)) {
339     GOOGLE_LOG(FATAL) << current_test_name_
340                << ": error reading from test program: " << strerror(errno);
341   }
342 }
343 
344 }  // namespace protobuf
345 }  // namespace google
346