• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc.  All rights reserved.
3 // https://developers.google.com/protocol-buffers/
4 //
5 // Redistribution and use in source and binary forms, with or without
6 // modification, are permitted provided that the following conditions are
7 // met:
8 //
9 //     * Redistributions of source code must retain the above copyright
10 // notice, this list of conditions and the following disclaimer.
11 //     * Redistributions in binary form must reproduce the above
12 // copyright notice, this list of conditions and the following disclaimer
13 // in the documentation and/or other materials provided with the
14 // distribution.
15 //     * Neither the name of Google Inc. nor the names of its
16 // contributors may be used to endorse or promote products derived from
17 // this software without specific prior written permission.
18 //
19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 
31 #include <set>
32 #include <stdarg.h>
33 #include <string>
34 #include <fstream>
35 
36 #include "conformance.pb.h"
37 #include "conformance_test.h"
38 
39 #include <google/protobuf/stubs/stringprintf.h>
40 #include <google/protobuf/stubs/strutil.h>
41 #include <google/protobuf/message.h>
42 #include <google/protobuf/text_format.h>
43 #include <google/protobuf/util/field_comparator.h>
44 #include <google/protobuf/util/json_util.h>
45 #include <google/protobuf/util/message_differencer.h>
46 
47 using conformance::ConformanceRequest;
48 using conformance::ConformanceResponse;
49 using conformance::WireFormat;
50 using google::protobuf::TextFormat;
51 using google::protobuf::util::DefaultFieldComparator;
52 using google::protobuf::util::JsonToBinaryString;
53 using google::protobuf::util::MessageDifferencer;
54 using google::protobuf::util::Status;
55 using std::string;
56 
57 namespace google {
58 namespace protobuf {
59 
ConformanceRequestSetting(ConformanceLevel level,conformance::WireFormat input_format,conformance::WireFormat output_format,conformance::TestCategory test_category,const Message & prototype_message,const string & test_name,const string & input)60 ConformanceTestSuite::ConformanceRequestSetting::ConformanceRequestSetting(
61     ConformanceLevel level,
62     conformance::WireFormat input_format,
63     conformance::WireFormat output_format,
64     conformance::TestCategory test_category,
65     const Message& prototype_message,
66     const string& test_name, const string& input)
67     : level_(level),
68       input_format_(input_format),
69       output_format_(output_format),
70       prototype_message_(prototype_message),
71       prototype_message_for_compare_(prototype_message.New()),
72       test_name_(test_name) {
73   switch (input_format) {
74     case conformance::PROTOBUF: {
75       request_.set_protobuf_payload(input);
76       break;
77     }
78 
79     case conformance::JSON: {
80       request_.set_json_payload(input);
81       break;
82     }
83 
84     case conformance::JSPB: {
85       request_.set_jspb_payload(input);
86       break;
87     }
88 
89     case conformance::TEXT_FORMAT: {
90       request_.set_text_payload(input);
91       break;
92     }
93 
94     default:
95       GOOGLE_LOG(FATAL) << "Unspecified input format";
96   }
97 
98   request_.set_test_category(test_category);
99 
100   request_.set_message_type(prototype_message.GetDescriptor()->full_name());
101   request_.set_requested_output_format(output_format);
102 }
103 
104 Message* ConformanceTestSuite::ConformanceRequestSetting::
GetTestMessage() const105     GetTestMessage() const {
106   return prototype_message_for_compare_->New();
107 }
108 
109 string ConformanceTestSuite::ConformanceRequestSetting::
GetTestName() const110     GetTestName() const {
111   string rname =
112       prototype_message_.GetDescriptor()->file()->syntax() ==
113         FileDescriptor::SYNTAX_PROTO3 ? "Proto3" : "Proto2";
114 
115   return StrCat(ConformanceLevelToString(level_), ".",
116                 rname, ".",
117                 InputFormatString(input_format_),
118                 ".", test_name_, ".",
119                 OutputFormatString(output_format_));
120 }
121 
122 string ConformanceTestSuite::ConformanceRequestSetting::
ConformanceLevelToString(ConformanceLevel level) const123     ConformanceLevelToString(
124         ConformanceLevel level) const {
125   switch (level) {
126     case REQUIRED: return "Required";
127     case RECOMMENDED: return "Recommended";
128   }
129   GOOGLE_LOG(FATAL) << "Unknown value: " << level;
130   return "";
131 }
132 
133 string ConformanceTestSuite::ConformanceRequestSetting::
InputFormatString(conformance::WireFormat format) const134     InputFormatString(conformance::WireFormat format) const {
135   switch (format) {
136     case conformance::PROTOBUF:
137       return "ProtobufInput";
138     case conformance::JSON:
139       return "JsonInput";
140     case conformance::TEXT_FORMAT:
141       return "TextFormatInput";
142     default:
143       GOOGLE_LOG(FATAL) << "Unspecified output format";
144   }
145   return "";
146 }
147 
148 string ConformanceTestSuite::ConformanceRequestSetting::
OutputFormatString(conformance::WireFormat format) const149     OutputFormatString(conformance::WireFormat format) const {
150   switch (format) {
151     case conformance::PROTOBUF:
152       return "ProtobufOutput";
153     case conformance::JSON:
154       return "JsonOutput";
155     case conformance::TEXT_FORMAT:
156       return "TextFormatOutput";
157     default:
158       GOOGLE_LOG(FATAL) << "Unspecified output format";
159   }
160   return "";
161 }
162 
ReportSuccess(const string & test_name)163 void ConformanceTestSuite::ReportSuccess(const string& test_name) {
164   if (expected_to_fail_.erase(test_name) != 0) {
165     StringAppendF(&output_,
166                   "ERROR: test %s is in the failure list, but test succeeded.  "
167                   "Remove it from the failure list.\n",
168                   test_name.c_str());
169     unexpected_succeeding_tests_.insert(test_name);
170   }
171   successes_++;
172 }
173 
ReportFailure(const string & test_name,ConformanceLevel level,const ConformanceRequest & request,const ConformanceResponse & response,const char * fmt,...)174 void ConformanceTestSuite::ReportFailure(const string& test_name,
175                                          ConformanceLevel level,
176                                          const ConformanceRequest& request,
177                                          const ConformanceResponse& response,
178                                          const char* fmt, ...) {
179   if (expected_to_fail_.erase(test_name) == 1) {
180     expected_failures_++;
181     if (!verbose_)
182       return;
183   } else if (level == RECOMMENDED && !enforce_recommended_) {
184     StringAppendF(&output_, "WARNING, test=%s: ", test_name.c_str());
185   } else {
186     StringAppendF(&output_, "ERROR, test=%s: ", test_name.c_str());
187     unexpected_failing_tests_.insert(test_name);
188   }
189   va_list args;
190   va_start(args, fmt);
191   StringAppendV(&output_, fmt, args);
192   va_end(args);
193   StringAppendF(&output_, " request=%s, response=%s\n",
194                 request.ShortDebugString().c_str(),
195                 response.ShortDebugString().c_str());
196 }
197 
ReportSkip(const string & test_name,const ConformanceRequest & request,const ConformanceResponse & response)198 void ConformanceTestSuite::ReportSkip(const string& test_name,
199                                       const ConformanceRequest& request,
200                                       const ConformanceResponse& response) {
201   if (verbose_) {
202     StringAppendF(&output_, "SKIPPED, test=%s request=%s, response=%s\n",
203                   test_name.c_str(), request.ShortDebugString().c_str(),
204                   response.ShortDebugString().c_str());
205   }
206   skipped_.insert(test_name);
207 }
208 
RunValidInputTest(const ConformanceRequestSetting & setting,const string & equivalent_text_format)209 void ConformanceTestSuite::RunValidInputTest(
210     const ConformanceRequestSetting& setting,
211     const string& equivalent_text_format) {
212   Message* reference_message = setting.GetTestMessage();
213   GOOGLE_CHECK(
214       TextFormat::ParseFromString(equivalent_text_format, reference_message))
215           << "Failed to parse data for test case: " << setting.GetTestName()
216           << ", data: " << equivalent_text_format;
217   const string equivalent_wire_format = reference_message->SerializeAsString();
218   RunValidBinaryInputTest(setting, equivalent_wire_format);
219 }
220 
RunValidBinaryInputTest(const ConformanceRequestSetting & setting,const string & equivalent_wire_format)221 void ConformanceTestSuite::RunValidBinaryInputTest(
222     const ConformanceRequestSetting& setting,
223     const string& equivalent_wire_format) {
224   const ConformanceRequest& request = setting.GetRequest();
225   ConformanceResponse response;
226   RunTest(setting.GetTestName(), request, &response);
227   VerifyResponse(setting, equivalent_wire_format, response, true);
228 }
229 
VerifyResponse(const ConformanceRequestSetting & setting,const string & equivalent_wire_format,const ConformanceResponse & response,bool need_report_success)230 void ConformanceTestSuite::VerifyResponse(
231     const ConformanceRequestSetting& setting,
232     const string& equivalent_wire_format,
233     const ConformanceResponse& response,
234     bool need_report_success) {
235   Message* test_message = setting.GetTestMessage();
236   const ConformanceRequest& request = setting.GetRequest();
237   const string& test_name = setting.GetTestName();
238   ConformanceLevel level = setting.GetLevel();
239   Message* reference_message = setting.GetTestMessage();
240 
241   GOOGLE_CHECK(
242       reference_message->ParseFromString(equivalent_wire_format))
243           << "Failed to parse wire data for test case: " << test_name;
244 
245   switch (response.result_case()) {
246     case ConformanceResponse::RESULT_NOT_SET:
247       ReportFailure(test_name, level, request, response,
248                     "Response didn't have any field in the Response.");
249       return;
250 
251     case ConformanceResponse::kParseError:
252     case ConformanceResponse::kRuntimeError:
253     case ConformanceResponse::kSerializeError:
254       ReportFailure(test_name, level, request, response,
255                     "Failed to parse input or produce output.");
256       return;
257 
258     case ConformanceResponse::kSkipped:
259       ReportSkip(test_name, request, response);
260       return;
261 
262     default:
263       if (!ParseResponse(response, setting, test_message)) return;
264   }
265 
266   MessageDifferencer differencer;
267   DefaultFieldComparator field_comparator;
268   field_comparator.set_treat_nan_as_equal(true);
269   differencer.set_field_comparator(&field_comparator);
270   string differences;
271   differencer.ReportDifferencesToString(&differences);
272 
273   bool check;
274   check = differencer.Compare(*reference_message, *test_message);
275   if (check) {
276     if (need_report_success) {
277       ReportSuccess(test_name);
278     }
279   } else {
280     ReportFailure(test_name, level, request, response,
281                   "Output was not equivalent to reference message: %s.",
282                   differences.c_str());
283   }
284 }
285 
RunTest(const string & test_name,const ConformanceRequest & request,ConformanceResponse * response)286 void ConformanceTestSuite::RunTest(const string& test_name,
287                                    const ConformanceRequest& request,
288                                    ConformanceResponse* response) {
289   if (test_names_.insert(test_name).second == false) {
290     GOOGLE_LOG(FATAL) << "Duplicated test name: " << test_name;
291   }
292 
293   string serialized_request;
294   string serialized_response;
295   request.SerializeToString(&serialized_request);
296 
297   runner_->RunTest(test_name, serialized_request, &serialized_response);
298 
299   if (!response->ParseFromString(serialized_response)) {
300     response->Clear();
301     response->set_runtime_error("response proto could not be parsed.");
302   }
303 
304   if (verbose_) {
305     StringAppendF(&output_,
306                   "conformance test: name=%s, request=%s, response=%s\n",
307                   test_name.c_str(),
308                   request.ShortDebugString().c_str(),
309                   response->ShortDebugString().c_str());
310   }
311 }
312 
CheckSetEmpty(const std::set<string> & set_to_check,const std::string & write_to_file,const std::string & msg)313 bool ConformanceTestSuite::CheckSetEmpty(
314     const std::set<string>& set_to_check,
315     const std::string& write_to_file,
316     const std::string& msg) {
317   if (set_to_check.empty()) {
318     return true;
319   } else {
320     StringAppendF(&output_, "\n");
321     StringAppendF(&output_, "%s\n\n", msg.c_str());
322     for (std::set<string>::const_iterator iter = set_to_check.begin();
323          iter != set_to_check.end(); ++iter) {
324       StringAppendF(&output_, "  %s\n", iter->c_str());
325     }
326     StringAppendF(&output_, "\n");
327 
328     if (!write_to_file.empty()) {
329       std::ofstream os(write_to_file);
330       if (os) {
331         for (std::set<string>::const_iterator iter = set_to_check.begin();
332              iter != set_to_check.end(); ++iter) {
333           os << *iter << "\n";
334         }
335       } else {
336         StringAppendF(&output_, "Failed to open file: %s\n",
337                       write_to_file.c_str());
338       }
339     }
340 
341     return false;
342   }
343 }
344 
WireFormatToString(WireFormat wire_format)345 string ConformanceTestSuite::WireFormatToString(
346     WireFormat wire_format) {
347   switch (wire_format) {
348     case conformance::PROTOBUF:
349       return "PROTOBUF";
350     case conformance::JSON:
351       return "JSON";
352     case conformance::JSPB:
353       return "JSPB";
354     case conformance::TEXT_FORMAT:
355       return "TEXT_FORMAT";
356     case conformance::UNSPECIFIED:
357       return "UNSPECIFIED";
358     default:
359       GOOGLE_LOG(FATAL) << "unknown wire type: "
360                         << wire_format;
361   }
362   return "";
363 }
364 
AddExpectedFailedTest(const std::string & test_name)365 void ConformanceTestSuite::AddExpectedFailedTest(const std::string& test_name) {
366   expected_to_fail_.insert(test_name);
367 }
368 
RunSuite(ConformanceTestRunner * runner,std::string * output,const string & filename,conformance::FailureSet * failure_list)369 bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
370                                     std::string* output, const string& filename,
371                                     conformance::FailureSet* failure_list) {
372   runner_ = runner;
373   successes_ = 0;
374   expected_failures_ = 0;
375   skipped_.clear();
376   test_names_.clear();
377   unexpected_failing_tests_.clear();
378   unexpected_succeeding_tests_.clear();
379 
380   output_ = "\nCONFORMANCE TEST BEGIN ====================================\n\n";
381 
382   failure_list_filename_ = filename;
383   expected_to_fail_.clear();
384   for (const string& failure : failure_list->failure()) {
385     AddExpectedFailedTest(failure);
386   }
387   RunSuiteImpl();
388 
389   bool ok = true;
390   if (!CheckSetEmpty(expected_to_fail_, "nonexistent_tests.txt",
391                      "These tests were listed in the failure list, but they "
392                      "don't exist.  Remove them from the failure list by "
393                      "running:\n"
394                      "  ./update_failure_list.py " + failure_list_filename_ +
395                      " --remove nonexistent_tests.txt")) {
396     ok = false;
397   }
398   if (!CheckSetEmpty(unexpected_failing_tests_, "failing_tests.txt",
399                      "These tests failed.  If they can't be fixed right now, "
400                      "you can add them to the failure list so the overall "
401                      "suite can succeed.  Add them to the failure list by "
402                      "running:\n"
403                      "  ./update_failure_list.py " + failure_list_filename_ +
404                      " --add failing_tests.txt")) {
405     ok = false;
406   }
407   if (!CheckSetEmpty(unexpected_succeeding_tests_, "succeeding_tests.txt",
408                      "These tests succeeded, even though they were listed in "
409                      "the failure list.  Remove them from the failure list "
410                      "by running:\n"
411                      "  ./update_failure_list.py " + failure_list_filename_ +
412                      " --remove succeeding_tests.txt")) {
413     ok = false;
414   }
415 
416   if (verbose_) {
417     CheckSetEmpty(skipped_, "",
418                   "These tests were skipped (probably because support for some "
419                   "features is not implemented)");
420   }
421 
422   StringAppendF(&output_,
423                 "CONFORMANCE SUITE %s: %d successes, %d skipped, "
424                 "%d expected failures, %d unexpected failures.\n",
425                 ok ? "PASSED" : "FAILED", successes_, skipped_.size(),
426                 expected_failures_, unexpected_failing_tests_.size());
427   StringAppendF(&output_, "\n");
428 
429   output->assign(output_);
430 
431   return ok;
432 }
433 
434 }  // namespace protobuf
435 }  // namespace google
436