1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // https://developers.google.com/protocol-buffers/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 32 // This file defines a protocol for running the conformance test suite 33 // in-process. In other words, the suite itself will run in the same process as 34 // the code under test. 35 // 36 // For pros and cons of this approach, please see conformance.proto. 37 38 #ifndef CONFORMANCE_CONFORMANCE_TEST_H 39 #define CONFORMANCE_CONFORMANCE_TEST_H 40 41 #include <functional> 42 #include <string> 43 #include <google/protobuf/stubs/common.h> 44 #include <google/protobuf/util/type_resolver.h> 45 #include <google/protobuf/wire_format_lite.h> 46 47 #include "third_party/jsoncpp/json.h" 48 49 namespace conformance { 50 class ConformanceRequest; 51 class ConformanceResponse; 52 class TestAllTypes; 53 } // namespace conformance 54 55 namespace google { 56 namespace protobuf { 57 58 class ConformanceTestRunner { 59 public: ~ConformanceTestRunner()60 virtual ~ConformanceTestRunner() {} 61 62 // Call to run a single conformance test. 63 // 64 // "input" is a serialized conformance.ConformanceRequest. 65 // "output" should be set to a serialized conformance.ConformanceResponse. 66 // 67 // If there is any error in running the test itself, set "runtime_error" in 68 // the response. 69 virtual void RunTest(const std::string& test_name, 70 const std::string& input, 71 std::string* output) = 0; 72 }; 73 74 // Class representing the test suite itself. To run it, implement your own 75 // class derived from ConformanceTestRunner and then write code like: 76 // 77 // class MyConformanceTestRunner : public ConformanceTestRunner { 78 // public: 79 // virtual void RunTest(...) { 80 // // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. 81 // } 82 // }; 83 // 84 // int main() { 85 // MyConformanceTestRunner runner; 86 // google::protobuf::ConformanceTestSuite suite; 87 // 88 // std::string output; 89 // suite.RunSuite(&runner, &output); 90 // } 91 // 92 class ConformanceTestSuite { 93 public: ConformanceTestSuite()94 ConformanceTestSuite() : verbose_(false) {} 95 SetVerbose(bool verbose)96 void SetVerbose(bool verbose) { verbose_ = verbose; } 97 98 // Sets the list of tests that are expected to fail when RunSuite() is called. 99 // RunSuite() will fail unless the set of failing tests is exactly the same 100 // as this list. 101 void SetFailureList(const std::vector<std::string>& failure_list); 102 103 // Run all the conformance tests against the given test runner. 104 // Test output will be stored in "output". 105 // 106 // Returns true if the set of failing tests was exactly the same as the 107 // failure list. If SetFailureList() was not called, returns true if all 108 // tests passed. 109 bool RunSuite(ConformanceTestRunner* runner, std::string* output); 110 111 private: 112 void ReportSuccess(const std::string& test_name); 113 void ReportFailure(const string& test_name, 114 const conformance::ConformanceRequest& request, 115 const conformance::ConformanceResponse& response, 116 const char* fmt, ...); 117 void ReportSkip(const string& test_name, 118 const conformance::ConformanceRequest& request, 119 const conformance::ConformanceResponse& response); 120 void RunTest(const std::string& test_name, 121 const conformance::ConformanceRequest& request, 122 conformance::ConformanceResponse* response); 123 void RunValidInputTest(const string& test_name, const string& input, 124 conformance::WireFormat input_format, 125 const string& equivalent_text_format, 126 conformance::WireFormat requested_output); 127 void RunValidJsonTest(const string& test_name, const string& input_json, 128 const string& equivalent_text_format); 129 void RunValidJsonTestWithProtobufInput(const string& test_name, 130 const conformance::TestAllTypes& input, 131 const string& equivalent_text_format); 132 133 typedef std::function<bool(const Json::Value&)> Validator; 134 void RunValidJsonTestWithValidator(const string& test_name, 135 const string& input_json, 136 const Validator& validator); 137 void ExpectParseFailureForJson(const string& test_name, 138 const string& input_json); 139 void ExpectSerializeFailureForJson(const string& test_name, 140 const string& text_format); 141 void ExpectParseFailureForProto(const std::string& proto, 142 const std::string& test_name); 143 void ExpectHardParseFailureForProto(const std::string& proto, 144 const std::string& test_name); 145 void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); 146 bool CheckSetEmpty(const set<string>& set_to_check, const char* msg); 147 ConformanceTestRunner* runner_; 148 int successes_; 149 int expected_failures_; 150 bool verbose_; 151 std::string output_; 152 153 // The set of test names that are expected to fail in this run, but haven't 154 // failed yet. 155 std::set<std::string> expected_to_fail_; 156 157 // The set of test names that have been run. Used to ensure that there are no 158 // duplicate names in the suite. 159 std::set<std::string> test_names_; 160 161 // The set of tests that failed, but weren't expected to. 162 std::set<std::string> unexpected_failing_tests_; 163 164 // The set of tests that succeeded, but weren't expected to. 165 std::set<std::string> unexpected_succeeding_tests_; 166 167 // The set of tests that the testee opted out of; 168 std::set<std::string> skipped_; 169 170 google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver> 171 type_resolver_; 172 std::string type_url_; 173 }; 174 175 } // namespace protobuf 176 } // namespace google 177 178 #endif // CONFORMANCE_CONFORMANCE_TEST_H 179