1 // Protocol Buffers - Google's data interchange format 2 // Copyright 2008 Google Inc. All rights reserved. 3 // https://developers.google.com/protocol-buffers/ 4 // 5 // Redistribution and use in source and binary forms, with or without 6 // modification, are permitted provided that the following conditions are 7 // met: 8 // 9 // * Redistributions of source code must retain the above copyright 10 // notice, this list of conditions and the following disclaimer. 11 // * Redistributions in binary form must reproduce the above 12 // copyright notice, this list of conditions and the following disclaimer 13 // in the documentation and/or other materials provided with the 14 // distribution. 15 // * Neither the name of Google Inc. nor the names of its 16 // contributors may be used to endorse or promote products derived from 17 // this software without specific prior written permission. 18 // 19 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 31 32 // This file defines a protocol for running the conformance test suite 33 // in-process. In other words, the suite itself will run in the same process as 34 // the code under test. 35 // 36 // For pros and cons of this approach, please see conformance.proto. 37 38 #ifndef CONFORMANCE_CONFORMANCE_TEST_H 39 #define CONFORMANCE_CONFORMANCE_TEST_H 40 41 #include <functional> 42 #include <string> 43 #include <google/protobuf/stubs/common.h> 44 #include <google/protobuf/util/type_resolver.h> 45 #include <google/protobuf/wire_format_lite.h> 46 47 #include "third_party/jsoncpp/json.h" 48 49 namespace conformance { 50 class ConformanceRequest; 51 class ConformanceResponse; 52 class TestAllTypes; 53 } // namespace conformance 54 55 namespace google { 56 namespace protobuf { 57 58 class ConformanceTestRunner { 59 public: ~ConformanceTestRunner()60 virtual ~ConformanceTestRunner() {} 61 62 // Call to run a single conformance test. 63 // 64 // "input" is a serialized conformance.ConformanceRequest. 65 // "output" should be set to a serialized conformance.ConformanceResponse. 66 // 67 // If there is any error in running the test itself, set "runtime_error" in 68 // the response. 69 virtual void RunTest(const std::string& test_name, 70 const std::string& input, 71 std::string* output) = 0; 72 }; 73 74 // Class representing the test suite itself. To run it, implement your own 75 // class derived from ConformanceTestRunner and then write code like: 76 // 77 // class MyConformanceTestRunner : public ConformanceTestRunner { 78 // public: 79 // virtual void RunTest(...) { 80 // // INSERT YOUR FRAMEWORK-SPECIFIC CODE HERE. 81 // } 82 // }; 83 // 84 // int main() { 85 // MyConformanceTestRunner runner; 86 // google::protobuf::ConformanceTestSuite suite; 87 // 88 // std::string output; 89 // suite.RunSuite(&runner, &output); 90 // } 91 // 92 class ConformanceTestSuite { 93 public: ConformanceTestSuite()94 ConformanceTestSuite() : verbose_(false) {} 95 SetVerbose(bool verbose)96 void SetVerbose(bool verbose) { verbose_ = verbose; } 97 98 // Sets the list of tests that are expected to fail when RunSuite() is called. 99 // RunSuite() will fail unless the set of failing tests is exactly the same 100 // as this list. 101 // 102 // The filename here is *only* used to create/format useful error messages for 103 // how to update the failure list. We do NOT read this file at all. 104 void SetFailureList(const std::string& filename, 105 const std::vector<std::string>& failure_list); 106 107 // Run all the conformance tests against the given test runner. 108 // Test output will be stored in "output". 109 // 110 // Returns true if the set of failing tests was exactly the same as the 111 // failure list. If SetFailureList() was not called, returns true if all 112 // tests passed. 113 bool RunSuite(ConformanceTestRunner* runner, std::string* output); 114 115 private: 116 void ReportSuccess(const std::string& test_name); 117 void ReportFailure(const string& test_name, 118 const conformance::ConformanceRequest& request, 119 const conformance::ConformanceResponse& response, 120 const char* fmt, ...); 121 void ReportSkip(const string& test_name, 122 const conformance::ConformanceRequest& request, 123 const conformance::ConformanceResponse& response); 124 void RunTest(const std::string& test_name, 125 const conformance::ConformanceRequest& request, 126 conformance::ConformanceResponse* response); 127 void RunValidInputTest(const string& test_name, const string& input, 128 conformance::WireFormat input_format, 129 const string& equivalent_text_format, 130 conformance::WireFormat requested_output); 131 void RunValidJsonTest(const string& test_name, const string& input_json, 132 const string& equivalent_text_format); 133 void RunValidJsonTestWithProtobufInput(const string& test_name, 134 const conformance::TestAllTypes& input, 135 const string& equivalent_text_format); 136 137 typedef std::function<bool(const Json::Value&)> Validator; 138 void RunValidJsonTestWithValidator(const string& test_name, 139 const string& input_json, 140 const Validator& validator); 141 void ExpectParseFailureForJson(const string& test_name, 142 const string& input_json); 143 void ExpectSerializeFailureForJson(const string& test_name, 144 const string& text_format); 145 void ExpectParseFailureForProto(const std::string& proto, 146 const std::string& test_name); 147 void ExpectHardParseFailureForProto(const std::string& proto, 148 const std::string& test_name); 149 void TestPrematureEOFForType(google::protobuf::FieldDescriptor::Type type); 150 bool CheckSetEmpty(const set<string>& set_to_check, 151 const std::string& write_to_file, const std::string& msg); 152 ConformanceTestRunner* runner_; 153 int successes_; 154 int expected_failures_; 155 bool verbose_; 156 std::string output_; 157 std::string failure_list_filename_; 158 159 // The set of test names that are expected to fail in this run, but haven't 160 // failed yet. 161 std::set<std::string> expected_to_fail_; 162 163 // The set of test names that have been run. Used to ensure that there are no 164 // duplicate names in the suite. 165 std::set<std::string> test_names_; 166 167 // The set of tests that failed, but weren't expected to. 168 std::set<std::string> unexpected_failing_tests_; 169 170 // The set of tests that succeeded, but weren't expected to. 171 std::set<std::string> unexpected_succeeding_tests_; 172 173 // The set of tests that the testee opted out of; 174 std::set<std::string> skipped_; 175 176 google::protobuf::internal::scoped_ptr<google::protobuf::util::TypeResolver> 177 type_resolver_; 178 std::string type_url_; 179 }; 180 181 } // namespace protobuf 182 } // namespace google 183 184 #endif // CONFORMANCE_CONFORMANCE_TEST_H 185