1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2023 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 #include "google/protobuf/compiler/code_generator.h"
9
10 #include <cstdint>
11 #include <string>
12 #include <vector>
13
14 #include "google/protobuf/descriptor.pb.h"
15 #include <gmock/gmock.h>
16 #include <gtest/gtest.h>
17 #include "absl/log/absl_log.h"
18 #include "absl/status/status.h"
19 #include "absl/strings/str_format.h"
20 #include "absl/strings/str_replace.h"
21 #include "absl/strings/string_view.h"
22 #include "google/protobuf/compiler/parser.h"
23 #include "google/protobuf/io/tokenizer.h"
24 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
25 #include "google/protobuf/test_textproto.h"
26 #include "google/protobuf/unittest_features.pb.h"
27
28 // Must be included last.
29 #include "google/protobuf/port_def.inc"
30
31 namespace google {
32 namespace protobuf {
33 namespace compiler {
34 namespace {
35
36 #define ASSERT_OK(x) ASSERT_TRUE(x.ok()) << x.message();
37
38 using ::testing::HasSubstr;
39 using ::testing::NotNull;
40
41 class TestGenerator : public CodeGenerator {
42 public:
Generate(const FileDescriptor * file,const std::string & parameter,GeneratorContext * generator_context,std::string * error) const43 bool Generate(const FileDescriptor* file, const std::string& parameter,
44 GeneratorContext* generator_context,
45 std::string* error) const override {
46 return true;
47 }
48
GetSupportedFeatures() const49 uint64_t GetSupportedFeatures() const override { return features_; }
set_supported_features(uint64_t features)50 void set_supported_features(uint64_t features) { features_ = features; }
51
GetFeatureExtensions() const52 std::vector<const FieldDescriptor*> GetFeatureExtensions() const override {
53 return feature_extensions_;
54 }
set_feature_extensions(std::vector<const FieldDescriptor * > extensions)55 void set_feature_extensions(std::vector<const FieldDescriptor*> extensions) {
56 feature_extensions_ = extensions;
57 }
58
GetMinimumEdition() const59 Edition GetMinimumEdition() const override { return minimum_edition_; }
set_minimum_edition(Edition minimum_edition)60 void set_minimum_edition(Edition minimum_edition) {
61 minimum_edition_ = minimum_edition;
62 }
63
GetMaximumEdition() const64 Edition GetMaximumEdition() const override { return maximum_edition_; }
set_maximum_edition(Edition maximum_edition)65 void set_maximum_edition(Edition maximum_edition) {
66 maximum_edition_ = maximum_edition;
67 }
68
69 // Expose the protected methods for testing.
70 using CodeGenerator::GetResolvedSourceFeatures;
71 using CodeGenerator::GetUnresolvedSourceFeatures;
72
73 private:
74 uint64_t features_ = CodeGenerator::Feature::FEATURE_SUPPORTS_EDITIONS;
75 Edition minimum_edition_ = MinimumAllowedEdition();
76 Edition maximum_edition_ = MaximumAllowedEdition();
77 std::vector<const FieldDescriptor*> feature_extensions_ = {
78 GetExtensionReflection(pb::test)};
79 };
80
81 class SimpleErrorCollector : public io::ErrorCollector {
82 public:
RecordError(int line,int column,absl::string_view message)83 void RecordError(int line, int column, absl::string_view message) override {
84 ABSL_LOG(ERROR) << absl::StrFormat("%d:%d:%s", line, column, message);
85 }
86 };
87
88 class CodeGeneratorTest : public ::testing::Test {
89 protected:
BuildFile(absl::string_view schema)90 const FileDescriptor* BuildFile(absl::string_view schema) {
91 io::ArrayInputStream input_stream(schema.data(),
92 static_cast<int>(schema.size()));
93 SimpleErrorCollector error_collector;
94 io::Tokenizer tokenizer(&input_stream, &error_collector);
95 Parser parser;
96 parser.RecordErrorsTo(&error_collector);
97 FileDescriptorProto proto;
98 ABSL_CHECK(parser.Parse(&tokenizer, &proto)) << schema;
99 proto.set_name("test.proto");
100 return pool_.BuildFile(proto);
101 }
102
BuildFile(const FileDescriptor * file)103 const FileDescriptor* BuildFile(const FileDescriptor* file) {
104 FileDescriptorProto proto;
105 file->CopyTo(&proto);
106 return pool_.BuildFile(proto);
107 }
108
109 DescriptorPool pool_;
110 };
111
TEST_F(CodeGeneratorTest,GetUnresolvedSourceFeaturesRoot)112 TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesRoot) {
113 ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
114 ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
115 auto file = BuildFile(R"schema(
116 edition = "2023";
117 package protobuf_unittest;
118
119 import "google/protobuf/unittest_features.proto";
120
121 option features.field_presence = EXPLICIT; // 2023 default
122 option features.enum_type = CLOSED; // override
123 option features.(pb.test).file_feature = VALUE5;
124 option features.(pb.test).source_feature = VALUE6;
125 )schema");
126 ASSERT_THAT(file, NotNull());
127
128 EXPECT_THAT(TestGenerator::GetUnresolvedSourceFeatures(*file, pb::test),
129 google::protobuf::EqualsProto(R"pb(
130 file_feature: VALUE5
131 source_feature: VALUE6
132 )pb"));
133 }
134
TEST_F(CodeGeneratorTest,GetUnresolvedSourceFeaturesInherited)135 TEST_F(CodeGeneratorTest, GetUnresolvedSourceFeaturesInherited) {
136 ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
137 ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
138 auto file = BuildFile(R"schema(
139 edition = "2023";
140 package protobuf_unittest;
141
142 import "google/protobuf/unittest_features.proto";
143
144 option features.enum_type = OPEN;
145 option features.(pb.test).file_feature = VALUE4;
146 message EditionsMessage {
147 option features.(pb.test).message_feature = VALUE5;
148 option features.(pb.test).multiple_feature = VALUE6;
149
150 string field = 1 [
151 features.field_presence = EXPLICIT,
152 features.(pb.test).multiple_feature = VALUE3,
153 features.(pb.test).source_feature = VALUE2
154 ];
155 }
156 )schema");
157 ASSERT_THAT(file, NotNull());
158
159 const FieldDescriptor* field =
160 file->FindMessageTypeByName("EditionsMessage")->FindFieldByName("field");
161 ASSERT_THAT(field, NotNull());
162
163 EXPECT_THAT(TestGenerator::GetUnresolvedSourceFeatures(*field, pb::test),
164 google::protobuf::EqualsProto(R"pb(
165 multiple_feature: VALUE3
166 source_feature: VALUE2
167 )pb"));
168 }
169
TEST_F(CodeGeneratorTest,GetResolvedSourceFeaturesRoot)170 TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesRoot) {
171 TestGenerator generator;
172 generator.set_feature_extensions({GetExtensionReflection(pb::test)});
173 ASSERT_OK(pool_.SetFeatureSetDefaults(*generator.BuildFeatureSetDefaults()));
174
175 ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
176 ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
177 auto file = BuildFile(R"schema(
178 edition = "2023";
179 package protobuf_unittest;
180
181 import "google/protobuf/unittest_features.proto";
182
183 option features.field_presence = EXPLICIT; // 2023 default
184 option features.enum_type = CLOSED; // override
185 option features.(pb.test).file_feature = VALUE6;
186 option features.(pb.test).source_feature = VALUE5;
187 )schema");
188 ASSERT_THAT(file, NotNull());
189
190 const FeatureSet& features = TestGenerator::GetResolvedSourceFeatures(*file);
191 const pb::TestFeatures& ext = features.GetExtension(pb::test);
192
193 EXPECT_TRUE(features.has_repeated_field_encoding());
194 EXPECT_TRUE(features.field_presence());
195 EXPECT_EQ(features.field_presence(), FeatureSet::EXPLICIT);
196 EXPECT_EQ(features.enum_type(), FeatureSet::CLOSED);
197
198 EXPECT_EQ(ext.file_feature(), pb::EnumFeature::VALUE6);
199 EXPECT_EQ(ext.source_feature(), pb::EnumFeature::VALUE5);
200 EXPECT_EQ(ext.field_feature(), pb::EnumFeature::VALUE1);
201 }
202
TEST_F(CodeGeneratorTest,GetResolvedSourceFeaturesInherited)203 TEST_F(CodeGeneratorTest, GetResolvedSourceFeaturesInherited) {
204 TestGenerator generator;
205 generator.set_feature_extensions({GetExtensionReflection(pb::test)});
206 ASSERT_OK(pool_.SetFeatureSetDefaults(*generator.BuildFeatureSetDefaults()));
207
208 ASSERT_THAT(BuildFile(DescriptorProto::descriptor()->file()), NotNull());
209 ASSERT_THAT(BuildFile(pb::TestMessage::descriptor()->file()), NotNull());
210 auto file = BuildFile(R"schema(
211 edition = "2023";
212 package protobuf_unittest;
213
214 import "google/protobuf/unittest_features.proto";
215
216 option features.enum_type = CLOSED;
217 option features.(pb.test).source_feature = VALUE5;
218 option features.(pb.test).file_feature = VALUE6;
219 message EditionsMessage {
220 option features.(pb.test).message_feature = VALUE4;
221 option features.(pb.test).multiple_feature = VALUE3;
222 option features.(pb.test).source_feature2 = VALUE2;
223
224 string field = 1 [
225 features.field_presence = IMPLICIT,
226 features.(pb.test).multiple_feature = VALUE5,
227 features.(pb.test).source_feature2 = VALUE3
228 ];
229 }
230 )schema");
231 ASSERT_THAT(file, NotNull());
232
233 const FieldDescriptor* field =
234 file->FindMessageTypeByName("EditionsMessage")->FindFieldByName("field");
235 ASSERT_THAT(field, NotNull());
236 const FeatureSet& features = TestGenerator::GetResolvedSourceFeatures(*field);
237 const pb::TestFeatures& ext = features.GetExtension(pb::test);
238
239 EXPECT_EQ(features.enum_type(), FeatureSet::CLOSED);
240 EXPECT_EQ(features.field_presence(), FeatureSet::IMPLICIT);
241
242 EXPECT_EQ(ext.message_feature(), pb::EnumFeature::VALUE4);
243 EXPECT_EQ(ext.file_feature(), pb::EnumFeature::VALUE6);
244 EXPECT_EQ(ext.multiple_feature(), pb::EnumFeature::VALUE5);
245 EXPECT_EQ(ext.source_feature(), pb::EnumFeature::VALUE5);
246 EXPECT_EQ(ext.source_feature2(), pb::EnumFeature::VALUE3);
247 }
248
249 // TODO: Use the gtest versions once that's available in OSS.
250 MATCHER_P(HasError, msg_matcher, "") {
251 return arg.status().code() == absl::StatusCode::kFailedPrecondition &&
252 ExplainMatchResult(msg_matcher, arg.status().message(),
253 result_listener);
254 }
255 MATCHER_P(IsOkAndHolds, matcher, "") {
256 return arg.ok() && ExplainMatchResult(matcher, *arg, result_listener);
257 }
258
TEST_F(CodeGeneratorTest,BuildFeatureSetDefaultsInvalidExtension)259 TEST_F(CodeGeneratorTest, BuildFeatureSetDefaultsInvalidExtension) {
260 TestGenerator generator;
261 generator.set_feature_extensions({nullptr});
262 EXPECT_THAT(generator.BuildFeatureSetDefaults(),
263 HasError(HasSubstr("Unknown extension")));
264 }
265
TEST_F(CodeGeneratorTest,BuildFeatureSetDefaults)266 TEST_F(CodeGeneratorTest, BuildFeatureSetDefaults) {
267 TestGenerator generator;
268 generator.set_feature_extensions({});
269 generator.set_minimum_edition(EDITION_99997_TEST_ONLY);
270 generator.set_maximum_edition(EDITION_99999_TEST_ONLY);
271 EXPECT_THAT(generator.BuildFeatureSetDefaults(),
272 IsOkAndHolds(EqualsProto(R"pb(
273 defaults {
274 edition: EDITION_LEGACY
275 overridable_features {}
276 fixed_features {
277 field_presence: EXPLICIT
278 enum_type: CLOSED
279 repeated_field_encoding: EXPANDED
280 utf8_validation: NONE
281 message_encoding: LENGTH_PREFIXED
282 json_format: LEGACY_BEST_EFFORT
283 }
284 }
285 defaults {
286 edition: EDITION_PROTO3
287 overridable_features {}
288 fixed_features {
289 field_presence: IMPLICIT
290 enum_type: OPEN
291 repeated_field_encoding: PACKED
292 utf8_validation: VERIFY
293 message_encoding: LENGTH_PREFIXED
294 json_format: ALLOW
295 }
296 }
297 defaults {
298 edition: EDITION_2023
299 overridable_features {
300 field_presence: EXPLICIT
301 enum_type: OPEN
302 repeated_field_encoding: PACKED
303 utf8_validation: VERIFY
304 message_encoding: LENGTH_PREFIXED
305 json_format: ALLOW
306 }
307 fixed_features {}
308 }
309 minimum_edition: EDITION_99997_TEST_ONLY
310 maximum_edition: EDITION_99999_TEST_ONLY
311 )pb")));
312 }
313
TEST_F(CodeGeneratorTest,BuildFeatureSetDefaultsUnsupported)314 TEST_F(CodeGeneratorTest, BuildFeatureSetDefaultsUnsupported) {
315 TestGenerator generator;
316 generator.set_supported_features(0);
317 generator.set_feature_extensions({});
318 generator.set_minimum_edition(EDITION_99997_TEST_ONLY);
319 generator.set_maximum_edition(EDITION_99999_TEST_ONLY);
320 auto result = generator.BuildFeatureSetDefaults();
321
322 ASSERT_TRUE(result.ok()) << result.status().message();
323 EXPECT_EQ(result->minimum_edition(), MinimumAllowedEdition());
324 EXPECT_EQ(result->maximum_edition(), MaximumAllowedEdition());
325 }
326
TEST_F(CodeGeneratorTest,SupportedEditionRangeIsDense)327 TEST_F(CodeGeneratorTest, SupportedEditionRangeIsDense) {
328 for (int i = static_cast<int>(MinimumAllowedEdition());
329 i <= static_cast<int>(MaximumAllowedEdition()); ++i) {
330 EXPECT_TRUE(Edition_IsValid(i));
331 }
332 }
333
334 #include "google/protobuf/port_undef.inc"
335
336 } // namespace
337 } // namespace compiler
338 } // namespace protobuf
339 } // namespace google
340