1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 #include "google/protobuf/compiler/retention.h"
9
10 #include <memory>
11 #include <string>
12 #include <vector>
13
14 #include "google/protobuf/descriptor.pb.h"
15 #include <gtest/gtest.h>
16 #include "absl/strings/string_view.h"
17 #include "absl/strings/substitute.h"
18 #include "google/protobuf/compiler/parser.h"
19 #include "google/protobuf/dynamic_message.h"
20 #include "google/protobuf/io/tokenizer.h"
21 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
22 #include "google/protobuf/text_format.h"
23 #include "google/protobuf/unittest_retention.pb.h"
24 #include "google/protobuf/util/message_differencer.h"
25
26 namespace google {
27 namespace protobuf {
28 namespace internal {
29 namespace {
30
TEST(RetentionTest,DirectOptions)31 TEST(RetentionTest, DirectOptions) {
32 const FileOptions& file_options =
33 protobuf_unittest::OptionsMessage::descriptor()->file()->options();
34 EXPECT_EQ(file_options.GetExtension(protobuf_unittest::plain_option), 1);
35 EXPECT_EQ(
36 file_options.GetExtension(protobuf_unittest::runtime_retention_option), 2);
37 // RETENTION_SOURCE option should be stripped.
38 EXPECT_FALSE(
39 file_options.HasExtension(protobuf_unittest::source_retention_option));
40 EXPECT_EQ(file_options.GetExtension(protobuf_unittest::source_retention_option),
41 0);
42 }
43
CheckOptionsMessageIsStrippedCorrectly(const protobuf_unittest::OptionsMessage & options)44 void CheckOptionsMessageIsStrippedCorrectly(
45 const protobuf_unittest::OptionsMessage& options) {
46 EXPECT_EQ(options.plain_field(), 1);
47 EXPECT_EQ(options.runtime_retention_field(), 2);
48 // RETENTION_SOURCE field should be stripped.
49 EXPECT_FALSE(options.has_source_retention_field());
50 EXPECT_EQ(options.source_retention_field(), 0);
51 }
52
TEST(RetentionTest,FieldsNestedInRepeatedMessage)53 TEST(RetentionTest, FieldsNestedInRepeatedMessage) {
54 const FileOptions& file_options =
55 protobuf_unittest::OptionsMessage::descriptor()->file()->options();
56 ASSERT_EQ(1, file_options.ExtensionSize(protobuf_unittest::repeated_options));
57 const protobuf_unittest::OptionsMessage& options_message =
58 file_options.GetRepeatedExtension(protobuf_unittest::repeated_options)[0];
59 CheckOptionsMessageIsStrippedCorrectly(options_message);
60 }
61
TEST(RetentionTest,File)62 TEST(RetentionTest, File) {
63 CheckOptionsMessageIsStrippedCorrectly(
64 protobuf_unittest::OptionsMessage::descriptor()
65 ->file()
66 ->options()
67 .GetExtension(protobuf_unittest::file_option));
68 }
69
TEST(RetentionTest,TopLevelMessage)70 TEST(RetentionTest, TopLevelMessage) {
71 CheckOptionsMessageIsStrippedCorrectly(
72 protobuf_unittest::TopLevelMessage::descriptor()->options().GetExtension(
73 protobuf_unittest::message_option));
74 }
75
TEST(RetentionTest,NestedMessage)76 TEST(RetentionTest, NestedMessage) {
77 CheckOptionsMessageIsStrippedCorrectly(
78 protobuf_unittest::TopLevelMessage::NestedMessage::descriptor()
79 ->options()
80 .GetExtension(protobuf_unittest::message_option));
81 }
82
TEST(RetentionTest,TopLevelEnum)83 TEST(RetentionTest, TopLevelEnum) {
84 CheckOptionsMessageIsStrippedCorrectly(
85 protobuf_unittest::TopLevelEnum_descriptor()->options().GetExtension(
86 protobuf_unittest::enum_option));
87 }
88
TEST(RetentionTest,NestedEnum)89 TEST(RetentionTest, NestedEnum) {
90 CheckOptionsMessageIsStrippedCorrectly(
91 protobuf_unittest::TopLevelMessage::NestedEnum_descriptor()
92 ->options()
93 .GetExtension(protobuf_unittest::enum_option));
94 }
95
TEST(RetentionTest,EnumEntry)96 TEST(RetentionTest, EnumEntry) {
97 CheckOptionsMessageIsStrippedCorrectly(
98 protobuf_unittest::TopLevelEnum_descriptor()
99 ->value(0)
100 ->options()
101 .GetExtension(protobuf_unittest::enum_entry_option));
102 }
103
TEST(RetentionTest,TopLevelExtension)104 TEST(RetentionTest, TopLevelExtension) {
105 CheckOptionsMessageIsStrippedCorrectly(
106 protobuf_unittest::TopLevelMessage::descriptor()
107 ->file()
108 ->FindExtensionByName("i")
109 ->options()
110 .GetExtension(protobuf_unittest::field_option));
111 }
112
TEST(RetentionTest,NestedExtension)113 TEST(RetentionTest, NestedExtension) {
114 CheckOptionsMessageIsStrippedCorrectly(
115 protobuf_unittest::TopLevelMessage::descriptor()
116 ->extension(0)
117 ->options()
118 .GetExtension(protobuf_unittest::field_option));
119 }
120
TEST(RetentionTest,Field)121 TEST(RetentionTest, Field) {
122 CheckOptionsMessageIsStrippedCorrectly(
123 protobuf_unittest::TopLevelMessage::descriptor()
124 ->field(0)
125 ->options()
126 .GetExtension(protobuf_unittest::field_option));
127 }
128
TEST(RetentionTest,Oneof)129 TEST(RetentionTest, Oneof) {
130 CheckOptionsMessageIsStrippedCorrectly(
131 protobuf_unittest::TopLevelMessage::descriptor()
132 ->oneof_decl(0)
133 ->options()
134 .GetExtension(protobuf_unittest::oneof_option));
135 }
136
TEST(RetentionTest,ExtensionRange)137 TEST(RetentionTest, ExtensionRange) {
138 CheckOptionsMessageIsStrippedCorrectly(
139 protobuf_unittest::TopLevelMessage::descriptor()
140 ->extension_range(0)
141 ->options()
142 .GetExtension(protobuf_unittest::extension_range_option));
143 }
144
TEST(RetentionTest,Service)145 TEST(RetentionTest, Service) {
146 CheckOptionsMessageIsStrippedCorrectly(
147 protobuf_unittest::TopLevelMessage::descriptor()
148 ->file()
149 ->service(0)
150 ->options()
151 .GetExtension(protobuf_unittest::service_option));
152 }
153
TEST(RetentionTest,Method)154 TEST(RetentionTest, Method) {
155 CheckOptionsMessageIsStrippedCorrectly(
156 protobuf_unittest::TopLevelMessage::descriptor()
157 ->file()
158 ->service(0)
159 ->method(0)
160 ->options()
161 .GetExtension(protobuf_unittest::method_option));
162 }
163
164 class SimpleErrorCollector : public io::ErrorCollector {
165 public:
166 SimpleErrorCollector() = default;
RecordError(int line,io::ColumnNumber column,absl::string_view message)167 void RecordError(int line, io::ColumnNumber column,
168 absl::string_view message) override{};
169 };
170
TEST(RetentionTest,StripSourceRetentionOptionsWithSourceCodeInfo)171 TEST(RetentionTest, StripSourceRetentionOptionsWithSourceCodeInfo) {
172 // The tests above make assertions against the generated code, but this test
173 // case directly examines the result of the StripSourceRetentionOptions()
174 // function instead.
175 std::string proto_file =
176 absl::Substitute(R"(
177 syntax = "proto2";
178
179 package google.protobuf.internal;
180
181 import "$0";
182
183 option (source_retention_option) = 123;
184 option (options) = {
185 i1: 123
186 i2: 456
187 c { s: "abc" }
188 rc { s: "abc" }
189 };
190 option (repeated_options) = {
191 i1: 111 i2: 222
192 };
193
194 message Options {
195 optional int32 i1 = 1 [retention = RETENTION_SOURCE];
196 optional int32 i2 = 2;
197 message ChildMessage {
198 optional string s = 1 [retention = RETENTION_SOURCE];
199 }
200 optional ChildMessage c = 3;
201 repeated ChildMessage rc = 4;
202 }
203
204 extend google.protobuf.FileOptions {
205 optional int32 source_retention_option = 50000 [retention = RETENTION_SOURCE];
206 optional Options options = 50001;
207 repeated Options repeated_options = 50002;
208 })",
209 FileDescriptorSet::descriptor()->file()->name());
210 io::ArrayInputStream input_stream(proto_file.data(),
211 static_cast<int>(proto_file.size()));
212 SimpleErrorCollector error_collector;
213 io::Tokenizer tokenizer(&input_stream, &error_collector);
214 compiler::Parser parser;
215 FileDescriptorProto file_descriptor;
216 ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor));
217 file_descriptor.set_name("retention.proto");
218
219 DescriptorPool pool;
220 FileDescriptorProto descriptor_proto_descriptor;
221 FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor);
222 pool.BuildFile(descriptor_proto_descriptor);
223 pool.BuildFile(file_descriptor);
224
225 FileDescriptorProto stripped_file = compiler::StripSourceRetentionOptions(
226 *pool.FindFileByName("retention.proto"),
227 /*include_source_code_info=*/true);
228 EXPECT_EQ(stripped_file.source_code_info().location_size(), 63);
229 }
230
TEST(RetentionTest,RemoveEmptyOptions)231 TEST(RetentionTest, RemoveEmptyOptions) {
232 // If an options message is completely empty after stripping, that message
233 // should be removed.
234 std::string proto_file =
235 absl::Substitute(R"(
236 syntax = "proto2";
237
238 package google.protobuf.internal;
239
240 import "$0";
241
242 message Extendee {
243 extensions 1 to max [declaration = {
244 number: 1,
245 full_name: ".my.ext",
246 type: ".my.Message",
247 }];
248 })",
249 FileDescriptorSet::descriptor()->file()->name());
250 io::ArrayInputStream input_stream(proto_file.data(),
251 static_cast<int>(proto_file.size()));
252 SimpleErrorCollector error_collector;
253 io::Tokenizer tokenizer(&input_stream, &error_collector);
254 compiler::Parser parser;
255 FileDescriptorProto file_descriptor;
256 ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor));
257 file_descriptor.set_name("retention.proto");
258
259 DescriptorPool pool;
260 FileDescriptorProto descriptor_proto_descriptor;
261 FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor);
262 pool.BuildFile(descriptor_proto_descriptor);
263 pool.BuildFile(file_descriptor);
264
265 FileDescriptorProto stripped_file = compiler::StripSourceRetentionOptions(
266 *pool.FindFileByName("retention.proto"));
267 ASSERT_EQ(stripped_file.message_type_size(), 1);
268 ASSERT_EQ(stripped_file.message_type(0).extension_range_size(), 1);
269 EXPECT_FALSE(stripped_file.message_type(0).extension_range(0).has_options());
270 }
271
TEST(RetentionTest,InvalidDescriptor)272 TEST(RetentionTest, InvalidDescriptor) {
273 // This test creates an invalid descriptor and makes sure we can strip its
274 // options without crashing.
275 std::string proto_file =
276 absl::Substitute(R"(
277 syntax = "proto3";
278
279 package google.protobuf.internal;
280
281 import "$0";
282
283 // String option with invalid UTF-8
284 option (s) = "\xff";
285
286 extend google.protobuf.FileOptions {
287 optional string s = 50000;
288 })",
289 FileDescriptorSet::descriptor()->file()->name());
290 io::ArrayInputStream input_stream(proto_file.data(),
291 static_cast<int>(proto_file.size()));
292 SimpleErrorCollector error_collector;
293 io::Tokenizer tokenizer(&input_stream, &error_collector);
294 compiler::Parser parser;
295 FileDescriptorProto file_descriptor_proto;
296 ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor_proto));
297 file_descriptor_proto.set_name("retention.proto");
298
299 DescriptorPool pool;
300 FileDescriptorProto descriptor_proto_descriptor;
301 FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor);
302 ASSERT_NE(pool.BuildFile(descriptor_proto_descriptor), nullptr);
303 const FileDescriptor* file_descriptor = pool.BuildFile(file_descriptor_proto);
304 ASSERT_NE(file_descriptor, nullptr);
305
306 FileDescriptorProto stripped_file =
307 compiler::StripSourceRetentionOptions(*file_descriptor);
308 }
309
TEST(RetentionTest,MissingRequiredField)310 TEST(RetentionTest, MissingRequiredField) {
311 // Retention stripping should work correctly for a descriptor that has
312 // options with missing required fields.
313 std::string proto_file =
314 absl::Substitute(R"(
315 syntax = "proto2";
316
317 package google.protobuf.internal;
318
319 import "$0";
320
321 message WithRequiredField {
322 required int32 required_field = 1;
323 optional int32 optional_field = 2;
324 }
325
326 // Option with missing required field
327 option (m).optional_field = 42;
328
329 extend google.protobuf.FileOptions {
330 optional WithRequiredField m = 50000;
331 }
332
333 message Extendee {
334 extensions 1 to max [
335 declaration = {number: 1 full_name: ".my.ext" type: ".my.Type"}
336 ];
337 })",
338 FileDescriptorSet::descriptor()->file()->name());
339 io::ArrayInputStream input_stream(proto_file.data(),
340 static_cast<int>(proto_file.size()));
341 SimpleErrorCollector error_collector;
342 io::Tokenizer tokenizer(&input_stream, &error_collector);
343 compiler::Parser parser;
344 FileDescriptorProto file_descriptor_proto;
345 ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor_proto));
346 file_descriptor_proto.set_name("retention.proto");
347
348 DescriptorPool pool;
349 FileDescriptorProto descriptor_proto_descriptor;
350 FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor);
351 ASSERT_NE(pool.BuildFile(descriptor_proto_descriptor), nullptr);
352 const FileDescriptor* file_descriptor = pool.BuildFile(file_descriptor_proto);
353 ASSERT_NE(file_descriptor, nullptr);
354
355 FileDescriptorProto stripped_file =
356 compiler::StripSourceRetentionOptions(*file_descriptor);
357 ASSERT_EQ(stripped_file.message_type_size(), 2);
358 const DescriptorProto& extendee = stripped_file.message_type(1);
359 EXPECT_EQ(extendee.name(), "Extendee");
360 ASSERT_EQ(extendee.extension_range_size(), 1);
361 EXPECT_EQ(extendee.extension_range(0).options().declaration_size(), 0);
362 }
363
TEST(RetentionTest,InvalidRecursionDepth)364 TEST(RetentionTest, InvalidRecursionDepth) {
365 // The excessive nesting in this proto file will make it impossible for us to
366 // use a DynamicMessage to strip custom options, but we should still fall
367 // back to stripping built-in options (specifically extension declarations).
368 std::string proto_file =
369 absl::Substitute(R"(
370 syntax = "proto2";
371
372 package google.protobuf.internal;
373
374 import "$0";
375
376 message Recursive {
377 optional Recursive r = 1;
378 }
379
380 option (r).r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r
381 .r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r
382 .r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r.r
383 .r.r.r.r.r.r.r.r.r.r.r.r = {};
384
385 extend google.protobuf.FileOptions {
386 optional Recursive r = 50000;
387 }
388
389 message Extendee {
390 extensions 1 to max [
391 declaration = {number: 1 full_name: ".my.ext" type: ".my.Type"}
392 ];
393 })",
394 FileDescriptorSet::descriptor()->file()->name());
395 io::ArrayInputStream input_stream(proto_file.data(),
396 static_cast<int>(proto_file.size()));
397 SimpleErrorCollector error_collector;
398 io::Tokenizer tokenizer(&input_stream, &error_collector);
399 compiler::Parser parser;
400 FileDescriptorProto file_descriptor_proto;
401 ASSERT_TRUE(parser.Parse(&tokenizer, &file_descriptor_proto));
402 file_descriptor_proto.set_name("retention.proto");
403
404 DescriptorPool pool;
405 FileDescriptorProto descriptor_proto_descriptor;
406 FileDescriptorSet::descriptor()->file()->CopyTo(&descriptor_proto_descriptor);
407 ASSERT_NE(pool.BuildFile(descriptor_proto_descriptor), nullptr);
408 const FileDescriptor* file_descriptor = pool.BuildFile(file_descriptor_proto);
409 ASSERT_NE(file_descriptor, nullptr);
410
411 FileDescriptorProto stripped_file =
412 compiler::StripSourceRetentionOptions(*file_descriptor);
413 ASSERT_EQ(stripped_file.message_type_size(), 2);
414 const DescriptorProto& extendee = stripped_file.message_type(1);
415 EXPECT_EQ(extendee.name(), "Extendee");
416 ASSERT_EQ(extendee.extension_range_size(), 1);
417 EXPECT_EQ(extendee.extension_range(0).options().declaration_size(), 0);
418 }
419
420 } // namespace
421 } // namespace internal
422 } // namespace protobuf
423 } // namespace google
424