1 // Protocol Buffers - Google's data interchange format
2 // Copyright 2008 Google Inc. All rights reserved.
3 //
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file or at
6 // https://developers.google.com/open-source/licenses/bsd
7
8 // Author: kenton@google.com (Kenton Varda)
9 // Based on original Protocol Buffers design by
10 // Sanjay Ghemawat, Jeff Dean, and others.
11
12 #include "google/protobuf/compiler/importer.h"
13
14 #include <memory>
15
16 #include "google/protobuf/testing/file.h"
17 #include "google/protobuf/testing/file.h"
18 #include "google/protobuf/testing/file.h"
19 #include "google/protobuf/testing/googletest.h"
20 #include <gtest/gtest.h>
21 #include "absl/container/flat_hash_map.h"
22 #include "absl/log/absl_check.h"
23 #include "absl/status/status.h"
24 #include "absl/strings/str_cat.h"
25 #include "absl/strings/substitute.h"
26 #include "google/protobuf/descriptor.h"
27 #include "google/protobuf/io/zero_copy_stream_impl.h"
28
29 namespace google {
30 namespace protobuf {
31 namespace compiler {
32
33 namespace {
34
FileExists(const std::string & path)35 bool FileExists(const std::string& path) {
36 return File::Exists(path);
37 }
38
39 #define EXPECT_SUBSTRING(needle, haystack) \
40 EXPECT_PRED_FORMAT2(testing::IsSubstring, (needle), (haystack))
41
42 class MockErrorCollector : public MultiFileErrorCollector {
43 public:
MockErrorCollector()44 MockErrorCollector() {}
~MockErrorCollector()45 ~MockErrorCollector() override {}
46
47 std::string text_;
48 std::string warning_text_;
49
50 // implements ErrorCollector ---------------------------------------
RecordError(absl::string_view filename,int line,int column,absl::string_view message)51 void RecordError(absl::string_view filename, int line, int column,
52 absl::string_view message) override {
53 absl::SubstituteAndAppend(&text_, "$0:$1:$2: $3\n", filename, line, column,
54 message);
55 }
56
RecordWarning(absl::string_view filename,int line,int column,absl::string_view message)57 void RecordWarning(absl::string_view filename, int line, int column,
58 absl::string_view message) override {
59 absl::SubstituteAndAppend(&warning_text_, "$0:$1:$2: $3\n", filename, line,
60 column, message);
61 }
62 };
63
64 // -------------------------------------------------------------------
65
66 // A dummy implementation of SourceTree backed by a simple map.
67 class MockSourceTree : public SourceTree {
68 public:
MockSourceTree()69 MockSourceTree() {}
~MockSourceTree()70 ~MockSourceTree() override {}
71
AddFile(absl::string_view name,const char * contents)72 void AddFile(absl::string_view name, const char* contents) {
73 files_[name] = contents;
74 }
75
76 // implements SourceTree -------------------------------------------
Open(absl::string_view filename)77 io::ZeroCopyInputStream* Open(absl::string_view filename) override {
78 auto it = files_.find(filename);
79 if (it == files_.end()) return nullptr;
80 return new io::ArrayInputStream(it->second,
81 static_cast<int>(strlen(it->second)));
82 }
83
GetLastErrorMessage()84 std::string GetLastErrorMessage() override { return "File not found."; }
85
86 private:
87 absl::flat_hash_map<std::string, const char*> files_;
88 };
89
90 // ===================================================================
91
92 class ImporterTest : public testing::Test {
93 protected:
ImporterTest()94 ImporterTest() : importer_(&source_tree_, &error_collector_) {}
95
AddFile(const std::string & filename,const char * text)96 void AddFile(const std::string& filename, const char* text) {
97 source_tree_.AddFile(filename, text);
98 }
99
100 // Return the collected error text
warning() const101 std::string warning() const { return error_collector_.warning_text_; }
102
103 MockErrorCollector error_collector_;
104 MockSourceTree source_tree_;
105 Importer importer_;
106 };
107
TEST_F(ImporterTest,Import)108 TEST_F(ImporterTest, Import) {
109 // Test normal importing.
110 AddFile("foo.proto",
111 "syntax = \"proto2\";\n"
112 "message Foo {}\n");
113
114 const FileDescriptor* file = importer_.Import("foo.proto");
115 EXPECT_EQ("", error_collector_.text_);
116 ASSERT_TRUE(file != nullptr);
117
118 ASSERT_EQ(1, file->message_type_count());
119 EXPECT_EQ("Foo", file->message_type(0)->name());
120
121 // Importing again should return same object.
122 EXPECT_EQ(file, importer_.Import("foo.proto"));
123 }
124
TEST_F(ImporterTest,ImportNested)125 TEST_F(ImporterTest, ImportNested) {
126 // Test that importing a file which imports another file works.
127 AddFile("foo.proto",
128 "syntax = \"proto2\";\n"
129 "import \"bar.proto\";\n"
130 "message Foo {\n"
131 " optional Bar bar = 1;\n"
132 "}\n");
133 AddFile("bar.proto",
134 "syntax = \"proto2\";\n"
135 "message Bar {}\n");
136
137 // Note that both files are actually parsed by the first call to Import()
138 // here, since foo.proto imports bar.proto. The second call just returns
139 // the same ProtoFile for bar.proto which was constructed while importing
140 // foo.proto. We test that this is the case below by checking that bar
141 // is among foo's dependencies (by pointer).
142 const FileDescriptor* foo = importer_.Import("foo.proto");
143 const FileDescriptor* bar = importer_.Import("bar.proto");
144 EXPECT_EQ("", error_collector_.text_);
145 ASSERT_TRUE(foo != nullptr);
146 ASSERT_TRUE(bar != nullptr);
147
148 // Check that foo's dependency is the same object as bar.
149 ASSERT_EQ(1, foo->dependency_count());
150 EXPECT_EQ(bar, foo->dependency(0));
151
152 // Check that foo properly cross-links bar.
153 ASSERT_EQ(1, foo->message_type_count());
154 ASSERT_EQ(1, bar->message_type_count());
155 ASSERT_EQ(1, foo->message_type(0)->field_count());
156 ASSERT_EQ(FieldDescriptor::TYPE_MESSAGE,
157 foo->message_type(0)->field(0)->type());
158 EXPECT_EQ(bar->message_type(0),
159 foo->message_type(0)->field(0)->message_type());
160 }
161
TEST_F(ImporterTest,FileNotFound)162 TEST_F(ImporterTest, FileNotFound) {
163 // Error: Parsing a file that doesn't exist.
164 EXPECT_TRUE(importer_.Import("foo.proto") == nullptr);
165 EXPECT_EQ("foo.proto:-1:0: File not found.\n", error_collector_.text_);
166 }
167
TEST_F(ImporterTest,ImportNotFound)168 TEST_F(ImporterTest, ImportNotFound) {
169 // Error: Importing a file that doesn't exist.
170 AddFile("foo.proto",
171 "syntax = \"proto2\";\n"
172 "import \"bar.proto\";\n");
173
174 EXPECT_TRUE(importer_.Import("foo.proto") == nullptr);
175 EXPECT_EQ(
176 "bar.proto:-1:0: File not found.\n"
177 "foo.proto:1:0: Import \"bar.proto\" was not found or had errors.\n",
178 error_collector_.text_);
179 }
180
TEST_F(ImporterTest,RecursiveImport)181 TEST_F(ImporterTest, RecursiveImport) {
182 // Error: Recursive import.
183 AddFile("recursive1.proto",
184 "syntax = \"proto2\";\n"
185 "\n"
186 "import \"recursive2.proto\";\n");
187 AddFile("recursive2.proto",
188 "syntax = \"proto2\";\n"
189 "import \"recursive1.proto\";\n");
190
191 EXPECT_TRUE(importer_.Import("recursive1.proto") == nullptr);
192 EXPECT_EQ(
193 "recursive1.proto:2:0: File recursively imports itself: "
194 "recursive1.proto "
195 "-> recursive2.proto -> recursive1.proto\n"
196 "recursive2.proto:1:0: Import \"recursive1.proto\" was not found "
197 "or had errors.\n"
198 "recursive1.proto:2:0: Import \"recursive2.proto\" was not found "
199 "or had errors.\n",
200 error_collector_.text_);
201 }
202
TEST_F(ImporterTest,RecursiveImportSelf)203 TEST_F(ImporterTest, RecursiveImportSelf) {
204 // Error: Recursive import.
205 AddFile("recursive.proto",
206 "syntax = \"proto2\";\n"
207 "\n"
208 "import \"recursive.proto\";\n");
209
210 EXPECT_TRUE(importer_.Import("recursive.proto") == nullptr);
211 EXPECT_EQ(
212 "recursive.proto:2:0: File recursively imports itself: "
213 "recursive.proto -> recursive.proto\n",
214 error_collector_.text_);
215 }
216
TEST_F(ImporterTest,LiteRuntimeImport)217 TEST_F(ImporterTest, LiteRuntimeImport) {
218 // Error: Recursive import.
219 AddFile("bar.proto",
220 "syntax = \"proto2\";\n"
221 "option optimize_for = LITE_RUNTIME;\n");
222 AddFile("foo.proto",
223 "syntax = \"proto2\";\n"
224 "import \"bar.proto\";\n");
225
226 EXPECT_TRUE(importer_.Import("foo.proto") == nullptr);
227 EXPECT_EQ(
228 "foo.proto:1:0: Files that do not use optimize_for = LITE_RUNTIME "
229 "cannot import files which do use this option. This file is not "
230 "lite, but it imports \"bar.proto\" which is.\n",
231 error_collector_.text_);
232 }
233
234
235 // ===================================================================
236
237 class DiskSourceTreeTest : public testing::Test {
238 protected:
SetUp()239 void SetUp() override {
240 dirnames_.push_back(
241 absl::StrCat(TestTempDir(), "/test_proto2_import_path_1"));
242 dirnames_.push_back(
243 absl::StrCat(TestTempDir(), "/test_proto2_import_path_2"));
244
245 for (int i = 0; i < dirnames_.size(); i++) {
246 if (FileExists(dirnames_[i])) {
247 File::DeleteRecursively(dirnames_[i], NULL, NULL);
248 }
249 ABSL_CHECK_OK(File::CreateDir(dirnames_[i], 0777));
250 }
251 }
252
TearDown()253 void TearDown() override {
254 for (int i = 0; i < dirnames_.size(); i++) {
255 if (FileExists(dirnames_[i])) {
256 File::DeleteRecursively(dirnames_[i], NULL, NULL);
257 }
258 }
259 }
260
AddFile(const std::string & filename,const char * contents)261 void AddFile(const std::string& filename, const char* contents) {
262 ABSL_CHECK_OK(File::SetContents(filename, contents, true));
263 }
264
AddSubdir(const std::string & dirname)265 void AddSubdir(const std::string& dirname) {
266 ABSL_CHECK_OK(File::CreateDir(dirname, 0777));
267 }
268
ExpectFileContents(const std::string & filename,const char * expected_contents)269 void ExpectFileContents(const std::string& filename,
270 const char* expected_contents) {
271 std::unique_ptr<io::ZeroCopyInputStream> input(source_tree_.Open(filename));
272
273 ASSERT_FALSE(input == nullptr);
274
275 // Read all the data from the file.
276 std::string file_contents;
277 const void* data;
278 int size;
279 while (input->Next(&data, &size)) {
280 file_contents.append(reinterpret_cast<const char*>(data), size);
281 }
282
283 EXPECT_EQ(expected_contents, file_contents);
284 }
285
ExpectCannotOpenFile(const std::string & filename,const std::string & error_message)286 void ExpectCannotOpenFile(const std::string& filename,
287 const std::string& error_message) {
288 std::unique_ptr<io::ZeroCopyInputStream> input(source_tree_.Open(filename));
289 EXPECT_TRUE(input == nullptr);
290 EXPECT_EQ(error_message, source_tree_.GetLastErrorMessage());
291 }
292
293 DiskSourceTree source_tree_;
294
295 // Paths of two on-disk directories to use during the test.
296 std::vector<std::string> dirnames_;
297 };
298
TEST_F(DiskSourceTreeTest,MapRoot)299 TEST_F(DiskSourceTreeTest, MapRoot) {
300 // Test opening a file in a directory that is mapped to the root of the
301 // source tree.
302 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
303 source_tree_.MapPath("", dirnames_[0]);
304
305 ExpectFileContents("foo", "Hello World!");
306 ExpectCannotOpenFile("bar", "File not found.");
307 }
308
TEST_F(DiskSourceTreeTest,MapDirectory)309 TEST_F(DiskSourceTreeTest, MapDirectory) {
310 // Test opening a file in a directory that is mapped to somewhere other
311 // than the root of the source tree.
312
313 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
314 source_tree_.MapPath("baz", dirnames_[0]);
315
316 ExpectFileContents("baz/foo", "Hello World!");
317 ExpectCannotOpenFile("baz/bar", "File not found.");
318 ExpectCannotOpenFile("foo", "File not found.");
319 ExpectCannotOpenFile("bar", "File not found.");
320
321 // Non-canonical file names should not work.
322 ExpectCannotOpenFile("baz//foo",
323 "Backslashes, consecutive slashes, \".\", or \"..\" are "
324 "not allowed in the virtual path");
325 ExpectCannotOpenFile("baz/../baz/foo",
326 "Backslashes, consecutive slashes, \".\", or \"..\" are "
327 "not allowed in the virtual path");
328 ExpectCannotOpenFile("baz/./foo",
329 "Backslashes, consecutive slashes, \".\", or \"..\" are "
330 "not allowed in the virtual path");
331 ExpectCannotOpenFile("baz/foo/", "File not found.");
332 }
333
TEST_F(DiskSourceTreeTest,NoParent)334 TEST_F(DiskSourceTreeTest, NoParent) {
335 // Test that we cannot open files in a parent of a mapped directory.
336
337 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
338 AddSubdir(absl::StrCat(dirnames_[0], "/bar"));
339 AddFile(absl::StrCat(dirnames_[0], "/bar/baz"), "Blah.");
340 source_tree_.MapPath("", absl::StrCat(dirnames_[0], "/bar"));
341
342 ExpectFileContents("baz", "Blah.");
343 ExpectCannotOpenFile("../foo",
344 "Backslashes, consecutive slashes, \".\", or \"..\" are "
345 "not allowed in the virtual path");
346 ExpectCannotOpenFile("../bar/baz",
347 "Backslashes, consecutive slashes, \".\", or \"..\" are "
348 "not allowed in the virtual path");
349 }
350
TEST_F(DiskSourceTreeTest,MapFile)351 TEST_F(DiskSourceTreeTest, MapFile) {
352 // Test opening a file that is mapped directly into the source tree.
353
354 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
355 source_tree_.MapPath("foo", absl::StrCat(dirnames_[0], "/foo"));
356
357 ExpectFileContents("foo", "Hello World!");
358 ExpectCannotOpenFile("bar", "File not found.");
359 }
360
TEST_F(DiskSourceTreeTest,SearchMultipleDirectories)361 TEST_F(DiskSourceTreeTest, SearchMultipleDirectories) {
362 // Test mapping and searching multiple directories.
363
364 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
365 AddFile(absl::StrCat(dirnames_[1], "/foo"), "This file should be hidden.");
366 AddFile(absl::StrCat(dirnames_[1], "/bar"), "Goodbye World!");
367 source_tree_.MapPath("", dirnames_[0]);
368 source_tree_.MapPath("", dirnames_[1]);
369
370 ExpectFileContents("foo", "Hello World!");
371 ExpectFileContents("bar", "Goodbye World!");
372 ExpectCannotOpenFile("baz", "File not found.");
373 }
374
TEST_F(DiskSourceTreeTest,OrderingTrumpsSpecificity)375 TEST_F(DiskSourceTreeTest, OrderingTrumpsSpecificity) {
376 // Test that directories are always searched in order, even when a latter
377 // directory is more-specific than a former one.
378
379 // Create the "bar" directory so we can put a file in it.
380 ABSL_CHECK_OK(File::CreateDir(absl::StrCat(dirnames_[0], "/bar"),
381 0777));
382
383 // Add files and map paths.
384 AddFile(absl::StrCat(dirnames_[0], "/bar/foo"), "Hello World!");
385 AddFile(absl::StrCat(dirnames_[1], "/foo"), "This file should be hidden.");
386 source_tree_.MapPath("", dirnames_[0]);
387 source_tree_.MapPath("bar", dirnames_[1]);
388
389 // Check.
390 ExpectFileContents("bar/foo", "Hello World!");
391 }
392
TEST_F(DiskSourceTreeTest,DiskFileToVirtualFile)393 TEST_F(DiskSourceTreeTest, DiskFileToVirtualFile) {
394 // Test DiskFileToVirtualFile.
395
396 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
397 AddFile(absl::StrCat(dirnames_[1], "/foo"), "This file should be hidden.");
398 source_tree_.MapPath("bar", dirnames_[0]);
399 source_tree_.MapPath("bar", dirnames_[1]);
400
401 std::string virtual_file;
402 std::string shadowing_disk_file;
403
404 EXPECT_EQ(DiskSourceTree::NO_MAPPING,
405 source_tree_.DiskFileToVirtualFile("/foo", &virtual_file,
406 &shadowing_disk_file));
407
408 EXPECT_EQ(DiskSourceTree::SHADOWED, source_tree_.DiskFileToVirtualFile(
409 absl::StrCat(dirnames_[1], "/foo"),
410 &virtual_file, &shadowing_disk_file));
411 EXPECT_EQ("bar/foo", virtual_file);
412 EXPECT_EQ(absl::StrCat(dirnames_[0], "/foo"), shadowing_disk_file);
413
414 EXPECT_EQ(
415 DiskSourceTree::CANNOT_OPEN,
416 source_tree_.DiskFileToVirtualFile(absl::StrCat(dirnames_[1], "/baz"),
417 &virtual_file, &shadowing_disk_file));
418 EXPECT_EQ("bar/baz", virtual_file);
419
420 EXPECT_EQ(DiskSourceTree::SUCCESS, source_tree_.DiskFileToVirtualFile(
421 absl::StrCat(dirnames_[0], "/foo"),
422 &virtual_file, &shadowing_disk_file));
423 EXPECT_EQ("bar/foo", virtual_file);
424 }
425
TEST_F(DiskSourceTreeTest,DiskFileToVirtualFileCanonicalization)426 TEST_F(DiskSourceTreeTest, DiskFileToVirtualFileCanonicalization) {
427 // Test handling of "..", ".", etc. in DiskFileToVirtualFile().
428
429 source_tree_.MapPath("dir1", "..");
430 source_tree_.MapPath("dir2", "../../foo");
431 source_tree_.MapPath("dir3", "./foo/bar/.");
432 source_tree_.MapPath("dir4", ".");
433 source_tree_.MapPath("", "/qux");
434 source_tree_.MapPath("dir5", "/quux/");
435
436 std::string virtual_file;
437 std::string shadowing_disk_file;
438
439 // "../.." should not be considered to be under "..".
440 EXPECT_EQ(DiskSourceTree::NO_MAPPING,
441 source_tree_.DiskFileToVirtualFile("../../baz", &virtual_file,
442 &shadowing_disk_file));
443
444 // "/foo" is not mapped (it should not be misinterpreted as being under ".").
445 EXPECT_EQ(DiskSourceTree::NO_MAPPING,
446 source_tree_.DiskFileToVirtualFile("/foo", &virtual_file,
447 &shadowing_disk_file));
448
449 #ifdef WIN32
450 // "C:\foo" is not mapped (it should not be misinterpreted as being under
451 // ".").
452 EXPECT_EQ(DiskSourceTree::NO_MAPPING,
453 source_tree_.DiskFileToVirtualFile("C:\\foo", &virtual_file,
454 &shadowing_disk_file));
455 #endif // WIN32
456
457 // But "../baz" should be.
458 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
459 source_tree_.DiskFileToVirtualFile("../baz", &virtual_file,
460 &shadowing_disk_file));
461 EXPECT_EQ("dir1/baz", virtual_file);
462
463 // "../../foo/baz" is under "../../foo".
464 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
465 source_tree_.DiskFileToVirtualFile("../../foo/baz", &virtual_file,
466 &shadowing_disk_file));
467 EXPECT_EQ("dir2/baz", virtual_file);
468
469 // "foo/./bar/baz" is under "./foo/bar/.".
470 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
471 source_tree_.DiskFileToVirtualFile("foo/bar/baz", &virtual_file,
472 &shadowing_disk_file));
473 EXPECT_EQ("dir3/baz", virtual_file);
474
475 // "bar" is under ".".
476 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
477 source_tree_.DiskFileToVirtualFile("bar", &virtual_file,
478 &shadowing_disk_file));
479 EXPECT_EQ("dir4/bar", virtual_file);
480
481 // "/qux/baz" is under "/qux".
482 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
483 source_tree_.DiskFileToVirtualFile("/qux/baz", &virtual_file,
484 &shadowing_disk_file));
485 EXPECT_EQ("baz", virtual_file);
486
487 // "/quux/bar" is under "/quux".
488 EXPECT_EQ(DiskSourceTree::CANNOT_OPEN,
489 source_tree_.DiskFileToVirtualFile("/quux/bar", &virtual_file,
490 &shadowing_disk_file));
491 EXPECT_EQ("dir5/bar", virtual_file);
492 }
493
TEST_F(DiskSourceTreeTest,VirtualFileToDiskFile)494 TEST_F(DiskSourceTreeTest, VirtualFileToDiskFile) {
495 // Test VirtualFileToDiskFile.
496
497 AddFile(absl::StrCat(dirnames_[0], "/foo"), "Hello World!");
498 AddFile(absl::StrCat(dirnames_[1], "/foo"), "This file should be hidden.");
499 AddFile(absl::StrCat(dirnames_[1], "/quux"),
500 "This file should not be hidden.");
501 source_tree_.MapPath("bar", dirnames_[0]);
502 source_tree_.MapPath("bar", dirnames_[1]);
503
504 // Existent files, shadowed and non-shadowed case.
505 std::string disk_file;
506 EXPECT_TRUE(source_tree_.VirtualFileToDiskFile("bar/foo", &disk_file));
507 EXPECT_EQ(absl::StrCat(dirnames_[0], "/foo"), disk_file);
508 EXPECT_TRUE(source_tree_.VirtualFileToDiskFile("bar/quux", &disk_file));
509 EXPECT_EQ(absl::StrCat(dirnames_[1], "/quux"), disk_file);
510
511 // Nonexistent file in existent directory and vice versa.
512 std::string not_touched = "not touched";
513 EXPECT_FALSE(source_tree_.VirtualFileToDiskFile("bar/baz", ¬_touched));
514 EXPECT_EQ("not touched", not_touched);
515 EXPECT_FALSE(source_tree_.VirtualFileToDiskFile("baz/foo", ¬_touched));
516 EXPECT_EQ("not touched", not_touched);
517
518 // Accept NULL as output parameter.
519 EXPECT_TRUE(source_tree_.VirtualFileToDiskFile("bar/foo", nullptr));
520 EXPECT_FALSE(source_tree_.VirtualFileToDiskFile("baz/foo", nullptr));
521 }
522
523 } // namespace
524
525 } // namespace compiler
526 } // namespace protobuf
527 } // namespace google
528