• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <dirent.h>
18 #include <err.h>
19 #include <limits.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <sys/stat.h>
23 #include <sys/types.h>
24 #include <unistd.h>
25 
26 #if defined(__linux__)
27 #include <sched.h>
28 #endif
29 
30 #include <atomic>
31 #include <chrono>
32 #include <functional>
33 #include <iostream>
34 #include <map>
35 #include <memory>
36 #include <set>
37 #include <sstream>
38 #include <string>
39 #include <string_view>
40 #include <thread>
41 #include <unordered_map>
42 #include <vector>
43 
44 #include <llvm/ADT/StringRef.h>
45 
46 #include <android-base/file.h>
47 #include <android-base/macros.h>
48 #include <android-base/parseint.h>
49 #include <android-base/strings.h>
50 
51 #include "Arch.h"
52 #include "DeclarationDatabase.h"
53 #include "Driver.h"
54 #include "Preprocessor.h"
55 #include "SymbolDatabase.h"
56 #include "Utils.h"
57 #include "VFS.h"
58 
59 #include "versioner.h"
60 
61 using namespace std::chrono_literals;
62 using namespace std::string_literals;
63 
64 bool strict;
65 bool verbose;
66 bool add_include;
67 
68 static int getCpuCount();
69 static int max_thread_count = getCpuCount();
70 
getCpuCount()71 static int getCpuCount() {
72 #if defined(__linux__)
73   cpu_set_t cpu_set;
74   int rc = sched_getaffinity(getpid(), sizeof(cpu_set), &cpu_set);
75   if (rc != 0) {
76     err(1, "sched_getaffinity failed");
77   }
78   return CPU_COUNT(&cpu_set);
79 #else
80   return 1;
81 #endif
82 }
83 
84 namespace {
85 struct HeaderLocationInformation {
86   std::string header_path;
87   std::string dependency_dir;
88   // Absolute paths to ignore all children -- including subdirectories -- of.
89   std::unordered_set<std::string> ignored_directories;
90 };
91 }
92 
is_dir(const std::string & path)93 static bool is_dir(const std::string& path) {
94   struct stat st;
95   return stat(path.c_str(), &st) == 0 && S_ISDIR(st.st_mode);
96 }
97 
collectRequirements(const Arch & arch,const HeaderLocationInformation & location)98 static CompilationRequirements collectRequirements(const Arch& arch,
99                                                    const HeaderLocationInformation& location) {
100   std::vector<std::string> headers =
101       collectHeaders(location.header_path, location.ignored_directories);
102   std::vector<std::string> dependencies;
103 
104   if (is_dir(location.header_path)) {
105     dependencies.emplace_back(location.header_path);
106   }
107 
108   if (!location.dependency_dir.empty()) {
109     auto collect_children = [&dependencies](const std::string& dir_path) {
110       DIR* dir = opendir(dir_path.c_str());
111       if (!dir) {
112         err(1, "failed to open dependency directory '%s'", dir_path.c_str());
113       }
114 
115       struct dirent* dent;
116       while ((dent = readdir(dir))) {
117         if (dent->d_name[0] == '.') {
118           continue;
119         }
120 
121         // TODO: Resolve symlinks.
122         std::string dependency = dir_path + "/" + dent->d_name;
123 
124         struct stat st;
125         if (stat(dependency.c_str(), &st) != 0) {
126           err(1, "failed to stat dependency '%s'", dependency.c_str());
127         }
128 
129         if (!S_ISDIR(st.st_mode)) {
130           errx(1, "'%s' is not a directory", dependency.c_str());
131         }
132 
133         dependencies.push_back(dependency);
134       }
135 
136       closedir(dir);
137     };
138 
139     collect_children(location.dependency_dir + "/common");
140     collect_children(location.dependency_dir + "/" + to_string(arch));
141   }
142 
143   auto new_end = std::remove_if(headers.begin(), headers.end(), [&arch](llvm::StringRef header) {
144     for (const auto& it : header_blacklist) {
145       if (it.second.find(arch) == it.second.end()) {
146         continue;
147       }
148 
149       if (header.endswith("/" + it.first)) {
150         return true;
151       }
152     }
153     return false;
154   });
155 
156   headers.erase(new_end, headers.end());
157 
158   CompilationRequirements result = { .headers = headers, .dependencies = dependencies };
159   return result;
160 }
161 
generateCompilationTypes(const std::set<Arch> selected_architectures,const std::set<int> & selected_levels)162 static std::set<CompilationType> generateCompilationTypes(const std::set<Arch> selected_architectures,
163                                                           const std::set<int>& selected_levels) {
164   std::set<CompilationType> result;
165   for (const auto& arch : selected_architectures) {
166     int min_api = arch_min_api[arch];
167     for (int api_level : selected_levels) {
168       if (api_level < min_api) {
169         continue;
170       }
171 
172       for (int file_offset_bits : { 32, 64 }) {
173         for (bool cpp : { true, false }) {
174           CompilationType type = {
175             .arch = arch, .cpp = cpp, .api_level = api_level, .file_offset_bits = file_offset_bits
176           };
177           result.insert(type);
178         }
179       }
180     }
181   }
182   return result;
183 }
184 
compileHeaders(const std::set<CompilationType> & types,const HeaderLocationInformation & location)185 static std::unique_ptr<HeaderDatabase> compileHeaders(const std::set<CompilationType>& types,
186                                                       const HeaderLocationInformation& location) {
187   if (types.empty()) {
188     errx(1, "compileHeaders received no CompilationTypes");
189   }
190 
191   auto vfs = createCommonVFS(location.header_path, location.dependency_dir, add_include);
192 
193   size_t thread_count = max_thread_count;
194   std::vector<std::thread> threads;
195 
196   std::map<CompilationType, HeaderDatabase> header_databases;
197   std::unordered_map<Arch, CompilationRequirements> requirements;
198 
199   auto result = std::make_unique<HeaderDatabase>();
200   for (const auto& type : types) {
201     if (requirements.count(type.arch) == 0) {
202       requirements[type.arch] = collectRequirements(type.arch, location);
203     }
204   }
205 
206   initializeTargetCC1FlagCache(vfs, types, requirements);
207 
208   std::vector<std::pair<CompilationType, const std::string&>> jobs;
209   std::atomic<size_t> job_index(0);
210   for (CompilationType type : types) {
211     CompilationRequirements& req = requirements[type.arch];
212     for (const std::string& header : req.headers) {
213       jobs.emplace_back(type, header);
214     }
215   }
216 
217   // Dup an empty file to stdin, so that we can use `clang -include a.h -` instead of `clang a.h`,
218   // since some warnings don't get generated in files that are compiled directly.
219   FILE* empty_file = tmpfile();
220   if (!empty_file) {
221     err(1, "failed to create temporary file");
222   }
223 
224   int empty_file_fd = fileno(empty_file);
225   if (empty_file_fd == -1) {
226     errx(1, "fileno failed on tmpfile");
227   }
228 
229   dup2(empty_file_fd, STDIN_FILENO);
230   fclose(empty_file);
231 
232   thread_count = std::min(thread_count, jobs.size());
233 
234   if (thread_count == 1) {
235     for (const auto& job : jobs) {
236       compileHeader(vfs, result.get(), job.first, job.second);
237     }
238   } else {
239     // Spawn threads.
240     for (size_t i = 0; i < thread_count; ++i) {
241       threads.emplace_back([&jobs, &job_index, &result, vfs]() {
242         while (true) {
243           size_t idx = job_index++;
244           if (idx >= jobs.size()) {
245             return;
246           }
247 
248           const auto& job = jobs[idx];
249           compileHeader(vfs, result.get(), job.first, job.second);
250         }
251       });
252     }
253 
254     // Reap them.
255     for (auto& thread : threads) {
256       thread.join();
257     }
258     threads.clear();
259   }
260 
261   return result;
262 }
263 
getCompilationTypes(const Declaration * decl)264 static std::set<CompilationType> getCompilationTypes(const Declaration* decl) {
265   std::set<CompilationType> result;
266   for (const auto& it : decl->availability) {
267     result.insert(it.first);
268   }
269   return result;
270 }
271 
272 template<typename T>
Intersection(const std::set<T> & a,const std::set<T> & b)273 static std::vector<T> Intersection(const std::set<T>& a, const std::set<T>& b) {
274   std::vector<T> intersection;
275   std::set_intersection(a.begin(), a.end(), b.begin(), b.end(), std::back_inserter(intersection));
276   return intersection;
277 }
278 
279 // Perform a sanity check on a symbol's declarations, enforcing the following invariants:
280 //   1. At most one inline definition of the function exists (overloaded inline functions for
281 //      _FORTIFY_SOURCE do not count because they are usually introduced to intercept the original
282 //      functions or usually have enable_if attributes).
283 //   2. All of the availability declarations for a symbol are compatible.
284 //      If a function is declared as an inline before a certain version, the inline definition
285 //      should have no version tag.
286 //   3. Each availability type must only be present globally or on a per-arch basis.
287 //      (e.g. __INTRODUCED_IN_ARM(9) __INTRODUCED_IN_X86(10) __DEPRECATED_IN(11) is fine,
288 //      but not __INTRODUCED_IN(9) __INTRODUCED_IN_X86(10))
checkSymbol(const Symbol & symbol)289 static bool checkSymbol(const Symbol& symbol) {
290   std::string cwd = getWorkingDir() + "/";
291 
292   std::unordered_map<const Declaration*, std::set<CompilationType>> inline_definitions;
293   for (const auto& decl_it : symbol.declarations) {
294     const Declaration* decl = &decl_it.second;
295     if (decl->is_definition && !decl->fortify_inline) {
296       std::set<CompilationType> compilation_types = getCompilationTypes(decl);
297       for (const auto& inline_def_it : inline_definitions) {
298         auto intersection = Intersection(compilation_types, inline_def_it.second);
299         if (!intersection.empty()) {
300           fprintf(stderr, "versioner: conflicting inline definitions for symbol %s:\n",
301                   symbol.name.c_str());
302           fprintf(stderr, "  declarations visible in: %s\n", Join(intersection, ", ").c_str());
303           decl->dump(cwd, stderr, 4);
304           inline_def_it.first->dump(cwd, stderr, 4);
305           return false;
306         }
307       }
308 
309       inline_definitions[decl] = std::move(compilation_types);
310     }
311 
312     DeclarationAvailability availability;
313     if (!decl->calculateAvailability(&availability)) {
314       fprintf(stderr, "versioner: failed to calculate availability for declaration:\n");
315       decl->dump(cwd, stderr, 2);
316       return false;
317     }
318 
319     if (decl->is_definition && !availability.empty()) {
320       fprintf(stderr, "versioner: inline definition has non-empty versioning information:\n");
321       decl->dump(cwd, stderr, 2);
322       return false;
323     }
324   }
325 
326   DeclarationAvailability availability;
327   if (!symbol.calculateAvailability(&availability)) {
328     fprintf(stderr, "versioner: inconsistent availability for symbol '%s'\n", symbol.name.c_str());
329     symbol.dump(cwd);
330     return false;
331   }
332 
333   // TODO: Check invariant #3.
334   return true;
335 }
336 
sanityCheck(const HeaderDatabase * database)337 static bool sanityCheck(const HeaderDatabase* database) {
338   bool error = false;
339   std::string cwd = getWorkingDir() + "/";
340 
341   for (const auto& symbol_it : database->symbols) {
342     if (!checkSymbol(symbol_it.second)) {
343       error = true;
344     }
345   }
346   return !error;
347 }
348 
349 // Check that our symbol availability declarations match the actual NDK
350 // platform symbol availability.
checkVersions(const std::set<CompilationType> & types,const HeaderDatabase * header_database,const NdkSymbolDatabase & symbol_database)351 static bool checkVersions(const std::set<CompilationType>& types,
352                           const HeaderDatabase* header_database,
353                           const NdkSymbolDatabase& symbol_database) {
354   std::string cwd = getWorkingDir() + "/";
355   bool failed = false;
356 
357   std::map<Arch, std::set<CompilationType>> arch_types;
358   for (const CompilationType& type : types) {
359     arch_types[type.arch].insert(type);
360   }
361 
362   std::set<std::string> completely_unavailable;
363   std::map<std::string, std::set<CompilationType>> missing_availability;
364   std::map<std::string, std::set<CompilationType>> extra_availability;
365 
366   for (const auto& symbol_it : header_database->symbols) {
367     const auto& symbol_name = symbol_it.first;
368     DeclarationAvailability symbol_availability;
369 
370     if (!symbol_it.second.calculateAvailability(&symbol_availability)) {
371       errx(1, "failed to calculate symbol availability");
372     }
373 
374     const auto platform_availability_it = symbol_database.find(symbol_name);
375     if (platform_availability_it == symbol_database.end()) {
376       completely_unavailable.insert(symbol_name);
377       continue;
378     }
379 
380     const auto& platform_availability = platform_availability_it->second;
381 
382     for (const CompilationType& type : types) {
383       bool should_be_available = true;
384       const auto& global_availability = symbol_availability.global_availability;
385       const auto& arch_availability = symbol_availability.arch_availability[type.arch];
386       if (global_availability.introduced != 0 && global_availability.introduced > type.api_level) {
387         should_be_available = false;
388       }
389 
390       if (arch_availability.introduced != 0 && arch_availability.introduced > type.api_level) {
391         should_be_available = false;
392       }
393 
394       if (global_availability.obsoleted != 0 && global_availability.obsoleted <= type.api_level) {
395         should_be_available = false;
396       }
397 
398       if (arch_availability.obsoleted != 0 && arch_availability.obsoleted <= type.api_level) {
399         should_be_available = false;
400       }
401 
402       // The function declaration might be (validly) missing for the given CompilationType.
403       if (!symbol_it.second.hasDeclaration(type)) {
404         should_be_available = false;
405       }
406 
407       bool is_available = platform_availability.count(type);
408 
409       if (should_be_available != is_available) {
410         if (is_available) {
411           extra_availability[symbol_name].insert(type);
412         } else {
413           missing_availability[symbol_name].insert(type);
414         }
415       }
416     }
417   }
418 
419   for (const auto& it : symbol_database) {
420     const std::string& symbol_name = it.first;
421 
422     bool symbol_error = false;
423     if (auto missing_it = missing_availability.find(symbol_name);
424         missing_it != missing_availability.end()) {
425       printf("%s: declaration marked available but symbol missing in [%s]\n", symbol_name.c_str(),
426              Join(missing_it->second, ", ").c_str());
427       symbol_error = true;
428       failed = true;
429     }
430 
431     if (strict) {
432       if (auto extra_it = extra_availability.find(symbol_name);
433           extra_it != extra_availability.end()) {
434         printf("%s: declaration marked unavailable but symbol available in [%s]\n",
435                symbol_name.c_str(), Join(extra_it->second, ", ").c_str());
436         symbol_error = true;
437         failed = true;
438       }
439     }
440 
441     if (symbol_error) {
442       if (auto symbol_it = header_database->symbols.find(symbol_name);
443           symbol_it != header_database->symbols.end()) {
444         symbol_it->second.dump(cwd);
445       } else {
446         errx(1, "failed to find symbol in header database");
447       }
448     }
449   }
450 
451   // TODO: Verify that function/variable declarations are actually function/variable symbols.
452   return !failed;
453 }
454 
usage(bool help=false)455 static void usage(bool help = false) {
456   fprintf(stderr, "Usage: versioner [OPTION]... [HEADER_PATH] [DEPS_PATH]\n");
457   if (!help) {
458     printf("Try 'versioner -h' for more information.\n");
459     exit(1);
460   } else {
461     fprintf(stderr, "Version headers at HEADER_PATH, with DEPS_PATH/ARCH/* on the include path\n");
462     fprintf(stderr, "Autodetects paths if HEADER_PATH and DEPS_PATH are not specified\n");
463     fprintf(stderr, "\n");
464     fprintf(stderr, "Target specification (defaults to all):\n");
465     fprintf(stderr, "  -a API_LEVEL\tbuild with specified API level (can be repeated)\n");
466     fprintf(stderr, "    \t\tdefaults to %s\n", Join(default_levels).c_str());
467     fprintf(stderr, "  -r ARCH\tbuild with specified architecture (can be repeated)\n");
468     fprintf(stderr, "    \t\tvalid architectures are %s\n", Join(supported_archs).c_str());
469     fprintf(stderr, "\n");
470     fprintf(stderr, "Validation:\n");
471     fprintf(stderr, "  -p PATH\tcompare against NDK platform at PATH\n");
472     fprintf(stderr, "  -s\t\tenable strict warnings\n");
473     fprintf(stderr, "\n");
474     fprintf(stderr, "Preprocessing:\n");
475     fprintf(stderr, "  -o PATH\tpreprocess header files and emit them at PATH\n");
476     fprintf(stderr, "  -f\t\tpreprocess header files even if validation fails\n");
477     fprintf(stderr, "\n");
478     fprintf(stderr, "Miscellaneous:\n");
479     fprintf(stderr, "  -F\t\tdo not ignore FORTIFY headers by default\n");
480     fprintf(stderr, "  -d\t\tdump function availability\n");
481     fprintf(stderr, "  -j THREADS\tmaximum number of threads to use\n");
482     fprintf(stderr, "  -v\t\tenable verbose logging\n");
483     fprintf(stderr, "  -h\t\tdisplay this message\n");
484     exit(0);
485   }
486 }
487 
488 // versioner uses a prebuilt version of clang, which is not up-to-date wrt/
489 // container annotations. So disable container overflow checking. b/37775238
__asan_default_options()490 extern "C" const char* __asan_default_options() {
491   return "detect_container_overflow=0";
492 }
493 
main(int argc,char ** argv)494 int main(int argc, char** argv) {
495   std::string cwd = getWorkingDir() + "/";
496   std::string platform_dir;
497   std::set<Arch> selected_architectures;
498   std::set<int> selected_levels;
499   std::string preprocessor_output_path;
500   bool force = false;
501   bool dump = false;
502   bool ignore_fortify_headers = true;
503 
504   int c;
505   while ((c = getopt(argc, argv, "a:r:p:so:fdj:vhFi")) != -1) {
506     switch (c) {
507       case 'a': {
508         char* end;
509         int api_level = strtol(optarg, &end, 10);
510         if (end == optarg || strlen(end) > 0) {
511           usage();
512         }
513 
514         selected_levels.insert(api_level);
515         break;
516       }
517 
518       case 'r': {
519         std::optional<Arch> arch = arch_from_string(optarg);
520         if (!arch) {
521           errx(1, "unknown architecture '%s'", optarg);
522         }
523         selected_architectures.insert(*arch);
524         break;
525       }
526 
527       case 'p': {
528         if (!platform_dir.empty()) {
529           usage();
530         }
531 
532         platform_dir = optarg;
533 
534         if (platform_dir.empty()) {
535           usage();
536         }
537 
538         struct stat st;
539         if (stat(platform_dir.c_str(), &st) != 0) {
540           err(1, "failed to stat platform directory '%s'", platform_dir.c_str());
541         }
542         if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) {
543           errx(1, "'%s' is not a file or directory", optarg);
544         }
545         break;
546       }
547 
548       case 's':
549         strict = true;
550         break;
551 
552       case 'o':
553         if (!preprocessor_output_path.empty()) {
554           usage();
555         }
556         preprocessor_output_path = optarg;
557         if (preprocessor_output_path.empty()) {
558           usage();
559         }
560         break;
561 
562       case 'f':
563         force = true;
564         break;
565 
566       case 'd':
567         dump = true;
568         break;
569 
570       case 'j':
571         if (!android::base::ParseInt<int>(optarg, &max_thread_count, 1)) {
572           usage();
573         }
574         break;
575 
576       case 'v':
577         verbose = true;
578         break;
579 
580       case 'h':
581         usage(true);
582         break;
583 
584       case 'i':
585         // Secret option for tests to -include <android/versioning.h>.
586         add_include = true;
587         break;
588 
589       case 'F':
590         ignore_fortify_headers = false;
591         break;
592 
593       default:
594         usage();
595         break;
596     }
597   }
598 
599   if (argc - optind > 2 || optind > argc) {
600     usage();
601   }
602 
603   HeaderLocationInformation location;
604 
605   const char* top = getenv("ANDROID_BUILD_TOP");
606   if (!top && (optind == argc || add_include)) {
607     fprintf(stderr, "versioner: failed to autodetect bionic paths. Is ANDROID_BUILD_TOP set?\n");
608     usage();
609   }
610 
611   if (optind == argc) {
612     // Neither HEADER_PATH nor DEPS_PATH were specified, so try to figure them out.
613     std::string versioner_dir = to_string(top) + "/bionic/tools/versioner";
614     location.header_path = versioner_dir + "/current";
615     location.dependency_dir = versioner_dir + "/dependencies";
616     if (platform_dir.empty()) {
617       platform_dir = versioner_dir + "/platforms";
618     }
619   } else {
620     if (!android::base::Realpath(argv[optind], &location.header_path)) {
621       err(1, "failed to get realpath for path '%s'", argv[optind]);
622     }
623 
624     if (argc - optind == 2) {
625       location.dependency_dir = argv[optind + 1];
626     }
627   }
628 
629   // Every file that lives in bits/fortify is logically a part of a header outside of bits/fortify.
630   // This makes the files there impossible to build on their own.
631   if (ignore_fortify_headers) {
632     std::string fortify_path = location.header_path;
633     if (!android::base::EndsWith(location.header_path, "/")) {
634       fortify_path += '/';
635     }
636     fortify_path += "bits/fortify";
637     location.ignored_directories.insert(std::move(fortify_path));
638   }
639 
640   if (selected_levels.empty()) {
641     selected_levels = default_levels;
642   }
643 
644   if (selected_architectures.empty()) {
645     selected_architectures = supported_archs;
646   }
647 
648 
649   struct stat st;
650   if (const char *path = location.header_path.c_str(); stat(path, &st) != 0) {
651     err(1, "failed to stat '%s'", path);
652   }
653 
654   std::set<CompilationType> compilation_types;
655   std::optional<NdkSymbolDatabase> symbol_database;
656 
657   compilation_types = generateCompilationTypes(selected_architectures, selected_levels);
658 
659   // Do this before compiling so that we can early exit if the platforms don't match what we
660   // expect.
661   if (!platform_dir.empty()) {
662     symbol_database = parsePlatforms(compilation_types, platform_dir);
663   }
664 
665   auto start = std::chrono::high_resolution_clock::now();
666   std::unique_ptr<HeaderDatabase> declaration_database =
667       compileHeaders(compilation_types, location);
668   auto end = std::chrono::high_resolution_clock::now();
669 
670   if (verbose) {
671     auto diff = (end - start) / 1.0ms;
672     printf("Compiled headers for %zu targets in %0.2LFms\n", compilation_types.size(), diff);
673   }
674 
675   bool failed = false;
676   if (dump) {
677     declaration_database->dump(location.header_path + "/");
678   } else {
679     if (!sanityCheck(declaration_database.get())) {
680       printf("versioner: sanity check failed\n");
681       failed = true;
682     }
683 
684     if (symbol_database) {
685       if (!checkVersions(compilation_types, declaration_database.get(), *symbol_database)) {
686         printf("versioner: version check failed\n");
687         failed = true;
688       }
689     }
690   }
691 
692   if (!preprocessor_output_path.empty() && (force || !failed)) {
693     failed = !preprocessHeaders(preprocessor_output_path, location.header_path,
694                                 declaration_database.get());
695   }
696   return failed;
697 }
698