1// Copyright 2017 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package build 16 17import ( 18 "bytes" 19 "io/ioutil" 20 "os" 21 "path/filepath" 22 "strings" 23 24 "android/soong/finder" 25 "android/soong/finder/fs" 26 "android/soong/ui/logger" 27 28 "android/soong/ui/metrics" 29) 30 31// This file provides an interface to the Finder type for soong_ui. Finder is 32// used to recursively traverse the source tree to gather paths of files, such 33// as Android.bp or Android.mk, and store the lists/database of paths in files 34// under `$OUT_DIR/.module_paths`. This directory can also be dist'd. 35 36// NewSourceFinder returns a new Finder configured to search for source files. 37// Callers of NewSourceFinder should call <f.Shutdown()> when done 38func NewSourceFinder(ctx Context, config Config) (f *finder.Finder) { 39 ctx.BeginTrace(metrics.RunSetupTool, "find modules") 40 defer ctx.EndTrace() 41 42 // Set up the working directory for the Finder. 43 dir, err := os.Getwd() 44 if err != nil { 45 ctx.Fatalf("No working directory for module-finder: %v", err.Error()) 46 } 47 filesystem := fs.OsFs 48 49 // .out-dir and .find-ignore are markers for Finder to ignore siblings and 50 // subdirectories of the directory Finder finds them in, hence stopping the 51 // search recursively down those branches. It's possible that these files 52 // are in the root directory, and if they are, then the subsequent error 53 // messages are very confusing, so check for that here. 54 pruneFiles := []string{".out-dir", ".find-ignore"} 55 for _, name := range pruneFiles { 56 prunePath := filepath.Join(dir, name) 57 _, statErr := filesystem.Lstat(prunePath) 58 if statErr == nil { 59 ctx.Fatalf("%v must not exist", prunePath) 60 } 61 } 62 63 // Set up configuration parameters for the Finder cache. 64 cacheParams := finder.CacheParams{ 65 WorkingDirectory: dir, 66 RootDirs: androidBpSearchDirs(config), 67 FollowSymlinks: config.environ.IsEnvTrue("ALLOW_BP_UNDER_SYMLINKS"), 68 ExcludeDirs: []string{".git", ".repo"}, 69 PruneFiles: pruneFiles, 70 IncludeFiles: []string{ 71 // Kati build definitions. 72 "Android.mk", 73 // Product configuration files. 74 "AndroidProducts.mk", 75 // General Soong build definitions, using the Blueprint syntax. 76 "Android.bp", 77 // Kati clean definitions. 78 "CleanSpec.mk", 79 // Ownership definition. 80 "OWNERS", 81 // Test configuration for modules in directories that contain this 82 // file. 83 "TEST_MAPPING", 84 // METADATA file of packages 85 "METADATA", 86 }, 87 IncludeSuffixes: []string{ 88 // .mk files for product/board configuration. 89 ".mk", 90 // otatools cert files 91 ".pk8", 92 ".pem", 93 ".avbpubkey", 94 }, 95 } 96 dumpDir := config.FileListDir() 97 f, err = finder.New(cacheParams, filesystem, logger.New(ioutil.Discard), 98 filepath.Join(dumpDir, "files.db")) 99 if err != nil { 100 ctx.Fatalf("Could not create module-finder: %v", err) 101 } 102 return f 103} 104 105func androidBpSearchDirs(config Config) []string { 106 dirs := []string{"."} // always search from root of source tree. 107 if config.searchApiDir { 108 // Search in out/api_surfaces 109 dirs = append(dirs, config.ApiSurfacesOutDir()) 110 } 111 return dirs 112} 113 114func findProductAndBoardConfigFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { 115 matches := []string{} 116 for _, foundName := range entries.FileNames { 117 if foundName != "Android.mk" && 118 foundName != "AndroidProducts.mk" && 119 foundName != "CleanSpec.mk" && 120 strings.HasSuffix(foundName, ".mk") { 121 matches = append(matches, foundName) 122 } 123 } 124 return entries.DirNames, matches 125} 126 127func findOtaToolsCertFiles(entries finder.DirEntries) (dirNames []string, fileNames []string) { 128 matches := []string{} 129 for _, foundName := range entries.FileNames { 130 if strings.HasSuffix(foundName, ".pk8") || 131 strings.HasSuffix(foundName, ".pem") || 132 strings.HasSuffix(foundName, ".avbpubkey") { 133 matches = append(matches, foundName) 134 } 135 } 136 return entries.DirNames, matches 137} 138 139// FindSources searches for source files known to <f> and writes them to the filesystem for 140// use later. 141func FindSources(ctx Context, config Config, f *finder.Finder) { 142 // note that dumpDir in FindSources may be different than dumpDir in NewSourceFinder 143 // if a caller such as multiproduct_kati wants to share one Finder among several builds 144 dumpDir := config.FileListDir() 145 os.MkdirAll(dumpDir, 0777) 146 147 // Stop searching a subdirectory recursively after finding an Android.mk. 148 androidMks := f.FindFirstNamedAt(".", "Android.mk") 149 androidMks = ignoreSomeAndroidMks(androidMks) 150 blockAndroidMks(ctx, androidMks) 151 err := dumpListToFile(ctx, config, androidMks, filepath.Join(dumpDir, "Android.mk.list")) 152 if err != nil { 153 ctx.Fatalf("Could not export module list: %v", err) 154 } 155 156 // Gate collecting/reporting mk metrics on builds that specifically request 157 // it, as identifying the total number of mk files adds 4-5ms onto null 158 // builds. 159 if config.reportMkMetrics { 160 androidMksTotal := f.FindNamedAt(".", "Android.mk") 161 162 ctx.Metrics.SetToplevelMakefiles(len(androidMks)) 163 ctx.Metrics.SetTotalMakefiles(len(androidMksTotal)) 164 ctx.Metrics.DumpMkMetrics(config.MkMetrics()) 165 } 166 167 // Stop searching a subdirectory recursively after finding a CleanSpec.mk. 168 cleanSpecs := f.FindFirstNamedAt(".", "CleanSpec.mk") 169 err = dumpListToFile(ctx, config, cleanSpecs, filepath.Join(dumpDir, "CleanSpec.mk.list")) 170 if err != nil { 171 ctx.Fatalf("Could not export module list: %v", err) 172 } 173 174 // Only consider AndroidProducts.mk in device/, vendor/ and product/, recursively in these directories. 175 androidProductsMks := f.FindNamedAt("device", "AndroidProducts.mk") 176 androidProductsMks = append(androidProductsMks, f.FindNamedAt("vendor", "AndroidProducts.mk")...) 177 androidProductsMks = append(androidProductsMks, f.FindNamedAt("product", "AndroidProducts.mk")...) 178 err = dumpListToFile(ctx, config, androidProductsMks, filepath.Join(dumpDir, "AndroidProducts.mk.list")) 179 if err != nil { 180 ctx.Fatalf("Could not export product list: %v", err) 181 } 182 183 // Recursively look for all OWNERS files. 184 owners := f.FindNamedAt(".", "OWNERS") 185 err = dumpListToFile(ctx, config, owners, filepath.Join(dumpDir, "OWNERS.list")) 186 if err != nil { 187 ctx.Fatalf("Could not find OWNERS: %v", err) 188 } 189 190 // Recursively look for all METADATA files. 191 metadataFiles := f.FindNamedAt(".", "METADATA") 192 metadataFiles = ignoreNonAndroidMetadataFiles(metadataFiles) 193 err = dumpListToFile(ctx, config, metadataFiles, filepath.Join(dumpDir, "METADATA.list")) 194 if err != nil { 195 ctx.Fatalf("Could not find METADATA: %v", err) 196 } 197 198 // Recursively look for all TEST_MAPPING files. 199 testMappings := f.FindNamedAt(".", "TEST_MAPPING") 200 err = dumpListToFile(ctx, config, testMappings, filepath.Join(dumpDir, "TEST_MAPPING.list")) 201 if err != nil { 202 ctx.Fatalf("Could not find TEST_MAPPING: %v", err) 203 } 204 205 // Recursively look for all otatools cert files. 206 otatools_cert_files := f.FindMatching("build/make/target/product/security", findOtaToolsCertFiles) 207 otatools_cert_files = append(otatools_cert_files, f.FindMatching("device", findOtaToolsCertFiles)...) 208 otatools_cert_files = append(otatools_cert_files, f.FindMatching("external/avb/test/data", findOtaToolsCertFiles)...) 209 otatools_cert_files = append(otatools_cert_files, f.FindMatching("packages/modules", findOtaToolsCertFiles)...) 210 otatools_cert_files = append(otatools_cert_files, f.FindMatching("vendor", findOtaToolsCertFiles)...) 211 err = dumpListToFile(ctx, config, otatools_cert_files, filepath.Join(dumpDir, "OtaToolsCertFiles.list")) 212 if err != nil { 213 ctx.Fatalf("Could not find otatools cert files: %v", err) 214 } 215 216 // Recursively look for all Android.bp files 217 androidBps := f.FindNamedAt(".", "Android.bp") 218 if len(androidBps) == 0 { 219 ctx.Fatalf("No Android.bp found") 220 } 221 err = dumpListToFile(ctx, config, androidBps, filepath.Join(dumpDir, "Android.bp.list")) 222 if err != nil { 223 ctx.Fatalf("Could not find modules: %v", err) 224 } 225 226 // Recursively look for all product/board config files. 227 configurationFiles := f.FindMatching(".", findProductAndBoardConfigFiles) 228 err = dumpListToFile(ctx, config, configurationFiles, filepath.Join(dumpDir, "configuration.list")) 229 if err != nil { 230 ctx.Fatalf("Could not export product/board configuration list: %v", err) 231 } 232 233 if config.Dist() { 234 f.WaitForDbDump() 235 // Dist the files.db plain text database. 236 distFile(ctx, config, f.DbPath, "module_paths") 237 } 238} 239 240// Write the .list files to disk. 241func dumpListToFile(ctx Context, config Config, list []string, filePath string) (err error) { 242 desiredText := strings.Join(list, "\n") 243 desiredBytes := []byte(desiredText) 244 actualBytes, readErr := ioutil.ReadFile(filePath) 245 if readErr != nil || !bytes.Equal(desiredBytes, actualBytes) { 246 err = ioutil.WriteFile(filePath, desiredBytes, 0777) 247 if err != nil { 248 return err 249 } 250 } 251 252 distFile(ctx, config, filePath, "module_paths") 253 254 return nil 255} 256 257func ignoreNonAndroidMetadataFiles(metadataFiles []string) []string { 258 result := make([]string, 0, len(metadataFiles)) 259 for _, file := range metadataFiles { 260 // Ignore files like prebuilts/clang/host/linux-x86/clang-r536225/python3/lib/python3.11/site-packages/pip-23.1.2.dist-info/METADATA 261 // these METADATA files are from upstream and are not the METADATA files used in Android codebase. 262 if strings.Contains(file, "prebuilts/clang/host/") && strings.Contains(file, "/site-packages/") { 263 continue 264 } 265 result = append(result, file) 266 } 267 return result 268} 269